freesolo-flash-dev 1.0.83__tar.gz → 1.0.85__tar.gz

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 1.0.83
+Version: 1.0.85
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/catalog.py

@@ -466,16 +466,18 @@ MODELS: dict[str, ModelInfo] = {
         # vllm 0.19.1 derives the 1,097,728-byte gdn state page as 1072 fp8 attention
         # tokens after the 16-token backend alignment.
         mamba_block_size=1072,
-        # peft targets shared-expert linears on every layer. fused routed-expert tensors are not
-        # nn.linear modules and are not adapter targets in the loaded model.
+        # peft targets ordinary linears plus both fused routed-expert tensors on every layer.
+        # each routed tensor has 256 expert slices, all using the adapter's uniform serving rank.
         lora_target_shapes=(
             (512, 2048, 40),
+            (512, 2048, 10_240),
             (1152, 1152, 27),
             (1152, 3456, 27),
             (1152, 4304, 27),
             (2048, 1, 40),
             (2048, 32, 60),
             (2048, 512, 100),
+            (2048, 1024, 10_240),
             (2048, 4096, 30),
             (2048, 8192, 40),
             (4096, 2048, 40),

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/__init__.py

@@ -22,10 +22,12 @@ from flash.engine.worker.adapter import (
     _download_adapter,
     _init_adapter_model,
     _resolve_adapter_ref,
+    lora_target_parameters,
     make_lora,
     prepare_fresh_lora_base,
     require_vllm_for_rollout_func,
     stamp_adapter_provenance,
+    validate_lora_target_parameters,
 )
 from flash.engine.worker.backend_common import collect_ray_failure_logs
 from flash.engine.worker.decoding import (
@@ -531,6 +533,7 @@ __all__ = [
     "liger_on",
     "load_mega_cache",
     "load_tokenizer",
+    "lora_target_parameters",
     "loraplus_optimizer_cls",
     "main",
     "make_checkpoint_upload_callback",
@@ -560,6 +563,7 @@ __all__ = [
     "thinking_text",
     "upload_debug_jsonl",
     "upload_resume_checkpoint",
+    "validate_lora_target_parameters",
     "wait_for_gpu",
     "wandb_finish",
     "wandb_report_to",

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/adapter.py

@@ -27,11 +27,35 @@ from flash.engine.worker.perf import optimal_attn_impl
 
 _ADAPTER_DOWNLOAD_RETRIES = 4
 _ADAPTER_DOWNLOAD_BACKOFF_S = 5.0
+_QWEN35_EXPERT_TARGET_PARAMETERS = (
+    "mlp.experts.gate_up_proj",
+    "mlp.experts.down_proj",
+)
+
+
+def lora_target_parameters(model_id: str | None) -> list[str] | None:
+    """return direct parameter targets required by the model's fused expert layout."""
+    if model_id == "Qwen/Qwen3.6-35B-A3B":
+        return list(_QWEN35_EXPERT_TARGET_PARAMETERS)
+    return None
+
+
+def validate_lora_target_parameters(config: dict, model_id: str) -> None:
+    """fail closed when a warm-start adapter omits required fused expert parameters."""
+    required = set(lora_target_parameters(model_id) or ())
+    if not required:
+        return
+    actual = set(config.get("target_parameters") or ())
+    missing = sorted(required - actual)
+    if missing:
+        raise ValueError(
+            f"warm-start adapter for {model_id} omits required expert targets {missing}; "
+            "retrain the source adapter with the current Flash version"
+        )
 
 
 def make_lora(model_id: str | None = None):
-    """Build LoRA config targeting all linear layers (VL models included: the vision tower /
-    projector / MTP linears are adapted too; on examples without images they simply get no gradient)."""
+    """build the model's complete serve-compatible lora target set."""
     from peft import LoraConfig
 
     targets = "all-linear"
@@ -46,7 +70,9 @@ def make_lora(model_id: str | None = None):
         "task_type": "CAUSAL_LM",
         "revision": model_revision or None,
     }
-    # PiSSA removed: it mutates the base, so its adapter corrupts serve + warm-start on the unmodified base.
+    if target_parameters := lora_target_parameters(model_id):
+        kwargs["target_parameters"] = target_parameters
+    # pissa removed: it mutates the base, so its adapter corrupts serve + warm-start on the unmodified base.
     kwargs["init_lora_weights"] = True
     print(
         "[lora] init_lora_weights=True (standard zero-B; PiSSA removed for serve/warm-start safety)"
@@ -160,6 +186,8 @@ def _init_adapter_model(model_id: str):
             "refusing to silently start from the base model. Verify the source run and access, "
             "or omit init_from_adapter to train a fresh LoRA."
         )
+    with open(os.path.join(adir, "adapter_config.json"), encoding="utf-8") as config_file:
+        validate_lora_target_parameters(json.load(config_file), model_id)
     print("[init-adapter] continuing the prepared source LoRA")
     _attn = optimal_attn_impl()
     attn_kw = {"attn_implementation": _attn} if _attn else {}

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/backend_common.py

@@ -808,6 +808,10 @@ _CHILD_TAIL_LINE_CHARS = 300
 # how many retained lines ride along on a pre-first-step heartbeat. narrower than what is retained:
 # this payload is uploaded every tick, so it stays small enough not to bloat the snapshot.
 STALL_TAIL_LINES = 15
+_RETRIABLE_VERL_CHILD_SIGNATURES = (
+    "cudaErrorDevicesUnavailable",
+    "CUDA-capable device(s) is/are busy or unavailable",
+)
 
 
 class ChildOutputTail:
@@ -826,13 +830,34 @@ class ChildOutputTail:
     def __init__(self, limit: int = CHILD_TAIL_LINES) -> None:
         self._lines: collections.deque[str] = collections.deque(maxlen=limit)
         self._written = 0
+        self._retriable_infra_signature: str | None = None
+        self._cuda_oom_evidence: str | None = None
 
     def record(self, line: str) -> None:
+        if self._retriable_infra_signature is None:
+            self._retriable_infra_signature = next(
+                (signature for signature in _RETRIABLE_VERL_CHILD_SIGNATURES if signature in line),
+                None,
+            )
+        if self._cuda_oom_evidence is None:
+            from flash.engine.worker.perf.lifecycle import cuda_oom_message_evidence
+
+            self._cuda_oom_evidence = cuda_oom_message_evidence(line)
         text = line.rstrip("\n")
         if text:
             self._lines.append(text[:_CHILD_TAIL_LINE_CHARS])
             self._written += 1
 
+    @property
+    def retriable_infra_signature(self) -> str | None:
+        """the first stable retriable-infrastructure signature observed in child output."""
+        return self._retriable_infra_signature
+
+    @property
+    def cuda_oom_evidence(self) -> str | None:
+        """the first authoritative cuda oom message evidence observed in child output."""
+        return self._cuda_oom_evidence
+
     @property
     def written(self) -> int:
         """how many non-empty lines the child has produced, ever.
@@ -851,6 +876,25 @@ class ChildOutputTail:
         return lines
 
 
+def raise_for_classified_verl_exit(return_code: int, tail: ChildOutputTail) -> None:
+    """raise a classified failure when a nonzero verl child reported authoritative evidence."""
+    if return_code == 0:
+        return
+    oom_evidence = tail.cuda_oom_evidence
+    if oom_evidence is not None:
+        raise RuntimeError(
+            f"verl subprocess exited with status {return_code} after reporting {oom_evidence}"
+        )
+    signature = tail.retriable_infra_signature
+    if signature is None:
+        return
+    from flash.engine.worker.perf.lifecycle import RetriableInfraError
+
+    raise RetriableInfraError(
+        f"verl subprocess exited with status {return_code} after reporting {signature}"
+    )
+
+
 class ChildTailStaleness:
     """tracks how long a child has been silent, across the ticks that sample its tail.
 
@@ -1073,6 +1117,7 @@ def run_verl_training(
     device memory that nothing will release.
     """
     step_re = re.compile(step_pattern)
+    child_tail = tail if tail is not None else ChildOutputTail()
     # before the child exists, so any grandchild it orphans reparents here and can actually be
     # reaped. this process is not pid 1 -- the runpod handler is -- so without it every wait below
     # answers ChildProcessError for a zombie nobody will collect.
@@ -1086,13 +1131,14 @@ def run_verl_training(
         bufsize=1,
         start_new_session=True,
     )
+    # start_new_session makes the leader pid the stable group id even after that leader is reaped.
+    process_group_id = proc.pid
     last_hb = 0.0
     try:
         assert proc.stdout is not None
         for line in proc.stdout:
             print(line, end="", flush=True)
-            if tail is not None:
-                tail.record(line)
+            child_tail.record(line)
             if on_line is not None:
                 on_line(line)
             m = step_re.search(line)
@@ -1104,7 +1150,7 @@ def run_verl_training(
                     heartbeat()
                     last_hb = now
     except BaseException:
-        kill_process_group(proc)
+        kill_process_group(proc, process_group_id=process_group_id)
         raise
     finally:
         if proc.poll() is None:
@@ -1114,7 +1160,13 @@ def run_verl_training(
         # caller of this -- runs on exceptions alone: a worker whose later jobs all succeed would
         # hold that zombie for its whole life, as pid 1 with nothing else to reap it (codex[bot]).
         reap_stragglers()
-    return int(proc.returncode)
+    return_code = int(proc.returncode)
+    try:
+        raise_for_classified_verl_exit(return_code, child_tail)
+    except BaseException:
+        kill_process_group(proc, process_group_id=process_group_id)
+        raise
+    return return_code
 
 
 _TEARDOWN_GRACE_S = 10.0
@@ -1373,13 +1425,14 @@ def _process_group_members(pgid: int) -> list[int] | None:
     return members
 
 
-def kill_process_group(proc: subprocess.Popen) -> None:
+def kill_process_group(proc: subprocess.Popen, *, process_group_id: int | None = None) -> None:
     """signal the child's whole process group, escalating to SIGKILL if anything survives.
 
     signalling the group rather than the pid is what reaches vllm's EngineCore grandchild; a survivor
     holds its cuda context and strands the gpu for every later run. the escalation is driven off the
     group, not off the direct child: the usual shape of this failure is the trainer dying on the term
     while the EngineCore ignores it, so waiting only on the child returns before the survivor is gone.
+    a captured ``process_group_id`` keeps the group addressable after the direct child is reaped.
     """
     # grpo drives its own subprocess and calls this directly, never through `run_verl_training`, so
     # the adoption is claimed here too rather than only at the other entry point. idempotent.
@@ -1388,13 +1441,15 @@ def kill_process_group(proc: subprocess.Popen) -> None:
     # entry rather than on exit because that is when such a process has had the longest to die, and
     # it runs before the early returns below so no path through this function skips it.
     reap_stragglers()
-    try:
-        pgid = os.getpgid(proc.pid)
-    except OSError:
-        # already reaped, so there is no group id left to address the survivors by.
-        with contextlib.suppress(subprocess.TimeoutExpired):
-            proc.wait(timeout=_TEARDOWN_GRACE_S)
-        return
+    pgid = process_group_id
+    if pgid is None:
+        try:
+            pgid = os.getpgid(proc.pid)
+        except OSError:
+            # already reaped, so there is no group id left to address the survivors by.
+            with contextlib.suppress(subprocess.TimeoutExpired):
+                proc.wait(timeout=_TEARDOWN_GRACE_S)
+            return
 
     with contextlib.suppress(ProcessLookupError, PermissionError, OSError):
         os.killpg(pgid, signal.SIGTERM)

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/opd_train.py

@@ -1553,6 +1553,14 @@ def build_opd_overrides(config: dict) -> list[str]:
         f"actor_rollout_ref.model.lora_rank={_hydra_val(config['lora_rank'])}",
         f"actor_rollout_ref.model.lora_alpha={_hydra_val(config['lora_alpha'])}",
         f"actor_rollout_ref.model.target_modules={_hydra_val(config['target_modules'])}",
+        *(
+            [
+                "++actor_rollout_ref.model.target_parameters="
+                + _hydra_val(config["target_parameters"])
+            ]
+            if config.get("target_parameters")
+            else []
+        ),
         f"actor_rollout_ref.model.lora_adapter_path={_hydra_val(config.get('lora_adapter_path'))}",
         "actor_rollout_ref.actor.strategy=fsdp",
         "actor_rollout_ref.actor.use_kl_loss=false",
@@ -2498,6 +2506,7 @@ def run_opd_train(spec=None) -> None:
             "lora_rank": lora_rank,
             "lora_alpha": lora_alpha,
             "target_modules": target_modules,
+            "target_parameters": _w.lora_target_parameters(model_id),
             "lora_adapter_path": warmstart_adapter,
             "learning_rate": knobs.learning_rate,
             "local_dir": local_dir,

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/perf/lifecycle.py

@@ -6,6 +6,10 @@ import contextlib
 import re
 
 RETRIABLE_INFRA_MARKER = "RETRIABLE_INFRA_GPU"
+_CUDA_OOM_PREFLIGHT_RE = re.compile(
+    r"free memory on device\s+cuda:\d+.*less than desired gpu memory utilization"
+)
+_CUDA_OOM_CACHE_BLOCKS = "no available memory for the cache blocks"
 
 
 class RetriableInfraError(RuntimeError):
@@ -30,6 +34,17 @@ def cuda_oom_count() -> int:
         return 0
 
 
+def cuda_oom_message_evidence(message: str) -> str | None:
+    """the authoritative text evidence for a deterministic cuda oom, if present."""
+    normalized = message.lower()
+    preflight = _CUDA_OOM_PREFLIGHT_RE.search(normalized)
+    if preflight is not None:
+        return preflight.group(0)
+    if _CUDA_OOM_CACHE_BLOCKS in normalized:
+        return _CUDA_OOM_CACHE_BLOCKS
+    return None
+
+
 def is_cuda_oom(exc: BaseException | None) -> bool:
     """Whether a failure was a CUDA OOM.
 
@@ -46,13 +61,7 @@ def is_cuda_oom(exc: BaseException | None) -> bool:
             return True
     except Exception:
         pass
-    msg = str(exc).lower()
-    if (
-        re.search(
-            r"free memory on device\s+cuda:\d+.*less than desired gpu memory utilization", msg
-        )
-        or "no available memory for the cache blocks" in msg
-    ):
+    if cuda_oom_message_evidence(str(exc)) is not None:
         return True
     return cuda_oom_count() > 0
 

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/rl_train.py

@@ -49,6 +49,7 @@ from flash.engine.worker._pkg import W as _w
 from flash.engine.worker.backend_common import (
     VERL_REQUIREMENT,
     BoundedThreadingHTTPServer,
+    ChildOutputTail,
     adopt_orphaned_descendants,
     agent_loop_workers,
     append_step_metrics,
@@ -60,6 +61,7 @@ from flash.engine.worker.backend_common import (
     parse_verl_metric,
     parse_verl_step_metrics,
     parse_wandb_link,
+    raise_for_classified_verl_exit,
     ray_num_cpus,
     reap_stragglers,
     render_wandb_link_shim,
@@ -356,6 +358,11 @@ def build_verl_overrides(cfg: dict) -> list[str]:
         f"actor_rollout_ref.model.lora_rank={cfg['lora_rank']}",
         f"actor_rollout_ref.model.lora_alpha={cfg['lora_alpha']}",
         f"actor_rollout_ref.model.target_modules={cfg['target_modules']}",
+        *(
+            ["++actor_rollout_ref.model.target_parameters=" + _hydra_val(cfg["target_parameters"])]
+            if cfg.get("target_parameters")
+            else []
+        ),
         # memory: match the retired trl path's gradient checkpointing.
         "actor_rollout_ref.model.enable_gradient_checkpointing=True",
         # 32k contexts: fused linear-CE computes logprobs/entropy from hidden states + lm_head in
@@ -564,6 +571,7 @@ def _build_verl_training_cfg(
         "lora_rank": inp["lora_rank"],
         "lora_alpha": inp["lora_alpha"],
         "target_modules": "all-linear",
+        "target_parameters": _w.lora_target_parameters(model_id),
         "multimodal": bool(inp.get("multimodal")),
         "lr": inp["lr"],
         "group_size": inp["group_size"],
@@ -2756,6 +2764,7 @@ def _resolve_grpo_inputs():
             )
         with open(os.path.join(warmstart_adapter, "adapter_config.json")) as f:
             _src_cfg = json.load(f)
+        _w.validate_lora_target_parameters(_src_cfg, model_id)
         # a patterned adapter trains some modules at higher rank than the base `r`; verl allocates
         # one uniform rank, so it must cover the MAXIMUM prepared rank or the load truncates.
         _ranks = [int(_src_cfg.get("r", lora_rank))]
@@ -3022,6 +3031,38 @@ def _resolve_grpo_inputs():
     }
 
 
+class _GrpoSubprocessStream:
+    """one grpo child stream and the evidence latched from that same stream."""
+
+    def __init__(self, proc) -> None:
+        self._proc = proc
+        # the caller uses start_new_session, so the leader pid remains the group's stable identity.
+        self._process_group_id = proc.pid
+        self._tail = ChildOutputTail()
+        self._terminated = False
+
+    def __iter__(self):
+        assert self._proc.stdout is not None
+        for line in self._proc.stdout:
+            self._tail.record(line)
+            yield line
+
+    def terminate(self) -> None:
+        if self._terminated:
+            return
+        kill_process_group(self._proc, process_group_id=self._process_group_id)
+        self._terminated = True
+
+    def wait_and_classify(self) -> int:
+        return_code = int(self._proc.wait())
+        try:
+            raise_for_classified_verl_exit(return_code, self._tail)
+        except BaseException:
+            self.terminate()
+            raise
+        return return_code
+
+
 def run_rl_train():
     """grpo training on verl, output-compatible with run_rl. see module docstring for scope."""
     t_start = time.time()
@@ -3408,8 +3449,9 @@ def run_rl_train():
                 env=env_for_verl,
                 start_new_session=True,
             )
+            child_stream = _GrpoSubprocessStream(proc)
             try:
-                for line in proc.stdout:
+                for line in child_stream:
                     print(f"[verl] {line}", end="", flush=True)
                     link = parse_wandb_link(line)
                     if link is not None:
@@ -3483,14 +3525,14 @@ def run_rl_train():
                         adv_min = parse_verl_metric(line, "critic/advantages/min")
                         if adv_max is not None and adv_min is not None:
                             adv_spread_history.append(adv_max - adv_min)
-                rc = proc.wait()
+                rc = child_stream.wait_and_classify()
             except BaseException:
                 # the stream loop died (upload error, cancel, oom in the parent): a still-running
                 # verl child would keep burning the gpu unattended, so kill its whole process group.
                 # this escalates to SIGKILL after the grace period, which a bare SIGTERM does not: a
                 # vllm EngineCore that ignores the term keeps its cuda context and strands the gpu
                 # for every later job on a reusable worker.
-                kill_process_group(proc)
+                child_stream.terminate()
                 raise
         if rc != 0:
             raise RuntimeError(

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/flash/engine/worker/sft_train.py

@@ -166,6 +166,11 @@ def build_sft_overrides(cfg: dict) -> list[str]:
         f"model.lora_rank={_hydra_val(cfg['lora_rank'])}",
         f"model.lora_alpha={_hydra_val(cfg['lora_alpha'])}",
         f"model.target_modules={_hydra_val(cfg['target_modules'])}",
+        *(
+            [f"++model.target_parameters={_hydra_val(cfg['target_parameters'])}"]
+            if cfg.get("target_parameters")
+            else []
+        ),
         f"model.lora_adapter_path={_hydra_val(cfg.get('lora_adapter_path'))}",
         "model.use_remove_padding=true",
         # 32k contexts: the fused linear-CE forward never materializes the [tokens, vocab] logits
@@ -762,6 +767,7 @@ def _warmstart_adapter_path(model_id: str, model_revision: str, expected_rank: i
             f"SFT warm-start adapter rank {rank} does not match the prepared train.lora_rank "
             f"{expected_rank}; rank changes are not supported"
         )
+    _w.validate_lora_target_parameters(config, model_id)
     base = str(config.get("base_model_name_or_path") or "").strip()
     if base and base != model_id:
         raise ValueError("SFT warm-start adapter base model does not match the target model")
@@ -1270,6 +1276,7 @@ def run_sft_train(spec=None) -> None:
         "lora_rank": lora_rank,
         "lora_alpha": lora_alpha,
         "target_modules": target_modules,
+        "target_parameters": _w.lora_target_parameters(model_id),
         "lora_adapter_path": warmstart_adapter,
         "ulysses_sp_size": gpu_count,
         "lr": learning_rate,

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.0.83"
+version = "1.0.85"
 description = "Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -127,7 +127,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.0.83"
+version = "1.0.85"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.0.83 → freesolo_flash_dev-1.0.85}/tests/test_allocator.py

@@ -1073,7 +1073,7 @@ def test_catalog_model_algorithm_gpu_matrix_routes_to_fitting_cards(monkeypatch)
     from flash.catalog import ALGORITHMS, MODELS
     from flash.cost import RunConfig, estimate_cost
     from flash.providers import allocator
-    from flash.providers.base import get_gpu_info, provisional_gpu
+    from flash.providers.base import GPU_INFO, combined_vram_gb, get_gpu_info, provisional_gpu
 
     monkeypatch.setattr(allocator, "available_providers", lambda: ("runpod",))
     expected = {
@@ -1090,25 +1090,50 @@ def test_catalog_model_algorithm_gpu_matrix_routes_to_fitting_cards(monkeypatch)
                 continue
             train = {}
             need = allocator.required_vram_gb(model_id, algo, train=train, thinking=False)
+            # most cells fit one card. the 35B MoE trains 256 routed experts per fused tensor, so its
+            # GRPO/OPD peaks exceed every single card and the route is only defined across two.
+            cards = 2 if need > max(g.vram_gb for g in GPU_INFO.values() if g.validated) else 1
 
-            preview_gpu = provisional_gpu(model_id, algo, train=train, thinking=False)
+            preview_gpu = provisional_gpu(
+                model_id, algo, train=train, thinking=False, gpu_count=cards
+            )
             preview_info = get_gpu_info(preview_gpu)
             assert preview_info.validated
             assert preview_info.enum_member
-            assert preview_info.vram_gb >= need, (model_id, algo, preview_gpu, need)
+            # sharding is not free: combined_vram_gb applies the per-card overhead, so two cards hold
+            # less than twice one card. sizing with a naive sum would overstate what the shape holds.
+            assert combined_vram_gb(preview_info.vram_gb, cards) >= need, (
+                model_id,
+                algo,
+                preview_gpu,
+                need,
+            )
 
-            alloc = allocator.allocate(model_id, algo, train=train, thinking=False)
+            alloc = allocator.allocate(
+                model_id, algo, train=train, thinking=False, max_gpu_count=cards
+            )
             alloc_info = get_gpu_info(alloc.gpu)
             assert alloc.provider == "runpod"
             assert alloc.min_vram_gb == need
             assert alloc_info.validated
             assert alloc_info.enum_member
-            assert alloc_info.vram_gb >= need, (model_id, algo, alloc.gpu, need)
-            assert all(c.vram_gb >= need for c in alloc.candidates)
+            assert combined_vram_gb(alloc_info.vram_gb, cards) >= need, (
+                model_id,
+                algo,
+                alloc.gpu,
+                need,
+            )
+            assert all(combined_vram_gb(c.vram_gb, cards) >= need for c in alloc.candidates)
 
-            estimate = estimate_cost(RunConfig(model_id, algo, 1, provider="runpod"))
-            assert estimate.required_vram_gb == need
-            assert estimate.gpu_vram_gb >= need, (model_id, algo, estimate.gpu, need)
+            estimate = estimate_cost(
+                RunConfig(model_id, algo, 1, provider="runpod", gpu_count=cards)
+            )
+            assert estimate.gpu_vram_gb * estimate.gpu_count >= estimate.required_vram_gb, (
+                model_id,
+                algo,
+                estimate.gpu,
+                estimate.required_vram_gb,
+            )
             checked.add((model_id, algo))
 
     assert checked == expected
@@ -1151,7 +1176,18 @@ def test_catalog_model_algorithm_config_gpu_matrix_enforces_pins(monkeypatch):
                 }
                 key = (model_id, algo, configured_gpu)
                 if get_gpu_info(configured_gpu).vram_gb < need:
-                    with pytest.raises(ConfigError, match="requires at least"):
+                    # two distinct rejections, both correct: a pin that is merely too small names the
+                    # shortfall, while a run that outgrows EVERY validated class (35B GRPO/OPD, once
+                    # the routed experts train) fails earlier with no fitting class at all.
+                    biggest = max(g.vram_gb for g in GPU_INFO.values() if g.validated)
+                    # opd words its over-capacity error differently from the generic allocator one,
+                    # so match the shared "no ... validated GPU" shape rather than either wording.
+                    reason = (
+                        "requires at least"
+                        if need <= biggest
+                        else r"(no validated GPU class has|more than any single validated GPU)"
+                    )
+                    with pytest.raises(ConfigError, match=reason):
                         spec_from_dict(raw, run_id="matrix")
                     rejected.add(key)
                     continue