freesolo-flash-dev 1.0.76__tar.gz → 1.0.78__tar.gz

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/Dockerfile.worker

@@ -268,7 +268,7 @@ print('worker stack OK | torch', torch.__version__, '| cudnn', cudnn, '| triton'
 # flash's, so a job that sets WANDB_API_KEY loses its curves and run link unless wandb is importable
 # in here; resolve_verl_loggers() degrades to console when it is not, which is the right behaviour
 # for a foreign FLASH_VERL_PYTHON but silently drops W&B on the default image (codex[bot]).
-ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@b7492fa3b7ab843294d06dbf754e887950f559c7
+ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@1bea7d6825bbb9d2164e86e379b3680e7c53bb8a
 ENV FLASH_VERL_PYTHON=/opt/verl-venv/bin/python
 RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" > /tmp/verl-overrides.txt \
     && uv venv --seed /opt/verl-venv \

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 1.0.76
+Version: 1.0.78
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/catalog.py

@@ -369,6 +369,7 @@ MODELS: dict[str, ModelInfo] = {
         ),
         algos=ALGORITHMS,
         min_vram_gb=48,
+        min_disk_gb=120,
         # NOT QLoRA: peft bnb merge during GRPO rollout diverges trainer precision -> TRL ratio collapses to 0.
         grpo_min_vram_gb=80,
         quant="bf16",

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/cli/TRAINING.md

@@ -1286,13 +1286,13 @@ gpu.count=4`, and the same 1..8 bound rejects a bad value.
 most cards it may use, and it stops at the first count that fits: a class that fits the run on one
 card is allocated as one card, and a class that needs sharding gets the _smallest_ fitting
 combination, not the count you asked for. `--gpus 4` on a 9B **SFT** run pinned to a 24 GB class
-allocates 2. Combinations are also capped at 4, so 5-8 only lower the count that would otherwise be
-chosen.
+allocates 2. Eight cards is the public and allocatable maximum; ceilings between powers of two round
+down to the next rentable count.
 
 That example is SFT-specific on purpose. The fit test is per algorithm, and the same 9B pinned to
-the same 24 GB class does not allocate at all under GRPO or OPD — it raises `UnsupportedGpuError:
-... cannot fit this run even as a 4-card combination`, because rollout memory pushes it past what
-four such cards hold. Raising `--gpus` cannot rescue a pin the algorithm never fits on.
+the same 24 GB class may need a wider combination under GRPO or OPD because rollout memory raises
+the whole-run floor. Raising `--gpus` still cannot rescue a pin the algorithm does not fit on even
+as an 8-card combination.
 
 **There is no exact-count mechanism.** Pinning a small `[gpu] type` raises the floor above one card
 but still does not pin n — it only moves which combination is smallest. To see what a submit

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/cli/__init__.py

@@ -577,7 +577,7 @@ def _build_parser() -> argparse.ArgumentParser:
         help=(
             "most cards to run the job on; sets [gpu] count (1-8). a ceiling, not an exact "
             "count: allocation still picks one card when one fits the run alone, and only "
-            "rentable counts (1, 2, 4) are ever provisioned"
+            "rentable counts (1, 2, 4, 8) are ever provisioned"
         ),
     )
     train.add_argument("--dry-run", action="store_true")

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/cli/envpush.py

@@ -605,8 +605,15 @@ def _dynamic_import_name(node, callees: frozenset[str]) -> str | None:
     return first.value.split(".", 1)[0] or None
 
 
-def _imported_module_names(tree) -> set[str]:
-    """Top-level names a parsed module imports absolutely, statement or literal call."""
+def _imported_module_names(tree, *, relative_names_are_siblings: bool = True) -> set[str]:
+    """Top-level sibling names a parsed module imports, by statement or literal call.
+
+    Every name returned is resolved against the env root by the caller, so a relative import
+    only belongs here when the parsed file sits AT that root. Pass
+    `relative_names_are_siblings=False` for a file nested deeper: `from . import config` inside
+    `pkg/__init__.py` names `pkg/config.py`, which the package walk already ships, and reading it
+    as a root-level name published an unrelated top-level `config.py` instead (codex[bot]).
+    """
     import ast
 
     names: set[str] = set()
@@ -614,8 +621,22 @@ def _imported_module_names(tree) -> set[str]:
     for node in ast.walk(tree):
         if isinstance(node, ast.Import):
             names.update(alias.name.split(".", 1)[0] for alias in node.names)
-        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
-            names.add(node.module.split(".", 1)[0])
+        elif isinstance(node, ast.ImportFrom):
+            if node.level == 0:
+                if node.module:
+                    names.add(node.module.split(".", 1)[0])
+            elif node.level == 1 and relative_names_are_siblings:
+                # `from . import config` and `from .utils import load` name siblings of this file
+                # just as the absolute spellings do. an entrypoint written to work both as a
+                # package member and as a loose module puts the relative form first and falls back
+                # to the absolute one, so reading only absolute imports left the helper unpackaged
+                # whenever the relative spelling came first -- and the fallback then had no file to
+                # import. a bare `from . import x` carries the name in `names`, not in `module`.
+                if node.module:
+                    names.add(node.module.split(".", 1)[0])
+                else:
+                    names.update(alias.name.split(".", 1)[0] for alias in node.names)
+            # level >= 2 climbs above the package this push carries, so it can name no sibling.
         elif isinstance(node, ast.Call):
             dynamic = _dynamic_import_name(node, callees)
             if dynamic:
@@ -623,24 +644,109 @@ def _imported_module_names(tree) -> set[str]:
     return names
 
 
-def _helper_imports(helper: Path) -> set[str]:
+def _helper_imports(helper: Path, *, env_root: Path) -> set[str]:
     """Top-level names a packaged helper imports, for following its own dependencies.
 
     A helper that fails to parse is still shipped verbatim: refusing the push over a file the
     sidecar may never execute would be a harsher failure than the ModuleNotFoundError this
-    lookahead exists to avoid, and the sidecar's own syntax is validated separately."""
+    lookahead exists to avoid, and the sidecar's own syntax is validated separately.
+
+    Only a helper sitting directly at `env_root` has root-level siblings, so a relative import
+    from deeper in a packaged subdirectory is dropped rather than resolved against the wrong
+    directory -- the package walk that reached it already ships its true siblings."""
     import ast
 
     if helper.suffix != ".py":
         return set()
     try:
         return _imported_module_names(
-            ast.parse(helper.read_text(encoding="utf-8"), filename=str(helper))
+            ast.parse(helper.read_text(encoding="utf-8"), filename=str(helper)),
+            relative_names_are_siblings=helper.parent == env_root,
         )
     except (OSError, SyntaxError, UnicodeDecodeError):
         return set()
 
 
+def _iter_import_closure(
+    seed_modules: set[str], *, env_root: Path, entrypoint: Path, yielded: set[Path]
+) -> Iterator[tuple[Path, Path]]:
+    """Yield the sibling helpers reachable from `seed_modules`, following their imports too.
+
+    A helper that imports another sibling needs that sibling too: shipping `scorer.py` without
+    the `thresholds` it imports published an environment that raised ModuleNotFoundError on its
+    first case, having passed every local check because the source directory was importable
+    (cursor[bot]). The queue grows only through siblings actually reached, so it stays a closure
+    over files this push already carries rather than a general dependency resolver -- an
+    unimported neighbour stays local.
+
+    `yielded` is shared with the caller's other walks so a helper that IS `dataset/` (or lives
+    under it) is not emitted twice; `_check_env_push_limits` would otherwise charge those bytes
+    and members twice and reject a tree that is actually under the limit (codex[bot]).
+    """
+    import os
+
+    pending = sorted(seed_modules)
+    visited: set[str] = set()
+    while pending:
+        module_name = pending.pop()
+        if module_name in visited:
+            continue
+        visited.add(module_name)
+        # `from graders.rules import score` resolves to a sibling PACKAGE, not graders.py.
+        # matching only the module spelling published an environment whose sidecar raises
+        # ModuleNotFoundError on its first case, having passed every local check.
+        #
+        # resolution follows python's own order, which is NOT simply "directories first".
+        # a directory holding __init__.py is a regular package and outranks a same-named
+        # module, so shipping the .py and skipping the package published the file the
+        # sidecar never imports while dropping the one it does (cursor[bot]). but a PEP 420
+        # namespace directory (no __init__.py) is only a fallback portion: `graders.py`
+        # wins over a bare `graders/`. requiring the marker unconditionally sent a
+        # namespace helper to the .py fallback, which did not exist either, and published
+        # an archive missing the helper entirely (codex[bot]). verified by probe: with both
+        # present, `import graders` binds graders.py.
+        package = env_root / module_name
+        helper = env_root / f"{module_name}.py"
+        ships_package = (
+            package.is_dir()
+            and not package.is_symlink()
+            and ((package / "__init__.py").is_file() or not helper.is_file())
+        )
+        if ships_package and not _ignore_env_push_path(
+            package, env_root=env_root, entrypoint=entrypoint
+        ):
+            for root, dirs, files in os.walk(
+                package, topdown=True, followlinks=False, onerror=_raise_walk_error
+            ):
+                root_path = Path(root)
+                dirs[:] = sorted(
+                    name
+                    for name in dirs
+                    if not _ignore_env_push_path(
+                        root_path / name, env_root=env_root, entrypoint=entrypoint
+                    )
+                )
+                for name in sorted(files):
+                    child = root_path / name
+                    if child in yielded or _ignore_env_push_path(
+                        child, env_root=env_root, entrypoint=entrypoint
+                    ):
+                        continue
+                    yielded.add(child)
+                    pending.extend(_helper_imports(child, env_root=env_root))
+                    yield child, child.relative_to(env_root)
+            continue
+        if (
+            helper.is_file()
+            and helper != entrypoint
+            and helper not in yielded
+            and not _ignore_env_push_path(helper, env_root=env_root, entrypoint=entrypoint)
+        ):
+            yielded.add(helper)
+            pending.extend(_helper_imports(helper, env_root=env_root))
+            yield helper, helper.relative_to(env_root)
+
+
 def _iter_env_sidecar_files(
     env_root: Path, *, entrypoint: Path, include_full_tree: bool
 ) -> Iterator[tuple[Path, Path]]:
@@ -666,6 +772,24 @@ def _iter_env_sidecar_files(
             ):
                 yielded.add(doc)
                 yield doc, doc.relative_to(env_root)
+        # the entrypoint's OWN imports ship, exactly as its evaluation sidecar's do. seeding the
+        # closure only from evaluations.py published an environment whose entrypoint imported
+        # siblings without them: `env push` exits 0 and prints an id, so nothing surfaces until a
+        # GPU has been rented and the worker imports the module. parsed with the same reader as
+        # every other module here, so a lazily imported helper (inside load_environment) counts.
+        try:
+            entry_tree = ast.parse(entrypoint.read_text(encoding="utf-8"), filename=str(entrypoint))
+        except (OSError, SyntaxError, UnicodeDecodeError):
+            # the entrypoint is validated for syntax on its own path, and a push that cannot read
+            # it fails there with a better message. an unreadable file here just means no closure.
+            entry_tree = None
+        if entry_tree is not None:
+            yield from _iter_import_closure(
+                _imported_module_names(entry_tree),
+                env_root=env_root,
+                entrypoint=entrypoint,
+                yielded=yielded,
+            )
         # the eval sidecar ships with its entrypoint. `env eval` loads evaluations.py next to an
         # exact .py target, so omitting it here would publish an environment whose suite passed
         # locally and is simply absent once pushed.
@@ -675,8 +799,14 @@ def _iter_env_sidecar_files(
             and sidecar != entrypoint
             and not _ignore_env_push_path(sidecar, env_root=env_root, entrypoint=entrypoint)
         ):
-            yielded.add(sidecar)
-            yield sidecar, sidecar.relative_to(env_root)
+            # the entrypoint closure above shares `yielded` and runs first, so an entrypoint that
+            # imports `evaluations` already shipped it. yielding it a second time made
+            # `_check_env_push_limits` charge those bytes and that member twice, rejecting a tree
+            # that is actually under the limit (codex[bot]). only the yield is skipped: the syntax
+            # check and import walk below must still run either way.
+            if sidecar not in yielded:
+                yielded.add(sidecar)
+                yield sidecar, sidecar.relative_to(env_root)
             try:
                 tree = ast.parse(sidecar.read_text(encoding="utf-8"), filename=str(sidecar))
             except SyntaxError as exc:
@@ -688,73 +818,12 @@ def _iter_env_sidecar_files(
             # (flash/envs/evaluations.py). scanning only top-level nodes skips exactly that
             # pattern, so `env test` passes locally and the pushed environment is missing the
             # helper the sidecar imports the moment it grades a case.
-            imported_modules = _imported_module_names(tree)
-            # a helper that imports another sibling needs that sibling too: shipping `scorer.py`
-            # without the `thresholds` it imports published an environment that raised
-            # ModuleNotFoundError on its first case, having passed every local check because the
-            # source directory was importable (cursor[bot]). the queue is seeded from the sidecar
-            # and grows only through siblings actually reached, so it stays a closure over files
-            # this push already carries rather than a general dependency resolver.
-            pending = sorted(imported_modules)
-            visited: set[str] = set()
-            while pending:
-                module_name = pending.pop()
-                if module_name in visited:
-                    continue
-                visited.add(module_name)
-                # `from graders.rules import score` resolves to a sibling PACKAGE, not graders.py.
-                # matching only the module spelling published an environment whose sidecar raises
-                # ModuleNotFoundError on its first case, having passed every local check.
-                #
-                # resolution follows python's own order, which is NOT simply "directories first".
-                # a directory holding __init__.py is a regular package and outranks a same-named
-                # module, so shipping the .py and skipping the package published the file the
-                # sidecar never imports while dropping the one it does (cursor[bot]). but a PEP 420
-                # namespace directory (no __init__.py) is only a fallback portion: `graders.py`
-                # wins over a bare `graders/`. requiring the marker unconditionally sent a
-                # namespace helper to the .py fallback, which did not exist either, and published
-                # an archive missing the helper entirely (codex[bot]). verified by probe: with both
-                # present, `import graders` binds graders.py.
-                package = env_root / module_name
-                helper = env_root / f"{module_name}.py"
-                ships_package = (
-                    package.is_dir()
-                    and not package.is_symlink()
-                    and ((package / "__init__.py").is_file() or not helper.is_file())
-                )
-                if ships_package and not _ignore_env_push_path(
-                    package, env_root=env_root, entrypoint=entrypoint
-                ):
-                    for root, dirs, files in os.walk(
-                        package, topdown=True, followlinks=False, onerror=_raise_walk_error
-                    ):
-                        root_path = Path(root)
-                        dirs[:] = sorted(
-                            name
-                            for name in dirs
-                            if not _ignore_env_push_path(
-                                root_path / name, env_root=env_root, entrypoint=entrypoint
-                            )
-                        )
-                        for name in sorted(files):
-                            child = root_path / name
-                            if child in yielded or _ignore_env_push_path(
-                                child, env_root=env_root, entrypoint=entrypoint
-                            ):
-                                continue
-                            yielded.add(child)
-                            pending.extend(_helper_imports(child))
-                            yield child, child.relative_to(env_root)
-                    continue
-                if (
-                    helper.is_file()
-                    and helper != entrypoint
-                    and helper not in yielded
-                    and not _ignore_env_push_path(helper, env_root=env_root, entrypoint=entrypoint)
-                ):
-                    yielded.add(helper)
-                    pending.extend(_helper_imports(helper))
-                    yield helper, helper.relative_to(env_root)
+            yield from _iter_import_closure(
+                _imported_module_names(tree),
+                env_root=env_root,
+                entrypoint=entrypoint,
+                yielded=yielded,
+            )
         roots = [
             env_root / name
             for name in ("dataset", "datasets")

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/cost/analytical.py

@@ -7,19 +7,16 @@ policy/reference update.
 
 from __future__ import annotations
 
-from flash.providers.allocator import required_vram_gb, vram_headroom
-from flash.providers.base import largest_rentable_count
+from flash.providers.allocator import geometry_safe_gpu_cap, required_vram_gb, vram_headroom
 
 from .facts import (
     GPU_COMPUTE_TFLOPS,
     active_params_b,
     download_weight_gb,
     effective_train_tflops,
-    gpu_hourly_usd,
     gpu_vram_gb,
     has_nvlink,
     model_quant,
-    pick_gpu,
     reward_seconds_per_completion,
     teacher_seconds_per_completion,
     teacher_token_cost_usd,
@@ -309,16 +306,18 @@ def sft_seconds_for_tokens(config: RunConfig, gpu: str, train_tokens: float) ->
     return flops / (peak * mfu)
 
 
-def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str, int]:
-    """(chosen GPU class, required VRAM GB): the cheapest fitting class for the cost.
-
-    Uses ``pick_gpu``, which (unlike the submit-time allocator) intentionally stays gate-free —
-    it considers every fitting class, validated or not — so the estimate reflects the cheapest
-    card that *could* run the job. The live allocator restricts to the validated pool, so the
-    actually-provisioned class can be pricier than this. Catalog sizing is offline/deterministic."""
-    total_params_b(
-        config.model_id
-    )  # catalog-only: reject a non-catalog model before any (HF) sizing
+def _offline_gpu_shape(
+    config: RunConfig, *, max_wall_seconds: float = 0.0
+) -> tuple[str, int, int, str, float]:
+    """Offline structural quote: (gpu, need, count, provider, per-card rate).
+
+    Quote preparation must not query live capacity: a sold-out market or transient lookup failure is
+    exactly what the lifecycle retry machinery exists to survive, and consuming it here prevents the
+    run/status from being created at all. Rank the managed 1/2/4/8-card shapes on the same cost model
+    as allocation, then replace this provisional quote with the selected live candidate immediately
+    before provisioning.
+    """
+    total_params_b(config.model_id)  # catalog-only; no HF/network sizing in `--cost`
     need = required_vram_gb(
         config.model_id,
         config.method,
@@ -326,12 +325,70 @@ def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str
         thinking=config.thinking,
         model_revision=config.model_revision,
     )
-    # rank on job cost, not rental rate, so the quote names the class the allocator will pick.
-    gpu = pick_gpu(
-        need,
-        provider=config.provider,
-        max_wall_seconds=max_wall_seconds,
-        cost_key=step_cost_key(config),
+    from flash.providers.base import (
+        GPU_INFO,
+        canonical_gpu,
+        combined_vram_gb,
+        providers_for,
+        rentable_gpu_counts,
+    )
+
+    provider = config.provider if config.provider != "auto" else "auto"
+    if config.gpu_type:
+        names = (canonical_gpu(config.gpu_type),)
+    else:
+        names = tuple(
+            info.name for info in GPU_INFO.values() if info.enum_member and info.validated
+        )
+    safe_gpu_count = geometry_safe_gpu_cap(
+        config.model_id, config.gpu_count, model_revision=config.model_revision
+    )
+    ranked = []
+    for gpu in names:
+        info = GPU_INFO[gpu]
+        if provider != "auto" and provider not in providers_for(gpu):
+            continue
+        for count in rentable_gpu_counts(safe_gpu_count):
+            if combined_vram_gb(info.vram_gb, count) < need:
+                continue
+            # Provisional quoting is structural and must not touch a live market. Vast pricing is
+            # offer-backed (therefore capacity-backed), and Lambda's catalog can blip too. Use the
+            # provider's offline static rate here; the lifecycle replaces it from the selected candidate
+            # before provisioning, so the persisted/charged quote still carries the exact live rate.
+            if provider == "lambda":
+                from flash.providers.lambdalabs.pricing import static_hourly_rate
+
+                hourly = static_hourly_rate(gpu)
+            else:
+                hourly = info.hourly_usd
+            gpu_bound, fixed = step_seconds_split(config, gpu)
+            step_seconds = gpu_bound / multi_card_speedup(count, gpu) + fixed
+            ranked.append(
+                (
+                    hourly * count * step_seconds,
+                    count,
+                    combined_vram_gb(info.vram_gb, count),
+                    info.vram_gb,
+                    gpu,
+                    hourly,
+                )
+            )
+    if not ranked:
+        if config.gpu_type:
+            info = GPU_INFO[canonical_gpu(config.gpu_type)]
+            raise ValueError(
+                f"exact GPU {info.name!r} cannot fit this run: it requires at least {need} GB"
+            )
+        shape = f" across up to {safe_gpu_count} cards" if safe_gpu_count > 1 else ""
+        raise ValueError(f"no GPU class fits >= {need} GB{shape}")
+    _cost, count, _combined, _per_card, gpu, hourly = min(ranked)
+    return gpu, need, count, provider, hourly
+
+
+def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str, int]:
+    """(chosen GPU class, required VRAM GB) from the offline structural quote."""
+    gpu, need, _count, _provider, _hourly = _offline_gpu_shape(
+        config, max_wall_seconds=max_wall_seconds
     )
     return gpu, need
 
@@ -385,8 +442,15 @@ def _notes(
     return tuple(notes)
 
 
-def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S) -> CostEstimate:
-    """Deterministic pre-flight cost calculation."""
+def estimate_cost(
+    config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S, allocation=None
+) -> CostEstimate:
+    """Deterministic pre-flight cost calculation.
+
+    ``allocation`` is the exact live candidate selected by the retrying lifecycle. Preparation omits
+    it and stays offline; immediately before provisioning the persisted quote is replaced from this
+    candidate so successful billing uses the real provider, class, count, and per-card rate.
+    """
     # Billing cap: mirror the runner's max(60, max_wall_seconds) floor so a sub-60s cap isn't underpriced.
     cap_s = (
         max(60.0, float(config.max_wall_seconds))
@@ -404,63 +468,45 @@ def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S)
         market_wall_s = float(config.max_wall_seconds)
     else:
         market_wall_s = 0.0
-    if config.gpu_type:
-        # gpu_type is provisioned at launch through allocate (disk-aware, live rate). quote the same
-        # path with either an automatic or pinned provider so the estimate matches the actual launch rate,
-        # and a disk floor lifts the quote off the wrong class.
-        from flash.providers.allocator import allocate
-
-        allocation = allocate(
-            config.model_id,
-            config.method,
-            train=config.train_knobs(),
-            thinking=config.thinking,
-            max_wall_seconds=market_wall_s,
-            disk_gb=config.disk_gb,
-            provider=("" if config.provider == "auto" else config.provider),
-            gpu_type=config.gpu_type,
-            model_revision=config.model_revision,
-            # quote the same card ceiling launch allocates under, so a multi-card run is estimated on
-            # the combination it will actually rent rather than on a single-class search that ignores it.
-            max_gpu_count=config.gpu_count,
-        )
+    if allocation is not None:
         gpu = allocation.gpu
         quote_provider = allocation.provider
-        hourly = allocation.hourly_usd
-        need = allocation.min_vram_gb
-        # max_gpu_count is a CEILING, so the allocator may fit the run on fewer cards than requested
-        # (e.g. 2x of a class when 4 was allowed). bill the count it actually chose, not the ceiling.
-        billed_gpu_count = getattr(allocation, "gpu_count", 1) or 1
+        hourly = float(allocation.hourly_usd)
+        need = int(
+            getattr(allocation, "min_vram_gb", 0)
+            or required_vram_gb(
+                config.model_id,
+                config.method,
+                train=config.train_knobs(),
+                thinking=config.thinking,
+                model_revision=config.model_revision,
+            )
+        )
+        billed_gpu_count = int(getattr(allocation, "gpu_count", 1) or 1)
     else:
-        gpu, need = select_gpu(config, max_wall_seconds=market_wall_s)
-        quote_provider = config.provider
-        # no gpu_type pin means no allocate() call and so no combination search; this branch picks a
-        # single class that fits alone, and the run occupies that count of it. gpu.count is a CEILING
-        # though, and only rentable powers of two are ever provisioned -- billing the raw ceiling
-        # would charge a `--gpus 3` run for 3 cards when submit rents 2. this quote is persisted at
-        # submit and charged verbatim by _status_estimated_charge, so an over-count here overbills.
-        billed_gpu_count = largest_rentable_count(config.gpu_count)
-        # quote the same vram-floored vast market pick_gpu selected under (min_vram_gb=need): without the
-        # floor the rate lookup searches from the smallest managed class, letting cheap small-card offers
-        # crowd a high-vram selection off the limited page -> it silently falls back to the static rate.
-        hourly = gpu_hourly_usd(
-            gpu,
-            provider=quote_provider,
-            max_wall_seconds=market_wall_s,
-            min_vram_gb=need,
-            gpu_type="",  # this branch is only reached when gpu_type is empty
+        # Preparation and `flash train --cost` must stay independent of live capacity. A provider
+        # lookup blip here would consume the first allocation failure before a run/status exists, so
+        # the lifecycle could never retry it. This provisional structural quote is replaced from the
+        # exact selected candidate immediately before provisioning.
+        gpu, need, billed_gpu_count, quote_provider, hourly = _offline_gpu_shape(
+            config, max_wall_seconds=market_wall_s
         )
 
     setup = setup_seconds(config)
-    sps = seconds_per_step(config, gpu)
+    gpu_bound_s, fixed_s = step_seconds_split(config, gpu)
+    speedup = multi_card_speedup(billed_gpu_count, gpu)
+    sps = gpu_bound_s / speedup + fixed_s
     required_save_s = required_save_overhead_seconds(config)
     # A one-time kernel/graph compile is paid once on the first step (MoE-only; 0 for dense). It is
-    # training GPU time, so it belongs in the (billed) train term, not setup.
+    # training GPU time, so it belongs in the (billed) train term, not setup. Required saves are also
+    # synchronous fixed overhead; neither is divided by the number of cards.
     compile_s = compile_seconds(config, gpu)
     raw_train = compile_s + config.steps * sps + required_save_s
     if not config.is_grpo and config.train_tokens is not None:
         raw_train = (
-            compile_s + sft_seconds_for_tokens(config, gpu, config.train_tokens) + required_save_s
+            compile_s
+            + sft_seconds_for_tokens(config, gpu, config.train_tokens) / speedup
+            + required_save_s
         )
     sps = raw_train / config.steps
 

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/cost/spec.py

@@ -172,6 +172,6 @@ def runconfig_from_spec(spec) -> RunConfig:
     )
 
 
-def estimate_for_spec(spec) -> CostEstimate:
-    """The pre-flight ``CostEstimate`` for a parsed training ``JobSpec``."""
-    return estimate_cost(runconfig_from_spec(spec))
+def estimate_for_spec(spec, *, allocation=None) -> CostEstimate:
+    """Cost estimate for a parsed training spec, optionally pinned to the selected live candidate."""
+    return estimate_cost(runconfig_from_spec(spec), allocation=allocation)

{freesolo_flash_dev-1.0.76 → freesolo_flash_dev-1.0.78}/flash/engine/vram.py

@@ -1207,9 +1207,9 @@ def check_fit(
 
     gpu_gb = GPU_VRAM_GB.get(gpu, 32)
     # clamp to what the ceiling actually BUYS before sizing: only powers of two up to the
-    # combination cap are ever rented, so a ceiling of 3 provisions 2 cards and a ceiling of 8
-    # provisions 4. sizing on the raw ceiling would accept a shape submit can never allocate --
-    # the parse/submit divergence this parameter exists to close, inverted.
+    # combination cap are ever rented, so a ceiling of 3 provisions 2 cards. sizing on the raw
+    # ceiling would accept a shape submit can never allocate -- the parse/submit divergence this
+    # parameter exists to close, inverted.
     cards = largest_rentable_count(gpu_count)
     usable_gb = combined_vram_gb(gpu_gb, cards)
     if params_b is None: