freesolo-flash-dev 1.4.20__tar.gz → 1.4.22__tar.gz

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/.env.example

@@ -114,3 +114,13 @@ GITHUB_TOKEN=
 # The worker container image is not configurable: every provider runs the published
 # ghcr.io/freesolo-co/flash-worker tags (per-GPU-arch kernel-cache variants, falling back to the
 # base :cu128), selected by GPU class in flash/providers/_lifecycle/net/worker.py.
+
+# Optional Nebius-only Spot training. Install/authenticate the Nebius CLI on the control plane.
+# Public config: [gpu] spot = true. Spot is never an implicit Modal fallback.
+# Use a USD-billed project. Region must match the project/subnet and host image.
+# Host image must include Docker, NVIDIA driver/container toolkit, and outbound internet.
+# Worker images are the same curated per-GPU images used by Modal.
+FLASH_NEBIUS_PROJECT_ID=
+FLASH_NEBIUS_REGION=
+FLASH_NEBIUS_SUBNET_ID=
+FLASH_NEBIUS_IMAGE_ID=

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.20
+Version: 1.4.22
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

freesolo_flash_dev-1.4.22/SELF_HOSTING.md

@@ -0,0 +1,80 @@
+# Nebius Spot training
+
+Set `[gpu] spot = true` to opt into Nebius preemptible VMs. Modal remains
+the default; Spot never falls back to Modal or Nebius on-demand instances. Only
+H100, H200 and B200 are offered. RTX Pro 6000 and L40S are excluded.
+
+| GPU | Start below / fixed estimate | Keep running through |
+| --- | ---: | ---: |
+| H100 | $1.00 | $1.50 |
+| H200 | $1.00 | $2.00 |
+| B200 | $4.00 | $5.00 |
+
+The table lives in `flash/providers/nebius/pricing.py`. Live regional prices
+admit fitting rentals in H100, H200, B200 priority order, even when H200 is cheaper.
+The entry thresholds price Flash's estimates; running prices can be higher. Estimates exclude disk, network and taxes and are not Nebius invoices.
+Nebius presets are 1 or 8 GPUs. All rented cards contribute to the price.
+
+Customer charges use the accepted Spot-priced quote: $1 per GPU-hour for H100/H200
+or $4 for B200 in the work estimate. Completion charges that quote; early completion
+or cancellation charges its completed-work fraction, capped at the accepted quote.
+Recovery preserves Spot pricing even when a saved resource record lacks its provider.
+These are fixed Flash Spot rates, not a pass-through of fluctuating Nebius invoices.
+
+Flash reads Nebius's public spot-price API and queues jobs if prices are over the
+cap, absent, stale, or unavailable. It checks again every 60 seconds, without
+consuming training retries or creating a VM. Queueing respects cancellation and
+the run's wall-clock deadline. Admission is rechecked before launch, and every VM
+also carries a Nebius maximum-price policy so a later price rise cannot silently
+switch to uncapped pricing. On preemption, Flash provisions another eligible Spot
+VM and resumes the latest durable trainer checkpoint using its existing retry
+budget. Spot uses a greedy scheduler based on the latest completed step duration and checkpoints before another similar
+step would exceed **10 minutes** of unsaved training (for example, every 2 five-minute
+steps or 10 one-minute steps). It also saves the first and final completed step.
+Steps lasting 10 minutes or longer checkpoint after every step. This is a recovery target: indivisible steps and asynchronous uploads can exceed it.
+Do not set `train.save_every` for Spot. Optional `train.save_at_steps` exports
+cannot suppress automatic recovery saves.
+Work since the latest fully uploaded checkpoint may be repeated, including any
+checkpoint whose upload was still in progress. Portal is not used.
+
+On the control-plane host:
+
+1. Install and authenticate the current Nebius CLI with access to the intended
+   USD-billed project, compute resources and billing pricing policies.
+2. Set `FLASH_NEBIUS_PROJECT_ID`, `FLASH_NEBIUS_REGION`,
+   `FLASH_NEBIUS_SUBNET_ID`, and `FLASH_NEBIUS_IMAGE_ID`. The region must match
+   the project and subnet. The configured GPU host image must have Docker,
+   a suitable NVIDIA driver and NVIDIA container toolkit installed, plus outbound
+   access to the worker container registry and Azure Blob Storage. B200 requires the
+   driver/CUDA support of Flash's configured Blackwell worker image.
+3. Retain the existing worker image, artifact and runtime-secret configuration.
+   The VM launches the same Flash worker capsule and training container as Modal.
+4. Start the control plane and submit with `spot = true`. `flash gpus`
+   lists the fixed Spot estimates alongside Modal rates. Set `spot = false` for regular
+   Modal training. Legacy `provider = "spot"` is also accepted.
+
+Supported platform IDs are `gpu-h100-sxm`, `gpu-h200-sxm`, and `gpu-b200-sxm`;
+only platforms offered in the configured region can be admitted. Managed boot
+disks use fast ephemeral Network SSD Non-replicated storage, rounded to 93-GiB
+allocation units with a 279-GiB minimum for the image, model and checkpoints.
+They are deleted with the VM; fully uploaded Azure Blob Storage checkpoints are the
+durable recovery source. Runtime secrets are in root-only cloud-init files;
+restrict access to VM metadata and the image accordingly. Use a dedicated project
+for Flash training resources. The worker has a deadline shutdown, and normal
+cancellation, completion and orphan reconciliation delete the VM and its disk.
+
+References: [Nebius spot prices](https://docs.nebius.com/signup-billing/spot-prices),
+[maximum-price policies](https://docs.nebius.com/signup-billing/manage-pricing-policy),
+and [preemptible VM lifecycle](https://docs.nebius.com/compute/virtual-machines/preemptible).
+
+To qualify a configured Nebius account, create a disposable private Azure artifact repository, configure Azure artifact storage locally, and run:
+
+```sh
+python scripts/smoke_nebius_sft.py --run-paid-test --repo owner/private-test-dataset
+```
+
+This rents one H100 at a time, interrupts a two-step SFT job after checkpoint 1
+is fully uploaded, then verifies that a replacement VM resumes and finishes step 2. Each attempt is limited to 45 minutes to allow cold image downloads and full-state checkpoint transfers.
+The harness deletes its VMs and managed disks; test artifacts remain in the private
+repository. GPU quota and active billing are required. Price thresholds, GPU
+priority, cancellation and adaptive checkpoint timing are covered by offline tests.

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/cli/commands/ops/catalog.py

@@ -20,6 +20,7 @@ def cmd_gpus(args) -> int:
     """List validated managed GPU classes, VRAM, and estimated $/hr."""
     from flash.providers.core.base import GPU_INFO
     from flash.providers.modal.pricing import static_rates as modal_static_rates
+    from flash.providers.nebius.pricing import SPOT_GPUS
 
     modal_rates = modal_static_rates()
     infos = sorted(
@@ -28,10 +29,16 @@ def cmd_gpus(args) -> int:
     tip = (
         "Tip: GPU allocation is automatic by default.\n"
         "The allocator picks the cheapest validated class that fits. Pin a specific class by "
-        'adding type = "<CLASS>" to the [gpu] section.'
+        'adding type = "<CLASS>" to the [gpu] section.\n'
+        "Set spot = true for Nebius only. Spot rates are entry estimates; jobs start strictly below them.\n"
+        "Spot priority: H100, H200, B200. Keep-running caps: $1.50, $2.00, $5.00 per GPU-hour."
     )
     if render.styled():
         rows = [(info.name, info.vram_gb, modal_rates.get(info.name)) for info in infos]
+        rows += [
+            (name + " (Spot)", GPU_INFO[name].vram_gb, float(shape.hourly_usd))
+            for name, shape in SPOT_GPUS.items()
+        ]
         print(tables.gpus_table(rows, tip))
         return 0
 
@@ -42,5 +49,7 @@ def cmd_gpus(args) -> int:
     for info in infos:
         modal_rate = modal_rates.get(info.name)
         print(f"{info.name:<16}{info.vram_gb:>5}G{fmt_rate(modal_rate):>11}")
+    for name, shape in SPOT_GPUS.items():
+        print(f"{name + ' (Spot)':<16}{GPU_INFO[name].vram_gb:>5}G{float(shape.hourly_usd):>11.3f}")
     print(f"\n{tip}")
     return 0

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/cli/scaffold/TRAINING.md

@@ -407,6 +407,17 @@ There is no "all" sentinel, so this number is duplicated from the data into the
 **can drift**: if the dataset later grows, a stale `max_examples` silently trains on a
 subset instead of erroring. Re-check it whenever the dataset or the params change.
 
+Select Spot directly in your editable training TOML:
+
+```toml
+[gpu]
+spot = false # set true for Nebius Spot; false uses regular training
+```
+
+Spot only uses Nebius and prefers fitting H100s, then H200s, then B200s.
+It queues until the price is below $1, $1, or $4 per GPU-hour respectively.
+Once running, the corresponding maximum prices are $1.50, $2, and $5.
+
 GPU allocation and Azure artifacts are **managed by default**: leave `[gpu] type` unset to
 let the allocator pick the cheapest fitting validated class, while `train.hf_repo` remains
 platform-managed. `[gpu] providers` takes an ordered list of provider names and prefers them in
@@ -941,7 +952,7 @@ an epoch is one pass over the retained prompt pool after `max_examples` and prom
 optimizer-step counts are derived from those epochs. A positive `[train] max_steps` replaces that
 derived count with an exact update horizon for every algorithm. `[train] save_at_steps`
 requires a positive `max_steps` so its horizon is authoritative even when SFT packing changes the
-realized batch shape. When non-empty, exact save steps suppress periodic `save_every` checkpoints, and the
+realized batch shape. For Modal, non-empty exact save steps suppress periodic `save_every` checkpoints, and the
 run fails if a requested exact save cannot be saved and published.
 
 ---
@@ -1644,3 +1655,15 @@ See the full reference at https://docs.freesolo.co.
 LoRA+ and entropy-quantile filtering have been removed, including their stored configuration
 fields and resume support. Configs and stored job payloads containing either removed field are
 rejected as unknown settings; no legacy optimizer or loss shim is installed.
+
+### Spot recovery checkpoints
+
+With `[gpu] spot = true`, Flash uses Nebius and manages recovery checkpoints automatically.
+A greedy scheduler uses the latest step duration, targeting at most 10 minutes of
+unsaved training: 5-minute steps save every 2 steps; 1-minute steps every 10.
+The first and final completed steps also save. Steps lasting 10 minutes or longer save after every step. Slow steps or uploads can exceed
+the recovery target. Omit `train.save_every`. Set `spot = false` for regular training.
+An optional `train.save_at_steps` schedule adds model exports to those automatic
+saves; it cannot disable them. After preemption, the replacement Nebius Spot VM
+resumes the latest fully uploaded trainer checkpoint. An interrupted upload is
+not a valid resume point.

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/cli/ui/tables.py

@@ -60,7 +60,11 @@ def gpus_table(rows: list[tuple[str, int, float | None]], tip: str) -> str:
     """GPU classes: (name, vram_gb, $/hr or None)."""
     body = []
     for name, vram, rate in rows:
-        rate_cell = (f"${rate:.2f}", _TEAL) if rate else ("-", _FAINT)
+        rate_cell = (
+            ((f"${rate:.3f}" if "(Spot)" in name else f"${rate:.2f}"), _TEAL)
+            if rate
+            else ("-", _FAINT)
+        )
         body.append([(name, _ACCENT2), (f"{vram} GB", _GRAY), rate_cell])
     table = _format_table(["GPU", "VRAM", "$/HR"], body, aligns=["l", "r", "r"])
     return _stdout_encodable_text(

freesolo_flash_dev-1.4.22/flash/core/checkpoint_policy.py

@@ -0,0 +1,20 @@
+"""Automatic Spot recovery policy shared by all training backends."""
+
+from dataclasses import replace
+
+SPOT_CHECKPOINT_SECONDS = 600
+
+
+def spot_checkpoint_train(train):
+    """Keep requested model exports separate from the runtime recovery clock.
+
+    Ten was the legacy automatic interval. Accept it on old persisted specs but
+    normalize it away; newly authored Spot jobs should omit save_every entirely.
+    """
+    if train.save_every not in (None, 10):
+        raise ValueError("Spot checkpointing is time-based and automatic; remove train.save_every")
+    return replace(train, save_every=None)
+
+
+def is_spot_spec(spec) -> bool:
+    return bool(getattr(getattr(spec, "gpu", None), "spot", False))

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/core/spec.py

@@ -424,6 +424,7 @@ class GpuSpec:
     # PLATFORM-MANAGED: runner assigns weight-cache volume; None = cold download.
     network_volume: str | None = None
     network_volume_gb: int = 100
+    spot: bool = False
     provider: str = ""
     providers: tuple[str, ...] = ()
     # number of cards of `type` a single training worker occupies (1..8). count > 1 provisions a
@@ -441,6 +442,17 @@ class GpuSpec:
         )
         if self.provider and providers:
             raise ValueError("gpu.provider and gpu.providers cannot both be set")
+        if type(self.spot) is not bool:
+            raise TypeError("gpu.spot must be a boolean")
+        if self.spot:
+            if self.provider not in ("", "spot") or providers not in ((), ("spot",)):
+                raise ValueError(
+                    "gpu.spot=true requires Nebius Spot; remove other provider choices"
+                )
+            if not providers:
+                object.__setattr__(self, "provider", "spot")
+        if self.provider == "spot" or providers == ("spot",):
+            object.__setattr__(self, "spot", True)
         if not isinstance(self.type, str):
             raise TypeError("gpu.type must be a string")
         gpu_type = (
@@ -564,6 +576,10 @@ class JobSpec:
     custom: dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self) -> None:
+        if self.gpu.provider.strip().lower() == "spot" or self.gpu.providers == ("spot",):
+            from flash.core.checkpoint_policy import spot_checkpoint_train
+
+            object.__setattr__(self, "train", spot_checkpoint_train(self.train))
         object.__setattr__(self, "seed", parse_seed(self.seed))
         object.__setattr__(self, "model_revision", _model_revision(self.model_revision))
         for field_name in ("model_revision_auto", "model_revision_force_pin"):
@@ -687,6 +703,9 @@ class JobSpec:
         differently, so removing either pop would break existing digests.
         """
         data = asdict(self)
+        # Keep pre-Spot preparation digests stable while binding opt-in Spot to integrity checks.
+        if not self.gpu.spot:
+            data["gpu"].pop("spot", None)
         if not data["custom"]:
             data.pop("custom")
         # Preserve public and worker preparation digests for all existing runs.
@@ -818,6 +837,7 @@ class JobSpec:
                 type=gpu_type,
                 provider=provider,
                 providers=providers,
+                spot=gpu.get("spot", False),
                 disk_gb=int(gpu.get("disk_gb", 60)),
                 max_wall_seconds=int(gpu.get("max_wall_seconds", 24 * 3600)),
                 max_retries=int(gpu.get("max_retries", 5)),

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/cost/analytical.py

@@ -438,6 +438,8 @@ def _quote_gpu_ceiling(
         return geometry_safe_gpu_cap(config.model_id, ceiling, model_revision=config.model_revision)
     from flash.providers.core.base import smallest_fitting_gpu_count
 
+    if config.provider == "spot":
+        return auto_cap
     return (
         smallest_fitting_gpu_count(
             need,
@@ -462,7 +464,7 @@ def _offline_preferred_gpu_shape(config: RunConfig) -> tuple[str, int, int, str,
     configured = available_providers()
     # an unconfigured plane (no credentials anywhere) has nothing to filter against; fall back to the
     # registered set so the quote keeps its historical structural answer instead of going empty.
-    eligible = configured or PROVIDER_NAMES
+    eligible = tuple(name for name in (configured or PROVIDER_NAMES) if name != "spot")
     for provider in config.providers:
         if provider not in eligible:
             continue
@@ -497,6 +499,10 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
     Preparation must not consume live-capacity failures before run creation. Rank rentable shapes
     offline, then replace the quote with the selected live candidate before provisioning.
     """
+    if config.providers == ("spot",):
+        from dataclasses import replace
+
+        return _offline_gpu_shape(replace(config, provider="spot", providers=()))
     if config.providers:
         return _offline_preferred_gpu_shape(config)
     # Fail closed on a model that cannot be sized at all. Curated entries answer from the catalog
@@ -522,7 +528,7 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
         combined_vram_gb,
     )
 
-    provider = config.provider if config.provider != "auto" else "auto"
+    provider = config.provider
     if config.gpu_type:
         # rank every class the author declared acceptable, not just the head. `allocate()` cost-ranks
         # the whole ordered set, so quoting the head alone prices a shape the run may never be given
@@ -547,7 +553,12 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
     ranked = []
     for gpu in names:
         info = GPU_INFO[gpu]
-        for count in rentable_gpu_counts(min(safe_gpu_count, info.max_count)):
+        counts = rentable_gpu_counts(min(safe_gpu_count, info.max_count))
+        if provider == "spot":
+            from flash.providers.nebius.pricing import SPOT_GPUS
+
+            counts = tuple(n for n in counts if n in SPOT_GPUS[gpu].counts)
+        for count in counts:
             # credit only the cards that JOIN the run, matching the allocator's `_fits`. quoting the
             # billed count here made the two disagree: a 27B at 128k over 10 rows was quoted 4x H200
             # (460 GB credited against a 422 GB need) while the allocator launches 2 ranks -- 234 GB
@@ -555,15 +566,20 @@ def _offline_gpu_shape(config: RunConfig) -> tuple[str, int, int, str, float]:
             launched = executed_gpu_count(config, count)
             if combined_vram_gb(info.vram_gb, launched) < need:
                 continue
-            from flash.providers.modal.pricing import hourly_rate
+            from flash.providers.core.registry import get_provider
 
-            hourly = hourly_rate(gpu)
+            hourly = get_provider("modal" if provider == "auto" else provider).hourly_rate(gpu)
             step_seconds = sharded_step_seconds(
                 config, gpu, count, "" if provider == "auto" else provider
             )
+            from flash.providers.nebius.pricing import priority as spot_priority
+
             ranked.append(
                 (
-                    hourly * count * step_seconds,
+                    (
+                        spot_priority(gpu) if provider == "spot" else 0,
+                        hourly * count * step_seconds,
+                    ),
                     count,
                     combined_vram_gb(info.vram_gb, launched),
                     info.vram_gb,

freesolo_flash_dev-1.4.22/flash/engine/worker/train/core/child/checkpoint_schedule.py

@@ -0,0 +1,70 @@
+"""Time-based checkpoint decisions; copied unchanged into isolated trainer children."""
+
+from __future__ import annotations
+
+import os
+import time
+
+SPOT_RECOVERY_SECONDS = 600
+
+
+def enabled() -> bool:
+    return os.environ.get("FLASH_SPOT_CHECKPOINT_SECONDS") == str(SPOT_RECOVERY_SECONDS)
+
+
+class CheckpointClock:
+    """Observe completed steps and save before another typical step exceeds the target.
+
+    Save the first completed step after startup/resume to establish a safe baseline.
+    Greedily allow another step only when its latest measured duration fits the budget.
+    Steps at or above the target save after every completed step.
+    Local checkpoint-writing time is excluded from the step-duration samples.
+    """
+
+    def __init__(self, *, clock=time.monotonic, target_seconds=SPOT_RECOVERY_SECONDS):
+        self.clock = clock
+        self.target_seconds = target_seconds
+        self.last_step = None
+        self.last_end = None
+        self.last_save = None
+
+    def due(self, step: int, *, required=(), final_step: int) -> bool:
+        now = self.clock()
+        if step <= 0 or (self.last_step is not None and step <= self.last_step):
+            return False
+        step_seconds = (
+            max(0.001, (now - self.last_end) / (step - self.last_step))
+            if self.last_end is not None
+            else 0
+        )
+        self.last_step, self.last_end = step, now
+        if self.last_save is None or step in required or step == final_step:
+            return True
+        elapsed = now - self.last_save
+        return elapsed >= self.target_seconds or elapsed + step_seconds > self.target_seconds
+
+    def saved(self) -> None:
+        self.last_save = self.last_end = self.clock()
+
+
+def should_save(owner, step: int, required, final_step: int, *, distributed=False) -> bool:
+    timer = getattr(owner, "_flash_checkpoint_clock", None)
+    if timer is None:
+        timer = owner._flash_checkpoint_clock = CheckpointClock()
+    decision = timer.due(step, required=required, final_step=final_step)
+    if distributed:
+        # SFT calls this on every FSDP rank. One rank's clock must decide for all
+        # ranks or some will skip a collective save while others enter it.
+        import torch
+        import torch.distributed as dist
+
+        if dist.is_initialized():
+            device = torch.cuda.current_device() if dist.get_backend() == "nccl" else "cpu"
+            flag = torch.tensor([int(decision)], dtype=torch.int32, device=device)
+            dist.broadcast(flag, src=0)
+            decision = bool(flag.item())
+    return decision
+
+
+def saved(owner) -> None:
+    owner._flash_checkpoint_clock.saved()

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/core/child/trainer.py

@@ -7,6 +7,7 @@ from verl.trainer.ppo.utils import Role
 from verl.trainer.ppo.v1.trainer_sync import PPOTrainerSync
 
 from flash.engine.worker.runtime.phases import training_phase
+from flash.engine.worker.train.core.child import checkpoint_schedule
 from flash.engine.worker.train.core.child.options import RuntimeOptions
 from flash.engine.worker.train.core.child.queue import _init_transfer_queue
 from flash.engine.worker.train.core.child.readiness import publish_ready
@@ -50,9 +51,20 @@ class FlashSyncTrainer(PPOTrainerSync):
                 self.role_worker_mapping[role] = ray.remote(FlashActorRolloutWorker)
 
     def _save_checkpoint(self):
-        if self.runtime.should_save(int(self.global_steps)):
+        adaptive = checkpoint_schedule.enabled()
+        due = (
+            checkpoint_schedule.should_save(
+                self, int(self.global_steps), self.runtime.save_at_steps, self.runtime.total_steps
+            )
+            if adaptive
+            else self.runtime.should_save(int(self.global_steps))
+        )
+        if due:
             with checkpoint_phase(int(self.global_steps), "capture"):
-                return super()._save_checkpoint()
+                result = super()._save_checkpoint()
+            if adaptive:
+                checkpoint_schedule.saved(self)
+            return result
         return None
 
     def on_train_begin(self):

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/entry/sft_train.py

@@ -227,6 +227,10 @@ def _build_verl_child_env(*, wandb_enabled: bool, context: WorkerContext) -> dic
         for key, value in os.environ.items()
         if key in _CHILD_ENV_EXACT or key.startswith(_CHILD_ENV_PREFIXES)
     }
+    from flash.core.checkpoint_policy import SPOT_CHECKPOINT_SECONDS, is_spot_spec
+
+    if is_spot_spec(context.config.spec):
+        child["FLASH_SPOT_CHECKPOINT_SECONDS"] = str(SPOT_CHECKPOINT_SECONDS)
     if wandb_enabled:
         child.update({key: value for key, value in os.environ.items() if key.startswith("WANDB_")})
     # retained runtime namespaces are filtered by applied-secret provenance. authenticated child-side

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/entry/sft_train_runner.py

@@ -405,6 +405,11 @@ def _prepare_sft_model(
         else max(1, min(options.save_every, update_horizon))
     )
 
+    from flash.core.checkpoint_policy import is_spot_spec
+
+    if is_spot_spec(options.spec):
+        save_freq = 1
+
     card_vram_gb = float(options.gpu_probe.get("memory_gb") or 0.0)
     info = MODELS.get(options.model_id)
     active_params_b = float(getattr(info, "active_params_b", 0.0) or 0.0) or None

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/rl/launch/inputs.py

@@ -521,7 +521,9 @@ def _assemble_grpo_inputs(
         "epochs": schedule["epochs"],
         "verl_total_epochs": schedule["verl_total_epochs"],
         "steps": schedule["steps"],
-        "save_freq": schedule["save_freq"],
+        "save_freq": 1
+        if context.config.spec and context.config.spec.gpu.spot
+        else schedule["save_freq"],
         "save_at_steps": schedule["save_at_steps"],
         "ckpt_to_keep": schedule["ckpt_to_keep"],
         # verl's default preserves the update horizon and on-policy baseline; with no generation reuse, each

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/sft/child/trainer.py

@@ -9,6 +9,7 @@ from verl.trainer.sft_trainer import SFTTrainer
 from verl.utils.checkpoint import CheckpointHandler
 from verl.utils.dataset.dataset_utils import SFTTensorCollator
 
+from flash.engine.worker.train.core.child import checkpoint_schedule
 from flash.engine.worker.train.core.child.options import RuntimeOptions
 from flash.engine.worker.train.core.lifecycle.timing import checkpoint_phase
 from flash.engine.worker.train.sft.child.dataset import FlashTokenizedSFTDataset
@@ -21,11 +22,22 @@ class FlashCheckpointHandler(CheckpointHandler):
         super().__init__(**kwargs)
 
     def save_checkpoint(self, step):
-        if self.runtime.should_save(step):
+        adaptive = checkpoint_schedule.enabled()
+        due = (
+            checkpoint_schedule.should_save(
+                self, step, self.runtime.save_at_steps, self.runtime.total_steps, distributed=True
+            )
+            if adaptive
+            else self.runtime.should_save(step)
+        )
+        if due:
             with checkpoint_phase(step, "retention_wait"):
                 self._wait_for_publication(step)
             with checkpoint_phase(step, "capture"):
-                return super().save_checkpoint(step)
+                result = super().save_checkpoint(step)
+            if adaptive:
+                checkpoint_schedule.saved(self)
+            return result
         return None
 
     def _wait_for_publication(self, step):

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/engine/worker/train/sft/setup/checkpoints.py

@@ -229,7 +229,12 @@ class _VerlCheckpointWatcher:
 
     def _publish(self, step: int, checkpoint_dir: str) -> None:
         if not self._should_publish(step):
+            from flash.core.checkpoint_policy import is_spot_spec
+
             self.lifecycle.mark_discovered(step)
+            if is_spot_spec(self.context.config.spec):
+                staged_dir = self._staged_source(step, checkpoint_dir)
+                self._persist_resume(step, staged_dir, job_label="sft")
             return
         staged_dir = self._staged_source(step, checkpoint_dir)
         try:

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/providers/core/allocator.py

@@ -301,7 +301,12 @@ def _structurally_fits(available, need: int, cap: int, executed_width, acceptabl
         for gpu_class in classes:
             if allowed and gpu_class.name not in allowed:
                 continue
-            for count in rentable_gpu_counts(cap):
+            counts = rentable_gpu_counts(min(cap, gpu_class.max_count))
+            if name == "spot":
+                from flash.providers.nebius.pricing import SPOT_GPUS
+
+                counts = tuple(n for n in counts if n in SPOT_GPUS[gpu_class.name].counts)
+            for count in counts:
                 if combined_vram_gb(gpu_class.vram_gb, executed_width(count)) >= need:
                     return True
     return False
@@ -629,7 +634,7 @@ def _resolved_gpu_count(
             need, max_gpu_count=auto_cap, gpu_names=gpu_names, executed_width=executed_width
         )
         if fitting_count is not None:
-            return fitting_count
+            return auto_cap if available == ("spot",) else fitting_count
         effective_count = auto_cap
     else:
         effective_count = geometry_safe_gpu_cap(
@@ -840,6 +845,10 @@ def allocate(
     if provider and providers:
         raise UnsupportedGpuError("provider and providers cannot both be set")
     available = available_providers()
+    if providers == ("spot",):
+        provider = "spot"
+    if provider != "spot":
+        available = tuple(name for name in available if name != "spot")
     # kept across the narrowing below so a rejection can ask what dropping the pin would restore.
     unpinned = None
     if provider:
@@ -923,7 +932,7 @@ def allocate(
     return _cheapest_allocation(
         candidates,
         need=need,
-        cost_per_step=cost_per_step,
+        cost_per_step=None if provider == "spot" else cost_per_step,
         provider_rank=provider_rank,
     )
 
@@ -942,12 +951,15 @@ def _cheapest_allocation(
     matches. A run the cost model cannot price (``cost_per_step`` is ``None``) falls back to total
     $/hr.
     """
+    from flash.providers.nebius.pricing import priority as spot_priority
+
     primary = cost_per_step if cost_per_step is not None else (lambda c: c.total_hourly_usd)
     unnamed_rank = len(provider_rank)
     ranked = sorted(
         candidates,
         key=lambda c: (
             provider_rank.get(c.provider, unnamed_rank),
+            spot_priority(c.gpu) if c.provider == "spot" else 0,
             primary(c),
             c.total_hourly_usd,
             c.gpu_count,
@@ -955,6 +967,13 @@ def _cheapest_allocation(
             c.gpu,
         ),
     )
+    # Spot prioritizes GPU class, then current price; fixed entry rates price estimates.
+    from flash.providers.nebius.pricing import hourly_rate as spot_hourly_rate
+
+    ranked = [
+        replace(c, hourly_usd=spot_hourly_rate(c.gpu)) if c.provider == "spot" else c
+        for c in ranked
+    ]
     best = ranked[0]
     return Allocation(
         provider=best.provider,

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/providers/core/base.py

@@ -175,8 +175,10 @@ def get_gpu_info(name: str) -> GpuClass:
 
 def providers_for(name: str) -> tuple[str, ...]:
     """The training substrate supporting this validated GPU class."""
-    get_gpu_info(name)
-    return ("modal",)
+    from flash.providers.nebius.pricing import SPOT_GPUS
+
+    gpu = get_gpu_info(name)
+    return ("modal", "spot") if gpu.name in SPOT_GPUS else ("modal",)
 
 
 def gpu_short(name: str) -> str:

freesolo_flash_dev-1.4.22/flash/providers/core/handles.py

@@ -0,0 +1,11 @@
+"""Validate persisted training identities before recovery and resource mutations."""
+
+
+def parse_handle(data):
+    if data.get("provider") == "spot":
+        from flash.providers.nebius.handles import SpotJobHandle
+
+        return SpotJobHandle.from_dict(data)
+    from flash.providers.modal.handles import ModalJobHandle
+
+    return ModalJobHandle.from_dict(data)

{freesolo_flash_dev-1.4.20 → freesolo_flash_dev-1.4.22}/flash/providers/core/preflight.py

@@ -19,7 +19,10 @@ __all__ = [
 
 logger = get_logger(__name__)
 
-_PROVIDER_SETUP = (("modal", "MODAL_TOKEN_ID + MODAL_TOKEN_SECRET", "Modal workspace credentials"),)
+_PROVIDER_SETUP = (
+    ("modal", "MODAL_TOKEN_ID + MODAL_TOKEN_SECRET", "Modal workspace credentials"),
+    ("spot", "FLASH_NEBIUS_PROJECT_ID/REGION/SUBNET_ID/IMAGE_ID", "Nebius CLI and GPU host image"),
+)
 
 
 def _present(var: str) -> bool:
@@ -121,4 +124,6 @@ def check_run_preflight() -> None:
     require_operator_config()
     if not _present("GITHUB_TOKEN"):
         logger.warning("GITHUB_TOKEN is not set; private environment repositories are unavailable.")
-    logger.info("GPU provider configured: modal")
+    from flash.providers.core.registry import available_providers
+
+    logger.info("GPU providers configured: %s", ", ".join(available_providers()))