freesolo-flash-dev 0.2.25__py3-none-any.whl

flash/client/http.py

@@ -0,0 +1,372 @@
+"""Stdlib HTTP client for the Flash control plane (no extra dependencies).
+
+Every CLI/MCP operation maps to one method here. Server errors (FastAPI's
+``{"detail": ...}``) surface as ``ApiError`` with the server's message; connection
+problems surface as ``ClientError`` with an actionable hint.
+"""
+
+from __future__ import annotations
+
+import codecs
+import contextlib
+import json
+import os
+import urllib.error
+import urllib.request
+from collections.abc import Callable, Iterator
+from typing import Any
+
+from .config import load_credentials_with_source
+
+# Called as ``progress(bytes_sent, total_bytes)`` as a request body streams to the server, so
+# the CLI can draw an upload bar. ``total_bytes`` is the full Content-Length, fixed up front.
+ProgressCallback = Callable[[int, int], None]
+
+
+class ClientError(RuntimeError):
+    """Expected client-side errors (no key, unreachable server) — printed cleanly."""
+
+
+class ApiError(ClientError):
+    def __init__(self, status: int, message: str):
+        super().__init__(message)
+        self.status = status
+
+
+# Login is handled by the freesolo backend (not the flash control plane): `flash login`
+# verifies the user's freesolo API key here. The same key authenticates the flash
+# control plane, which accepts freesolo-issued keys.
+DEFAULT_FREESOLO_BASE_URL = "https://api.freesolo.co"
+FREESOLO_AUTH_VERIFY_PATH = "/api/auth/verify"
+
+
+def freesolo_base_url(override: str | None = None) -> str:
+    return (override or os.environ.get("FREESOLO_BASE_URL") or DEFAULT_FREESOLO_BASE_URL).rstrip(
+        "/"
+    )
+
+
+def _detail_from_http_error(exc: urllib.error.HTTPError) -> str:
+    """Extract the server's error message from an HTTPError body (FastAPI ``detail``)."""
+    body = exc.read()
+    try:
+        detail = json.loads(body).get("detail") or body.decode()
+    except (ValueError, AttributeError):
+        detail = body.decode(errors="replace") if body else str(exc)
+    return str(detail)
+
+
+def verify_freesolo_key(api_key: str, base_url: str | None = None) -> None:
+    """Verify a freesolo API key against the freesolo backend's ``/api/auth/verify``.
+
+    Raises :class:`ClientError`/:class:`ApiError` if the key is rejected or the backend is
+    unreachable; returns ``None`` on success. Keys are issued from the freesolo sign-in page.
+    """
+    base = freesolo_base_url(base_url)
+    url = f"{base}{FREESOLO_AUTH_VERIFY_PATH}"
+    req = urllib.request.Request(
+        url,
+        method="GET",
+        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            resp.read()
+    except urllib.error.HTTPError as exc:
+        if exc.code in (401, 403):
+            raise ClientError(
+                "freesolo rejected this API key — create or copy a valid key at "
+                "https://freesolo.co/sign-in and pass it with `flash login --api-key` "
+                "(or FREESOLO_API_KEY)"
+            ) from exc
+        raise ApiError(exc.code, _detail_from_http_error(exc)) from exc
+    except urllib.error.URLError as exc:
+        raise ClientError(
+            f"cannot reach the freesolo backend at {base} ({exc.reason}); "
+            "check your network connection and FREESOLO_BASE_URL"
+        ) from exc
+
+
+class _ProgressReader:
+    """A read()-only file-like over an in-memory payload that reports bytes consumed.
+
+    ``http.client`` sends a body exposing ``read()`` in blocksize chunks; we forward the running
+    total to ``progress(sent, total)`` for each chunk so the CLI can draw an upload bar. The
+    caller sets Content-Length from ``len(payload)``, so the request is NOT chunked-encoded and
+    the server reads it exactly as a plain bytes body."""
+
+    def __init__(self, data: bytes, progress: ProgressCallback):
+        self._data = data
+        self._total = len(data)
+        self._pos = 0
+        self._progress = progress
+
+    def __len__(self) -> int:
+        return self._total
+
+    def read(self, size: int = -1) -> bytes:
+        if size is None or size < 0:
+            chunk = self._data[self._pos :]
+        else:
+            chunk = self._data[self._pos : self._pos + size]
+        self._pos += len(chunk)
+        # a rendering hiccup must never abort an in-flight upload
+        with contextlib.suppress(Exception):
+            self._progress(self._pos, self._total)
+        return chunk
+
+
+class ApiClient:
+    def __init__(
+        self,
+        api_url: str,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+        key_source: str | None = None,
+    ):
+        self.api_url = api_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.key_source = key_source
+
+    def _auth_error_detail(self, status: int, detail: str) -> str:
+        if status not in {401, 403} or self.key_source != "FREESOLO_API_KEY":
+            return detail
+        return (
+            f"{detail}; FREESOLO_API_KEY is set and overrides the key saved by "
+            "`flash login`. Unset FREESOLO_API_KEY or update it to a valid freesolo API key."
+        )
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        body: dict | None = None,
+        timeout: float | None = None,
+    ) -> Any:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        req = urllib.request.Request(
+            f"{self.api_url}{path}",
+            method=method,
+            data=json.dumps(body).encode() if body is not None else None,
+            headers=headers,
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=timeout or self.timeout) as resp:
+                raw = resp.read()
+                return json.loads(raw) if raw else {}
+        except urllib.error.HTTPError as exc:
+            detail = self._auth_error_detail(exc.code, _detail_from_http_error(exc))
+            raise ApiError(exc.code, detail) from exc
+        except urllib.error.URLError as exc:
+            raise ClientError(
+                f"cannot reach the Flash service at {self.api_url} ({exc.reason}); "
+                "check your network connection and FLASH_API_URL"
+            ) from exc
+
+    def _post_with_progress(
+        self,
+        path: str,
+        body: dict,
+        *,
+        progress: ProgressCallback,
+        timeout: float,
+    ) -> Any:
+        """POST a JSON body while reporting upload progress (see :class:`_ProgressReader`).
+
+        Same error mapping as :meth:`_request`; kept separate because the body is a streaming
+        reader with an explicit Content-Length rather than a one-shot bytes payload."""
+        payload = json.dumps(body).encode()
+        headers = {"Content-Type": "application/json", "Content-Length": str(len(payload))}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        req = urllib.request.Request(
+            f"{self.api_url}{path}",
+            method="POST",
+            data=_ProgressReader(payload, progress),
+            headers=headers,
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                raw = resp.read()
+                return json.loads(raw) if raw else {}
+        except urllib.error.HTTPError as exc:
+            detail = self._auth_error_detail(exc.code, _detail_from_http_error(exc))
+            raise ApiError(exc.code, detail) from exc
+        except urllib.error.URLError as exc:
+            raise ClientError(
+                f"cannot reach the Flash service at {self.api_url} ({exc.reason}); "
+                "check your network connection and FLASH_API_URL"
+            ) from exc
+
+    # -- identity ----------------------------------------------------------------------
+    def me(self) -> dict:
+        return self._request("GET", "/v1/me")
+
+    def health(self) -> dict:
+        return self._request("GET", "/v1/health", timeout=10.0)
+
+    # -- environments ------------------------------------------------------------------
+    def publish_env(
+        self,
+        *,
+        name: str,
+        package_b64: str,
+        progress: ProgressCallback | None = None,
+    ) -> dict:
+        """Upload a packaged Freesolo environment to the managed Environments Hub.
+
+        When ``progress`` is given the body streams to the server in chunks and
+        ``progress(bytes_sent, total_bytes)`` fires for each, so the CLI can render an upload
+        bar; otherwise the body is sent in one shot (the default, used off a TTY)."""
+        body = {"name": name, "package_b64": package_b64}
+        if progress is None:
+            return self._request("POST", "/v1/envs", body=body, timeout=1800.0)
+        return self._post_with_progress("/v1/envs", body, progress=progress, timeout=1800.0)
+
+    # -- runs --------------------------------------------------------------------------
+    def create_run(self, spec: dict, runtime_secrets: dict[str, str] | None = None) -> dict:
+        body = {"spec": spec}
+        if runtime_secrets:
+            body["runtime_secrets"] = runtime_secrets
+        return self._request("POST", "/v1/runs", body=body)
+
+    def list_runs(self) -> list[dict]:
+        return self._request("GET", "/v1/runs")["runs"]
+
+    def get_run(self, run_id: str) -> dict:
+        return self._request("GET", f"/v1/runs/{run_id}")
+
+    def get_logs(self, run_id: str, offset: int = 0) -> dict:
+        return self._request("GET", f"/v1/runs/{run_id}/logs?offset={int(offset)}")
+
+    def get_worker_output(self, run_id: str) -> dict[str, str]:
+        # The train-subprocess console/traceback ({console_<phase>.txt, error_<phase>.txt}) from the
+        # run's HF artifact repo, fetched server-side with the operator token — the real worker
+        # output the offset-paged log can't carry. Kept off the hot get_logs poll path. {} if none.
+        #
+        # Tolerate a managed server that predates the /worker route: a CLI upgraded ahead of the
+        # service rollout would otherwise hard-fail. FastAPI returns a bare 404 "Not Found" for an
+        # unmatched path -> treat ONLY that as "no worker output" ({}); real 404s still surface (an
+        # unknown run_id carries detail "unknown run_id: ...", not "Not Found").
+        try:
+            return self._request("GET", f"/v1/runs/{run_id}/worker").get("worker", {})
+        except ApiError as exc:
+            if exc.status == 404 and str(exc).strip().lower() == "not found":
+                return {}
+            raise
+
+    def cancel_run(self, run_id: str) -> dict:
+        return self._request("POST", f"/v1/runs/{run_id}/cancel")
+
+    def checkpoints(self, run_id: str) -> list[dict]:
+        """Deployable per-step RL checkpoints for a run (each `flash deploy --step N`-able)."""
+        return self._request("GET", f"/v1/runs/{run_id}/checkpoints")["checkpoints"]
+
+    # -- serving -----------------------------------------------------------------------
+    def deploy(
+        self,
+        run_id: str,
+        dry_run: bool = False,
+        step: int | None = None,
+    ) -> dict:
+        # Deploy blocks on registration and serving warmup, which can take many minutes.
+        deploy_timeout = 30 * 60 if not dry_run else None
+        body: dict = {"dry_run": dry_run}
+        if step is not None:
+            # Deploy a specific intermediate checkpoint instead of the run's final adapter.
+            # Reject a bool explicitly: `int(True)`/`int(False)` would silently coerce to step
+            # 1/0, but the server guard (_resolve_deploy_step) treats a bool as an invalid step
+            # and 400s — so fail fast here with a clear client-side error instead of sending a
+            # bogus 0/1 that the server rejects (or, worse, that hits a real checkpoint 0/1).
+            if isinstance(step, bool):
+                raise ClientError(f"invalid checkpoint step: {step!r} (must be an integer)")
+            body["step"] = int(step)
+        return self._request(
+            "POST",
+            f"/v1/runs/{run_id}/deploy",
+            body=body,
+            timeout=deploy_timeout,
+        )
+
+    def undeploy(self, run_id: str) -> dict:
+        return self._request("DELETE", f"/v1/runs/{run_id}/deploy")
+
+    def deployments(self) -> list[dict]:
+        return self._request("GET", "/v1/deployments")["deployments"]
+
+    def chat(
+        self,
+        run_id: str,
+        messages: list[dict],
+        temperature: float = 0.0,
+        max_tokens: int = 512,
+    ) -> dict:
+        # Serving warmup can take minutes; give inference a generous timeout.
+        return self._request(
+            "POST",
+            f"/v1/runs/{run_id}/chat",
+            body={"messages": messages, "temperature": temperature, "max_tokens": max_tokens},
+            timeout=30 * 60,
+        )
+
+    def chat_stream(
+        self,
+        run_id: str,
+        messages: list[dict],
+        temperature: float = 0.0,
+        max_tokens: int = 512,
+    ) -> Iterator[str]:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        req = urllib.request.Request(
+            f"{self.api_url}/v1/runs/{run_id}/chat",
+            method="POST",
+            data=json.dumps(
+                {
+                    "messages": messages,
+                    "temperature": temperature,
+                    "max_tokens": max_tokens,
+                    "stream": True,
+                }
+            ).encode(),
+            headers=headers,
+        )
+        decoder = codecs.getincrementaldecoder("utf-8")()
+        try:
+            with urllib.request.urlopen(req, timeout=30 * 60) as resp:
+                content_type = resp.headers.get("Content-Type", "")
+                if "application/json" in content_type:
+                    payload = json.loads(resp.read() or b"{}")
+                    content = (((payload.get("choices") or [{}])[0].get("message") or {}).get("content"))
+                    if content:
+                        yield str(content)
+                    return
+                while raw := resp.read(1):
+                    chunk = decoder.decode(raw)
+                    if chunk:
+                        yield chunk
+                tail = decoder.decode(b"", final=True)
+                if tail:
+                    yield tail
+        except urllib.error.HTTPError as exc:
+            detail = self._auth_error_detail(exc.code, _detail_from_http_error(exc))
+            raise ApiError(exc.code, detail) from exc
+        except urllib.error.URLError as exc:
+            raise ClientError(
+                f"cannot reach the Flash service at {self.api_url} ({exc.reason}); "
+                "check your network connection and FLASH_API_URL"
+            ) from exc
+
+
+def client_from_config(require_key: bool = True) -> ApiClient:
+    """Build a client from the stored credentials; fail with a clear hint when logged out."""
+    api_url, api_key, key_source = load_credentials_with_source()
+    if require_key and not api_key:
+        raise ClientError(
+            "not logged in — run `flash login` with your freesolo API key (or set FREESOLO_API_KEY)"
+        )
+    return ApiClient(api_url, api_key, key_source=key_source)

flash/client/runtime_secrets.py

@@ -0,0 +1,69 @@
+"""Client-side runtime secrets for managed runs.
+
+These values are read on the user's machine and sent only with the submit request. They are not
+part of JobSpec/TOML, and the control plane must not persist them in run status or artifacts.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+DEFAULT_RUNTIME_SECRET_KEYS = frozenset({"WANDB_API_KEY"})
+
+
+def _runtime_secret_keys(keys: tuple[str, ...] | list[str] | set[str] | None = None) -> set[str]:
+    return set(DEFAULT_RUNTIME_SECRET_KEYS) | {str(key) for key in (keys or ())}
+
+
+def _read_env_file(path: Path, keys: set[str]) -> dict[str, str]:
+    if not path.exists() or not path.is_file():
+        return {}
+    out: dict[str, str] = {}
+    try:
+        lines = path.read_text(errors="ignore").splitlines()
+    except OSError:
+        return out
+    for line in lines:
+        s = line.strip()
+        if not s or s.startswith("#") or "=" not in s:
+            continue
+        key, value = s.split("=", 1)
+        key = key.strip()
+        if key not in keys:
+            continue
+        value = value.strip().strip('"').strip("'")
+        if value:
+            out[key] = value
+    return out
+
+
+def runtime_secrets_from_local_env(
+    config_path: str | os.PathLike[str] | None = None,
+    keys: tuple[str, ...] | list[str] | set[str] | None = None,
+) -> dict[str, str]:
+    """Collect supported run secrets from the user's local env.
+
+    Process environment wins. As a convenience for local project workflows, also read `.env` and
+    `.env.local` in the current directory and next to the config file. This deliberately does not
+    scan arbitrary parent directories or serialize secrets into the run spec.
+    """
+
+    wanted = _runtime_secret_keys(keys)
+    secrets = {key: value for key in wanted if (value := os.environ.get(key))}
+    candidates = [Path.cwd() / ".env", Path.cwd() / ".env.local"]
+    if config_path:
+        cfg_dir = Path(config_path).expanduser().resolve().parent
+        candidates.extend([cfg_dir / ".env", cfg_dir / ".env.local"])
+    for path in candidates:
+        for key, value in _read_env_file(path, wanted).items():
+            secrets.setdefault(key, value)
+    required = {str(key) for key in (keys or ())}
+    missing = sorted(required - set(secrets))
+    if missing:
+        raise ValueError(
+            "missing declared environment secret(s): "
+            f"{', '.join(missing)}. Set them in your shell or local .env file before submitting; "
+            "do not put secret values in TOML."
+        )
+    return secrets

flash/client/specs.py

@@ -0,0 +1,20 @@
+"""Turn a locally validated JobSpec into the payload sent to the control plane.
+
+The client fills default pip requirements for Freesolo environments unless the
+config provided an explicit ``[environment] pip`` escape hatch.
+"""
+
+from __future__ import annotations
+
+from flash.spec import JobSpec
+
+
+def spec_payload(spec: JobSpec) -> dict:
+    out = spec.to_dict()
+    if not spec.environment.pip:
+        from flash.envs.registry import worker_pip_for_env
+
+        pip = worker_pip_for_env(spec.environment.id)
+        if pip:
+            out["environment"]["pip"] = pip
+    return out

flash/cost/__init__.py

@@ -0,0 +1,16 @@
+"""Flash training-cost estimator: a deterministic, equation-based pre-flight estimate
+(``estimate_cost``) of cost = wall-clock hours x market $/hr. No output multiplier."""
+
+from __future__ import annotations
+
+from .analytical import estimate_cost
+from .spec import estimate_for_spec, runconfig_from_spec
+from .types import CostEstimate, RunConfig
+
+__all__ = [
+    "CostEstimate",
+    "RunConfig",
+    "estimate_cost",
+    "estimate_for_spec",
+    "runconfig_from_spec",
+]

flash/cost/analytical.py

@@ -0,0 +1,175 @@
+"""The analytical cost model: total = wall-clock hours x GPU $/hr, where wall = cold-start
+setup + steps x per-step time (a FLOPs/MFU estimate). GRPO splits each step into a vLLM
+rollout + reward grading + policy/reference update."""
+
+from __future__ import annotations
+
+import math
+
+from flash.providers.allocator import required_vram_gb, vram_headroom
+
+from .facts import (
+    download_weight_gb,
+    gpu_hourly_usd,
+    gpu_tflops,
+    gpu_vram_gb,
+    model_quant,
+    pick_gpu,
+    reward_seconds_per_completion,
+    total_params_b,
+)
+from .types import CostEstimate, RunConfig
+
+# FLOPs per token per active-parameter.
+SFT_FLOPS_PER_TOKEN_PER_PARAM = 6.0  # forward (2) + backward (4)
+GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM = 2.0  # autoregressive rollout forward
+GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM = 8.0  # policy fwd+bwd (6) + frozen-ref fwd (2)
+
+# Model-FLOPs utilization (fraction of peak sustained), calibrated against real RunPod
+# wall clock. LoRA + small batches sit well below dense-pretraining MFU.
+MFU_TRAIN = 0.35  # GRPO policy/reference update
+MFU_SFT_TRAIN = 0.25  # SFT fwd/bwd (smaller effective batch, long sequences)
+MFU_DECODE = 0.12  # batched vLLM rollout (decode is memory-bandwidth-bound)
+
+# Reward grading is CONCURRENT: a step's completions score in parallel slots, so the reward
+# wall is ceil(completions / slots) waves x latency, not completions x latency.
+REWARD_CONCURRENCY = 16.0
+
+# Cold-start overhead (seconds): container boot + deps + model load (+ vLLM init for GRPO).
+#
+# Calibrated against a real fresh-worker run (0.8B SFT, RTX 3090 @ $0.239/hr) whose billed wall
+# was ~708s for only ~26 priced steps -- i.e. cold start, not training, dominated. A fresh worker
+# spent ~12.5 min in `sft_model_load` alone (download + checkpoint deserialize + GPU placement +
+# framework/CUDA init), so the MODEL-LOAD term -- not boot/deps -- is the dominant cost of a short
+# job. MODEL_LOAD_BASE_S is the fixed (size-independent) load/init overhead; the download term on
+# top of it scales with checkpoint size, so bigger models pay a longer cold start.
+WORKER_BOOT_S = 120.0  # container pull + start
+DEPS_INSTALL_S = 90.0  # pip/uv resolve + install
+MODEL_LOAD_BASE_S = 235.0  # fixed checkpoint deserialize + GPU placement + framework/CUDA init
+VLLM_INIT_S = 120.0
+DOWNLOAD_RATE_GBPS = 0.4  # effective HF snapshot download (hf_transfer), on top of the base load
+
+DEFAULT_WALL_CAP_S = 24 * 3600  # spec gpu.max_wall_seconds default
+
+
+def _fmt_duration(seconds: float) -> str:
+    """Human duration for notes: seconds < 1m, minutes < 1h, else whole/1-decimal hours."""
+    if seconds < 60:
+        return f"{seconds:.0f}s"
+    if seconds < 3600:
+        return f"{seconds / 60:.0f}m"
+    hours = seconds / 3600
+    return f"{hours:.0f}h" if abs(hours - round(hours)) < 1e-9 else f"{hours:.1f}h"
+
+
+def setup_seconds(config: RunConfig) -> float:
+    """Cold-start wall time billed before the first optimizer step: container boot + deps + model
+    load (a fixed deserialize/placement/init base + a size-scaled download), plus vLLM init for
+    GRPO. The model-load term dominates a short job's bill (see the constants above)."""
+    model_load = MODEL_LOAD_BASE_S + download_weight_gb(config.model_id) / DOWNLOAD_RATE_GBPS
+    s = WORKER_BOOT_S + DEPS_INSTALL_S + model_load
+    if config.is_grpo:
+        s += VLLM_INIT_S
+    return s
+
+
+def seconds_per_step(config: RunConfig, gpu: str) -> float:
+    """Steady-state wall time for one optimizer step on ``gpu``."""
+    n = config.normalized()
+    params = total_params_b(n.model_id) * 1e9
+    peak = gpu_tflops(gpu) * 1e12  # FLOP/s
+
+    if not n.is_grpo:
+        flops = SFT_FLOPS_PER_TOKEN_PER_PARAM * params * (n.batch_size * n.seq_len)
+        return flops / (peak * MFU_SFT_TRAIN)
+
+    # GRPO step = rollout (G completions/prompt) + concurrent reward grading + policy/ref update.
+    completions = n.batch_size * n.group_size
+    gen_tokens = completions * n.completion_len
+    gen_s = (GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_DECODE)
+    update_s = (GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_TRAIN)
+    latency = reward_seconds_per_completion(n.reward_seconds_per_completion)
+    reward_s = math.ceil(completions / REWARD_CONCURRENCY) * latency  # ceil: a partial wave still costs one latency
+    return gen_s + reward_s + update_s
+
+
+def select_gpu(config: RunConfig) -> tuple[str, int]:
+    """(chosen GPU class, required VRAM GB): the cheapest fitting class for the cost.
+
+    Uses ``pick_gpu``, which (unlike the submit-time allocator) intentionally stays gate-free —
+    it considers every fitting class, validated or not — so the estimate reflects the cheapest
+    card that *could* run the job. The live allocator restricts to the validated pool, so the
+    actually-provisioned class can be pricier than this. Catalog sizing is offline/deterministic."""
+    total_params_b(config.model_id)  # catalog-only: reject a non-catalog model before any (HF) sizing
+    need = required_vram_gb(
+        config.model_id,
+        config.method,
+        train=config.train_knobs(),
+        thinking=config.thinking,
+    )
+    gpu = pick_gpu(need, provider=config.provider)
+    return gpu, need
+
+
+def _notes(config: RunConfig, raw_train_s: float, wall_capped: bool, cap_s: float) -> tuple[str, ...]:
+    n = config.normalized()
+    notes: list[str] = []
+    if (quant := model_quant(n.model_id)) != "bf16":
+        notes.append(f"{quant}: smaller VRAM footprint -> cheaper GPU class fits")
+    if n.is_grpo:
+        comps = n.batch_size * n.group_size
+        rsec = reward_seconds_per_completion(n.reward_seconds_per_completion)
+        notes.append(
+            f"GRPO step = vLLM rollout of {n.batch_size}x{n.group_size}={comps} completions "
+            f"@ {n.completion_len} tok + reward ({rsec:.2f}s/completion"
+            + (f", env {n.environment}" if n.environment else "")
+            + ") + policy+reference update"
+        )
+    notes.append(f"GPU sized with {vram_headroom() - 1:.0%} VRAM headroom; static GPU $/hr")
+    if wall_capped:
+        per_seed = "" if config.setup_repeats == 1 else "per-seed "
+        notes.append(
+            f"training clamped to fit the {_fmt_duration(cap_s)} {per_seed}wall cap "
+            f"(after setup; uncapped: {_fmt_duration(raw_train_s)})"
+        )
+    return tuple(notes)
+
+
+def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S) -> CostEstimate:
+    """Deterministic pre-flight cost calculation."""
+    gpu, need = select_gpu(config)
+    hourly = gpu_hourly_usd(gpu, provider=config.provider)
+    # Mirror the runner's max(60, max_wall_seconds) floor so a sub-60s cap isn't underpriced.
+    cap_s = max(60.0, float(config.max_wall_seconds)) if config.max_wall_seconds is not None else wall_cap_s
+
+    # Each seed is its own job (own cold start + own wall cap): price one seed, clamp, x seeds.
+    seeds = config.setup_repeats
+    setup_per_seed = setup_seconds(config)
+    sps = seconds_per_step(config, gpu)
+    raw_train_per_seed = (config.steps / seeds) * sps
+
+    # The cap is on total per-seed wall; setup is billed too, so clamp training to fit it.
+    wall_capped = (setup_per_seed + raw_train_per_seed) > cap_s
+    setup_per_seed = min(setup_per_seed, cap_s)
+    train_per_seed = max(0.0, cap_s - setup_per_seed) if wall_capped else raw_train_per_seed
+
+    setup, train = setup_per_seed * seeds, train_per_seed * seeds
+    wall = setup + train
+
+    return CostEstimate(
+        model_id=config.model_id,
+        method=config.method,
+        steps=config.steps,
+        gpu=gpu,
+        provider=config.provider,
+        gpu_vram_gb=gpu_vram_gb(gpu),
+        required_vram_gb=need,
+        gpu_hourly_usd=hourly,
+        setup_seconds=setup,
+        seconds_per_step=sps,
+        train_seconds=train,
+        wall_clock_seconds=wall,
+        wall_capped=wall_capped,
+        total_usd=wall / 3600.0 * hourly,
+        notes=_notes(config, raw_train_per_seed, wall_capped, cap_s),
+    )