freesolo-flash-dev 1.0.87__tar.gz → 1.0.89__tar.gz

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/Dockerfile.worker

@@ -268,6 +268,15 @@ print('worker stack OK | torch', torch.__version__, '| cudnn', cudnn, '| triton'
 # flash's, so a job that sets WANDB_API_KEY loses its curves and run link unless wandb is importable
 # in here; resolve_verl_loggers() degrades to console when it is not, which is the right behaviour
 # for a foreign FLASH_VERL_PYTHON but silently drops W&B on the default image (codex[bot]).
+#
+# cachetools/uvicorn/fastapi/torchvision/tqdm/pyarrow are named for the same reason
+# resolve_verl_python names them: verl imports them at MODULE level on the launch path (main_ppo ->
+# ppo.ray_trainer -> rollout.llm_server imports cachetools; rollout.utils imports uvicorn + fastapi)
+# yet declares none of them. Some arrive as transitive vllm deps today, but that is vllm's dependency
+# choice, not a contract verl states. This image sets FLASH_VERL_PYTHON, so live workers never take
+# the resolve_verl_python path where that hardening lives -- letting the two lists drift means the
+# fallback protects nobody and a vllm respin breaks launch on the default image alone. Keep them in
+# lockstep with backend_common.py's install list.
 ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@1bea7d6825bbb9d2164e86e379b3680e7c53bb8a
 ENV FLASH_VERL_PYTHON=/opt/verl-venv/bin/python
 RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" > /tmp/verl-overrides.txt \
@@ -282,20 +291,41 @@ RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" > /tmp/verl-o
         "TransferQueue==0.1.7" \
         liger-kernel "bitsandbytes>=0.49" qwen-vl-utils \
         "wandb>=0.17" \
+        cachetools uvicorn fastapi torchvision tqdm pyarrow \
+        "flash-linear-attention @ git+https://github.com/fla-org/flash-linear-attention.git@f0e213dbd8b5fb90c3c7eca869ac1706d5377139" \
+        "tilelang==0.1.11" \
+        "apache-tvm-ffi==0.1.11" \
     && rm -f /tmp/verl-overrides.txt \
     && uv cache clean
 
 # flash-attn is REQUIRED in here, not best-effort like it is in the main interpreter above. All three
-# backends hard-enable remove-padding (rl_train.py:336, sft_train.py:164, opd_train.py:1505), and verl's
-# CUDA remove-padding path imports flash_attn.bert_padding UNGUARDED
-# (verl/utils/attention_utils.py:30, torch_functional.py:627,657). There is no SDPA fallback on that
-# path: without the module the run dies at the first training batch, which the import-only assert
-# below would not catch. Reuses FLASH_ATTN_SPEC so the venv gets the same wheel as the main
-# interpreter instead of a second source build.
+# backends default remove-padding ON (rl_train.py, sft_train.py, opd_train.py; only a GDN model whose
+# child cannot reset packed boundaries turns it off), and verl's CUDA remove-padding path imports
+# flash_attn.bert_padding UNGUARDED (verl/utils/attention_utils.py:30, torch_functional.py:627,657).
+# There is no SDPA fallback on that path: without the module the run dies at the first training
+# batch, which the import-only assert below would not catch. Reuses FLASH_ATTN_SPEC so the venv gets
+# the same wheel as the main interpreter instead of a second source build.
 RUN TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0 10.0 12.0" MAX_JOBS=4 \
     uv pip install --python /opt/verl-venv/bin/python --no-build-isolation "${FLASH_ATTN_SPEC}" \
     && uv cache clean
 
+# causal_conv1d in the VERL VENV, which is where the model actually trains. The main interpreter's
+# copy above does not reach here: /opt/verl-venv is built without --system-site-packages, so the two
+# have disjoint paths. Together with fla's cu_seqlens (installed above) this is what makes the packed
+# GDN boundary resets real rather than decorative: engine.worker.backend_common's boundary shim hands
+# the GDN layer cu_seq_lens_q + seq_idx, but transformers' pure-torch fallbacks take **kwargs and
+# silently DISCARD both, so a child missing these kernels trains across example boundaries while
+# looking patched. verl_child_can_reset_gdn_boundaries probes this venv for exactly that reason.
+# Best-effort with the same uninstall-on-broken-import guard as the main interpreter: a present but
+# ABI-broken package passes transformers' find_spec probe and then crashes at model load, and a
+# failed build must degrade to unpacked training (the gate returns False) rather than fail the image.
+RUN TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0 10.0 12.0" CAUSAL_CONV1D_FORCE_BUILD=TRUE MAX_JOBS=4 \
+    uv pip install --python /opt/verl-venv/bin/python "causal-conv1d==1.6.2.post1" --no-build-isolation \
+    && /opt/verl-venv/bin/python -c "import causal_conv1d" \
+    && echo "causal_conv1d (verl venv): installed (GDN boundary resets enabled)" \
+    || { echo "causal_conv1d (verl venv): build/import failed -> GDN trains unpacked (padded fallback)"; \
+         uv pip uninstall --python /opt/verl-venv/bin/python causal-conv1d 2>/dev/null || true; }
+
 # Sanity: the verl venv must be able to LAUNCH, not merely install. This is the assert that would
 # have caught the shipped-but-unrunnable venv. Import the three entrypoints flash actually invokes
 # (rl_train -> main_ppo, opd_plugin -> main_ppo_sync, sft_train -> sft_trainer; note it is
@@ -303,6 +333,11 @@ RUN TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0 10.0 12.0" MAX_JOBS=4 \
 # where a version mismatch would surface, and the fork-only rollout field the GRPO path emits as a
 # hydra override. Also assert the Qwen3.5 archs are registered, since that is the entire reason this
 # venv carries 0.19.1 instead of 0.11.2.
+#
+# fla is asserted HARD here (causal_conv1d is not, matching its best-effort install above): without
+# it transformers binds chunk_gated_delta_rule to the pure-torch fallback, which accepts cu_seqlens
+# and discards it, so packed GDN runs would train across example boundaries. A silently fla-less
+# child is exactly the failure this assert exists to make loud at build time rather than on a paid GPU.
 RUN /opt/verl-venv/bin/python -c "import importlib.metadata as M; \
 import importlib; \
 [importlib.import_module(m) for m in ('verl.trainer.main_ppo','verl.trainer.main_ppo_sync','verl.trainer.sft_trainer')]; \
@@ -310,6 +345,8 @@ import importlib; \
 from verl.workers.config.rollout import RolloutConfig; \
 assert 'mask_truncated_completions' in RolloutConfig.__dataclass_fields__, 'verl is not the freesolo fork (no mask_truncated_completions)'; \
 importlib.import_module('flash_attn.bert_padding'); \
+from transformers.utils.import_utils import is_flash_linear_attention_available; \
+assert is_flash_linear_attention_available(), 'flash-linear-attention missing from the verl venv: packed GDN boundary resets would be silently discarded'; \
 assert M.version('verl') == '0.8.0', 'opd structured gate requires verl exactly 0.8.0, got ' + M.version('verl'); \
 assert M.version('xgrammar') == '0.1.25', 'opd structured gate requires xgrammar exactly 0.1.25, got ' + M.version('xgrammar'); \
 from vllm import ModelRegistry; \
@@ -318,7 +355,9 @@ missing = {'Qwen3_5ForConditionalGeneration','Qwen3_5MoeForConditionalGeneration
 assert not missing, 'vllm cannot serve the default-model archs: ' + repr(sorted(missing)); \
 print('verl venv OK | verl', M.version('verl'), '| vllm', M.version('vllm'), '| torch', M.version('torch'), '| numpy', M.version('numpy'), '| xgrammar', M.version('xgrammar')); \
 print('  entrypoints: main_ppo (grpo), main_ppo_sync (opd), sft_trainer (sft)'); \
-print('  qwen3.5 archs registered; mask_truncated_completions present')"
+print('  qwen3.5 archs registered; mask_truncated_completions present'); \
+from transformers.utils.import_utils import is_causal_conv1d_available; \
+print('  gdn boundary resets: fla yes, causal_conv1d ' + ('yes' if is_causal_conv1d_available() else 'NO -> gdn trains padded'))"
 
 # RunPod Serverless worker entrypoint (Pattern D — Flash custom image). Generate /rp_handler.py
 # from the real _train_body source in providers/runpod/train/endpoints.py (single source of truth;

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 1.0.87
+Version: 1.0.89
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash
@@ -154,6 +154,48 @@ flash models deploy RUN_ID             # serve the trained adapter
 flash models chat RUN_ID -m "hello"    # talk to it
 ```
 
+### Workload profiles (SFT)
+
+The first `flash train` or `flash train --cost` on a new SFT config does not print a quote.
+It reports that no workload profile exists yet and starts one:
+
+```
+no exact workload profile exists for this config yet, so there is no training quote to
+print. the server started a separate profile run that loads your environment and tokenizes
+the exact dataset this training would consume.
+that profile run is real work and is billed on its own (estimated $0.03); no training run
+was created, no training gpu was allocated, and nothing was charged for training.
+follow it with `flash runs status profile-sft-...`, then re-run this command once it
+reports done.
+```
+
+A profile run loads your environment at its pinned SHA, renders and tokenizes every example
+this config would train on, and records aggregates: retained and dropped examples, tokens per
+epoch, supervised tokens, realized max length, packed blocks, and the update horizon. The SFT
+quote is then computed from those measured tokens rather than from an assumed average example
+length, and the same profile is what the training worker trains from.
+
+Consequences worth knowing before you submit:
+
+- **Profiles are separate runs and separate charges.** A profile appears in `flash runs list`
+  under its own id and is billed for its own (CPU-only, short) work. It is never rolled into
+  the training charge, and a failed profile cannot become training spend.
+- **Quoting fails closed.** If no trustworthy matching profile exists, no training run is
+  created, no GPU is allocated, and no quote is persisted. There is no fallback estimate.
+- **The cache key is the workload, not the run.** Profile ids are derived from environment id,
+  resolved SHA and params; model, revision and tokenizer revision; `seed`; `thinking`; worker
+  env; and the `[train]` fields `epochs`, `batch_size`, `max_context_tokens`, `max_steps` and
+  `max_examples`. Change any of them and it is a different workload needing its own profile.
+  Everything else about a run, including which GPU it lands on, is outside the key.
+- **Profiles are shared across users.** The id is a hash of that workload, not of your account,
+  so if someone else already measured your exact config you wait for their profile and are not
+  charged for a second one. Their run is not readable by your key, so the CLI tells you to
+  wait rather than pointing you at a run id that would answer 404.
+- **A failed profile is retried, not final.** If a profile fails or is cancelled, the next
+  submission of that config starts a replacement rather than reporting the workload as
+  permanently unquotable. Because the id is shared, exactly one of the waiting submitters
+  launches the replacement and the rest wait on it.
+
 Run management lives under `flash runs` (`status`, `log`, `cancel`, `checkpoint`) and
 serving under `flash models` (`deploy`, `chat`, `deployments`, `undeploy`, `export`).
 `flash models` on its own lists supported base models and `flash gpus` lists GPU classes

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/README.md

@@ -91,6 +91,48 @@ flash models deploy RUN_ID             # serve the trained adapter
 flash models chat RUN_ID -m "hello"    # talk to it
 ```
 
+### Workload profiles (SFT)
+
+The first `flash train` or `flash train --cost` on a new SFT config does not print a quote.
+It reports that no workload profile exists yet and starts one:
+
+```
+no exact workload profile exists for this config yet, so there is no training quote to
+print. the server started a separate profile run that loads your environment and tokenizes
+the exact dataset this training would consume.
+that profile run is real work and is billed on its own (estimated $0.03); no training run
+was created, no training gpu was allocated, and nothing was charged for training.
+follow it with `flash runs status profile-sft-...`, then re-run this command once it
+reports done.
+```
+
+A profile run loads your environment at its pinned SHA, renders and tokenizes every example
+this config would train on, and records aggregates: retained and dropped examples, tokens per
+epoch, supervised tokens, realized max length, packed blocks, and the update horizon. The SFT
+quote is then computed from those measured tokens rather than from an assumed average example
+length, and the same profile is what the training worker trains from.
+
+Consequences worth knowing before you submit:
+
+- **Profiles are separate runs and separate charges.** A profile appears in `flash runs list`
+  under its own id and is billed for its own (CPU-only, short) work. It is never rolled into
+  the training charge, and a failed profile cannot become training spend.
+- **Quoting fails closed.** If no trustworthy matching profile exists, no training run is
+  created, no GPU is allocated, and no quote is persisted. There is no fallback estimate.
+- **The cache key is the workload, not the run.** Profile ids are derived from environment id,
+  resolved SHA and params; model, revision and tokenizer revision; `seed`; `thinking`; worker
+  env; and the `[train]` fields `epochs`, `batch_size`, `max_context_tokens`, `max_steps` and
+  `max_examples`. Change any of them and it is a different workload needing its own profile.
+  Everything else about a run, including which GPU it lands on, is outside the key.
+- **Profiles are shared across users.** The id is a hash of that workload, not of your account,
+  so if someone else already measured your exact config you wait for their profile and are not
+  charged for a second one. Their run is not readable by your key, so the CLI tells you to
+  wait rather than pointing you at a run id that would answer 404.
+- **A failed profile is retried, not final.** If a profile fails or is cancelled, the next
+  submission of that config starts a replacement rather than reporting the workload as
+  permanently unquotable. Because the id is shared, exactly one of the waiting submitters
+  launches the replacement and the rest wait on it.
+
 Run management lives under `flash runs` (`status`, `log`, `cancel`, `checkpoint`) and
 serving under `flash models` (`deploy`, `chat`, `deployments`, `undeploy`, `export`).
 `flash models` on its own lists supported base models and `flash gpus` lists GPU classes

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/flash/catalog.py

@@ -370,7 +370,8 @@ MODELS: dict[str, ModelInfo] = {
         algos=ALGORITHMS,
         min_vram_gb=48,
         min_disk_gb=120,
-        # NOT QLoRA: peft bnb merge during GRPO rollout diverges trainer precision -> TRL ratio collapses to 0.
+        # NOT QLoRA: a peft bnb merge during GRPO rollout diverges the rollout's precision from the
+        # trainer's, and the importance ratio those two log-probs form collapses to 0.
         grpo_min_vram_gb=80,
         quant="bf16",
         recommended_gpu="A100 PCIe",

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/flash/cli/TRAINING.md

@@ -1024,8 +1024,7 @@ watch `truncation_rate`, which counts completions not ending in EOS and is not s
 For pure multi-turn GRPO, Flash gives each Flash-owned vLLM generation request a managed
 10-to-60-minute absolute deadline and at most two physical attempts. A timed-out request is
 aborted before retry. Enforcement is cooperative between engine polls, and there is no total
-episode elapsed-time cutoff. This policy does not time out OPD, TRL-native tool loops, or
-environment calls.
+episode elapsed-time cutoff. This policy does not time out OPD or environment calls.
 
 > **The reward-hacking signature:** a smoothed reward rising while mean generated
 > length collapses. Whenever any shortness or format pressure is active, verify the
@@ -1039,13 +1038,11 @@ environment calls.
 
 **First, check that per-turn credit is available to you at all.** It is supported on the
 pure multi-turn rollout path — an `EnvironmentMultiTurn` that drives its own turn loop. A
-multi-turn environment that exposes **tools** runs on TRL's native tool loop instead, and
-there `per_turn` is not implemented: the worker raises
-`credit_assignment='per_turn' is not supported for tool-calling multi-turn environments`
-and the run dies at startup. Nothing catches this earlier — `flash train --dry-run` accepts
-the config, because the trainer choice depends on the environment object, not the spec. If
-your environment exposes tools, stay on `per_episode`; emitting `per_turn_rewards` does not
-unlock it.
+multi-turn environment that exposes **tools** is not supported at all: the worker raises
+`verl grpo does not support openai function-calling tool environments; this env reports
+is_tool_env` and the run dies at startup. Nothing catches this earlier —
+`flash train --dry-run` accepts the config, because the check depends on the environment
+object, not the spec.
 
 On the pure multi-turn path, setting `credit_assignment = "per_turn"` is only _half_ the
 request. The trainer reads the

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/flash/cli/__init__.py

@@ -89,7 +89,7 @@ _HELP_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
             ("env setup", "scaffold a starter Freesolo environment"),
             ("env list", "list local environment sources"),
             ("env test", "validate a local environment offline"),
-            ("env eval", "score held-out suites against a deployed model"),
+            ("env eval", "score a deployed model against its published environment's suites"),
             ("env push", "upload a local environment"),
             ("env pull", "download a published environment or file"),
             ("env delete", "delete a published environment"),
@@ -396,20 +396,17 @@ def _build_parser() -> argparse.ArgumentParser:
 
     env_eval = env_sub.add_parser(
         "eval",
-        help="score held-out environment suites against a deployed model",
-        description="score held-out environment suites against a deployed model",
+        help="score a deployed model against its own environment's held-out suites",
+        description=(
+            "score a deployed model against the held-out suites of the published environment "
+            "its run trains on"
+        ),
     )
     env_eval.add_argument(
         "target",
         metavar="TARGET",
         help="a bare RUN_ID, RUN_ID/step-N, or full immutable adapter revision",
     )
-    env_eval.add_argument(
-        "path",
-        nargs="?",
-        default=".",
-        help="local environment directory or environment.py path",
-    )
     # the same two knobs `env test` exposes, and for the same reason: an env whose
     # `load_environment()` requires a difficulty or reads a non-default split cannot be evaluated
     # at all without them, and a held-out suite scored against a differently-configured
@@ -449,14 +446,17 @@ def _build_parser() -> argparse.ArgumentParser:
         help=f"parallel model requests (1-{_MAX_CONCURRENCY})",
     )
     env_eval.add_argument(
-        "--upload",
-        action="store_true",
-        help="record the results under a project so they show in the dashboard",
+        "--no-upload",
+        dest="upload",
+        action="store_false",
+        help="score locally without recording the results in the dashboard",
     )
     env_eval.add_argument(
         "--project",
         metavar="PROJECT_ID",
-        help="project id to record results under (required with --upload)",
+        help=(
+            "project id to record results under (default: the project the evaluated run belongs to)"
+        ),
     )
     env_eval.set_defaults(func=cmd_env_eval)
 
@@ -749,8 +749,10 @@ def _warn_if_login_shadowed(args) -> None:
     """Surface an ambient FREESOLO_API_KEY that binds a command to another org."""
     if getattr(args, "func", None) not in _ORG_BINDING_COMMANDS:
         return
-    # `train --cost` is catalog-only: it never loads credentials or reaches an organization, so a
-    # warning that names the environment key's org would describe a request this command never makes.
+    # `train --cost` cannot be classified here: the algorithm is only known once the config is
+    # parsed, which happens inside the command. grpo/opd stay catalog-only and must not warn about
+    # an org they never reach; sft authenticates and can start a billed profile run, so it emits
+    # this warning itself (see commands._cmd_train_cost_sft).
     if getattr(args, "cost", False):
         return
     message = shadowed_login_warning()

{freesolo_flash_dev-1.0.87 → freesolo_flash_dev-1.0.89}/flash/cli/commands.py

@@ -21,7 +21,11 @@ from flash.client import (
     save_credentials,
     verify_freesolo_key,
 )
-from flash.client.config import load_credentials, load_credentials_with_source
+from flash.client.config import (
+    load_credentials,
+    load_credentials_with_source,
+    shadowed_login_warning,
+)
 from flash.client.runtime_secrets import runtime_secrets_from_local_env
 from flash.client.specs import spec_payload
 from flash.cost.spec import runconfig_from_spec
@@ -29,7 +33,6 @@ from flash.runner import TERMINAL_STATES
 from flash.schema import (
     ConfigError,
     spec_and_train_keys_from_file,
-    spec_from_file,
     train_schema_metadata,
 )
 from flash.serve.urls import is_freesolo_hosted_url
@@ -320,12 +323,14 @@ def cmd_env_list(args) -> int:
 def _cmd_train_cost(args) -> int:
     """`flash train --cost`: print the pre-flight USD cost for the config and exit (no submit).
 
-    Catalog-only and deterministic. SFT cost never imports the environment; it requires a positive
-    [train].max_examples row count instead of guessing or locally counting a dataset."""
-    from flash.cost import estimate_cost
+    grpo and opd quote offline from the catalog. sft has no offline quote at all: its cost is
+    derived from the exact tokenized dataset, and only a workload profile knows that, so sft asks
+    the server for the same profile-backed quote a real submit would freeze. There is deliberately
+    no analytical sft fallback -- a guessed row count is what this whole path exists to remove.
+    """
     from flash.lora_rank import preflight_train_context_within_serving
 
-    spec = spec_from_file(
+    spec, authored_train_keys = spec_and_train_keys_from_file(
         args.config,
         run_id=None,
         overrides=args.overrides,
@@ -333,6 +338,15 @@ def _cmd_train_cost(args) -> int:
         project_required=True,
     )
     preflight_train_context_within_serving(spec)
+    if spec.algorithm == "sft":
+        return _cmd_train_cost_sft(args, spec, authored_train_keys)
+    return _cmd_train_cost_offline(spec)
+
+
+def _cmd_train_cost_offline(spec) -> int:
+    """Catalog-only quote for the algorithms that do not need workload evidence yet (grpo, opd)."""
+    from flash.cost import estimate_cost
+
     if spec.train.init_from_adapter:
         # --cost is offline/catalog-only and cannot read the source adapter, so the rank stays at the
         # local default. Warm starts train and are priced at the SOURCE adapter's authoritative rank
@@ -352,6 +366,168 @@ def _cmd_train_cost(args) -> int:
     return 0
 
 
+def _cmd_train_cost_sft(args, spec, authored_train_keys: frozenset[str]) -> int:
+    """Exact sft quote, served by the same authenticated path that freezes a real submit's quote."""
+    # cli._warn_if_login_shadowed() suppresses this warning for `--cost` because the catalog path
+    # never reaches an organization. the sft path does: it authenticates, resolves the project, and
+    # can start a billed profile run, so the warning has to fire here after all.
+    message = shadowed_login_warning()
+    if message:
+        print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
+    client = client_from_config()
+    try:
+        status = client.create_run(
+            spec_payload(spec, authored_train_keys=authored_train_keys),
+            runtime_secrets=runtime_secrets_from_local_env(
+                args.config, keys=spec.environment.secrets
+            )
+            or None,
+            dry_run=True,
+            client_train_schema=_client_train_schema(authored_train_keys),
+        )
+    except ApiError as exc:
+        _raise_if_workload_profile_pending(client, exc)
+        detail = _legacy_train_key_rejection_detail(exc, authored_train_keys)
+        if detail is None:
+            raise
+        raise ApiError(exc.status, detail, detail=detail) from exc
+    _print_exact_sft_cost(status, spec)
+    return 0
+
+
+def _client_train_schema(authored_train_keys: frozenset[str]) -> dict:
+    return {
+        "version": __version__,
+        "fields": train_schema_metadata(),
+        "authored_keys": sorted(authored_train_keys),
+    }
+
+
+def _raise_if_workload_profile_pending(client: ApiClient, exc: ApiError) -> None:
+    """Explain a profile-pending rejection and fail, or return so the caller keeps handling `exc`.
+
+    A miss is not a validation error: the server started a real, separately billed profile run that
+    tokenizes the exact dataset. Saying only "409" would leave the user with a charge they cannot
+    see the reason for, and re-running blindly would look like the same request failing twice.
+    """
+    detail = exc.detail
+    if exc.code != "workload_profile_pending" or not isinstance(detail, dict):
+        return
+    profile_run_id = str(detail.get("profile_run_id") or "")
+    state = str(detail.get("state") or "unknown")
+    # the profile id is deterministic in the workload, so this config's profile may already be
+    # running under another key. that run is not readable here and is not billed here either, so
+    # both the follow-up command and the charge sentence have to change.
+    owned = detail.get("owned") is not False
+    charge = _profile_charge(client, profile_run_id) if owned else None
+    if owned:
+        lines = [
+            "no exact workload profile exists for this config yet, so there is no training quote "
+            "to print. the server started a separate profile run that loads your environment and "
+            "tokenizes the exact dataset this training would consume.",
+            "that profile run is real work and is billed on its own"
+            + (f" (estimated ${charge:.2f})" if charge is not None else "")
+            + "; no training run was created, no training gpu was allocated, and nothing was "
+            "charged for training.",
+            f"follow it with `{CLI_NAME} runs status {profile_run_id}`, then re-run this command "
+            "once it reports done."
+            if profile_run_id
+            else "re-run this command once the profile reports done.",
+        ]
+    else:
+        lines = [
+            "no exact workload profile exists for this config yet, so there is no training quote "
+            "to print. one is already being measured for this exact config and will be reused, so "
+            "nothing was started or charged here.",
+            "re-run this command in a few minutes.",
+        ]
+    for line in lines:
+        print(render.note(line) if render.styled() else line, file=sys.stderr)
+    raise ClientError(
+        f"workload profile {profile_run_id or '(unknown)'} is {state}; "
+        "the exact quote is available once it succeeds"
+    )
+
+
+def _profile_charge(client: ApiClient, profile_run_id: str) -> float | None:
+    """The profile run's own quote, or None when it cannot be read (not owned by this key, etc)."""
+    if not profile_run_id:
+        return None
+    try:
+        status = client.get_run(profile_run_id)
+    except (ApiError, ClientError):
+        return None
+    quote = status.get("estimated_cost_usd") if isinstance(status, dict) else None
+    # bool is an int subclass, so an unchecked isinstance would render a JSON `true` as "$1.00" --
+    # a charge the user is told to expect that no profile ever quoted.
+    if not isinstance(quote, (int, float)) or isinstance(quote, bool):
+        return None
+    return float(quote)
+
+
+def _exact_sft_cost_rows(spec, profile: dict) -> list[tuple[str, str | None]]:
+    """Rows describing the exact measured workload behind an sft quote.
+
+    Only aggregates the server actually returned are shown. A field that is absent is dropped
+    rather than defaulted, so the panel never reports a count the profile did not measure.
+    """
+
+    def count(key: str) -> int | None:
+        value = profile.get(key)
+        return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+    steps = count("authoritative_steps")
+    retained, selected = count("retained_examples"), count("selected_examples")
+    dropped = count("dropped_examples")
+    compute, supervised = (
+        count("authoritative_compute_tokens"),
+        count("authoritative_supervised_tokens"),
+    )
+    packing = str(profile.get("packing_mode") or "")
+    architecture = str(profile.get("architecture_mode") or "")
+    digest = str(profile.get("content_digest") or "")
+    examples = None
+    if retained is not None and selected is not None:
+        examples = f"{retained:,} trained of {selected:,} selected"
+        if dropped:
+            examples += f" ({dropped:,} dropped)"
+    tokens = None
+    if compute is not None:
+        tokens = f"{compute:,} compute"
+        if supervised is not None:
+            tokens += f", {supervised:,} supervised"
+    return [
+        ("run", f"{spec.model}  [SFT{f', {steps} steps' if steps is not None else ''}]"),
+        ("workload", f"{packing} ({architecture})" if packing and architecture else None),
+        ("examples", examples),
+        ("tokens", tokens),
+        ("profile", digest[:12] or None),
+    ]
+
+
+def _print_exact_sft_cost(status: dict, spec) -> None:
+    total = status.get("estimated_cost_usd") if isinstance(status, dict) else None
+    if not isinstance(total, (int, float)) or isinstance(total, bool):
+        raise ClientError(
+            "the server accepted this SFT config but returned no cost estimate; "
+            f"run `{CLI_NAME} train --dry-run` to see the full server response"
+        )
+    profile = status.get("workload_profile")
+    rows = _exact_sft_cost_rows(spec, profile if isinstance(profile, dict) else {})
+    if render.styled():
+        print(render.exact_cost_panel(rows, float(total)))
+    else:
+        for key, value in rows:
+            if value is not None:
+                print(f"{key.ljust(8)}: {value}")
+        print(f"{'TOTAL'.ljust(8)}: ${float(total):.2f}")
+    print(
+        "quoted from the exact tokenized workload, not an assumed row count. no training gpu was "
+        "allocated and nothing was charged for training.",
+        file=sys.stderr,
+    )
+
+
 def _legacy_train_key_rejection_detail(
     exc: ApiError, authored_train_keys: frozenset[str]
 ) -> str | None:
@@ -443,18 +619,16 @@ def cmd_train(args) -> int:
     )
     payload = spec_payload(spec, authored_train_keys=authored_train_keys)
     client = client_from_config()
-    client_train_schema = {
-        "version": __version__,
-        "fields": train_schema_metadata(),
-        "authored_keys": sorted(authored_train_keys),
-    }
+    client_train_schema = _client_train_schema(authored_train_keys)
     runtime_secrets = (
         runtime_secrets_from_local_env(args.config, keys=spec.environment.secrets) or None
     )
     _warn_if_wandb_requested_without_key(spec, runtime_secrets, dry_run=bool(args.dry_run))
     if args.dry_run:
-        # dry-run runs submit-time server preflights without importing user code, allocating a gpu,
-        # or charging anything. a rejection surfaces as the server's error with exit status 1.
+        # dry-run runs submit-time server preflights without allocating a training gpu or charging
+        # for training. a rejection surfaces as the server's error with exit status 1. for sft it
+        # also requires an exact workload profile, so a miss starts a separate billed profile run
+        # (see _raise_if_workload_profile_pending) instead of previewing anything.
         try:
             status = client.create_run(
                 payload,
@@ -463,10 +637,11 @@ def cmd_train(args) -> int:
                 client_train_schema=client_train_schema,
             )
         except ApiError as exc:
+            _raise_if_workload_profile_pending(client, exc)
             detail = _legacy_train_key_rejection_detail(exc, authored_train_keys)
             if detail is None:
                 raise
-            raise ApiError(exc.status, detail) from exc
+            raise ApiError(exc.status, detail, detail=detail) from exc
         compatibility = status.pop("train_schema_compatibility", None)
         _print_train_schema_compatibility(compatibility)
         # the server fails open on a billing-infra problem, so "cost" is only in the validated list
@@ -474,12 +649,22 @@ def cmd_train(args) -> int:
         # equally not a verification -- so treat anything but an explicit True as unverified.
         affordability_verified = status.pop("affordability_verified", None) is True
         cost = "and cost" if affordability_verified else "but NOT cost"
+        # sft additionally required a matching workload profile to get this far, and that profile
+        # run already imported environment.py and tokenized the dataset. claiming otherwise here
+        # would understate what has been checked -- and what has already been billed.
+        environment = (
+            "your environment.py and the exact dataset were already loaded and tokenized by the "
+            "workload profile this quote is built on; model load and gpu/training are first "
+            "exercised on the worker after cold-start."
+            if spec.algorithm == "sft"
+            else "it did NOT import or run your environment.py; dataset loading, "
+            "start_episode/episode shapes, reward/scorer, worker imports, model load, and "
+            "gpu/training are first exercised on the worker after cold-start."
+        )
         print(
             "dry-run validated: config/schema, model+algorithm compatibility, lora rank, "
-            f"runtime-secret presence, warm-start source, serving context cap, {cost}. it did NOT "
-            "import or run your environment.py; dataset loading, start_episode/episode shapes, "
-            "reward/scorer, worker imports, model load, and gpu/training are first exercised on the "
-            "worker after cold-start.",
+            f"runtime-secret presence, warm-start source, serving context cap, {cost}. "
+            f"{environment}",
             file=sys.stderr,
         )
         if not affordability_verified:
@@ -497,11 +682,18 @@ def cmd_train(args) -> int:
         else:
             print(json.dumps(status, indent=2))
         return 0
-    status = client.create_run(
-        payload,
-        runtime_secrets=runtime_secrets,
-        client_train_schema=client_train_schema,
-    )
+    try:
+        status = client.create_run(
+            payload,
+            runtime_secrets=runtime_secrets,
+            client_train_schema=client_train_schema,
+        )
+    except ApiError as exc:
+        # a real submit misses the profile cache the same way a preview does, and the miss starts a
+        # separately billed profile run. without this the user sees a bare 409 for a charge they
+        # were never told about (see _raise_if_workload_profile_pending).
+        _raise_if_workload_profile_pending(client, exc)
+        raise
     run_id = status["run_id"]
     logger.info(
         "submitted run %s: model=%s algorithm=%s gpu=%s",