freesolo-flash-dev 0.2.55__tar.gz → 0.2.57__tar.gz

{freesolo_flash_dev-0.2.55 → freesolo_flash_dev-0.2.57}/.env.example

@@ -5,11 +5,6 @@
 # GPU substrate. RunPod is the default; Vast is opt-in (only required when set).
 RUNPOD_API_KEY=
 VAST_API_KEY=
-# Use the per-arch baked worker images (cu128-<sm>) to skip the ~10-15 min cold-start JIT. Requires
-# the per-SM images published first (.github/workflows/bake-kernel-cache.yml), and a manual re-bake
-# after any worker-deps change. The control-plane Dockerfile sets this to 1 by default; uncomment to
-# enable for a bare `flash-server` deploy.
-# FLASH_WORKER_IMAGE_PER_SM=1
 # HuggingFace token with write access to each run's [train] hf_repo (code upload +
 # streamed checkpoints/adapters land in that per-run dataset repo). The artifact repo
 # is per-run (set in the run TOML's [train] hf_repo), not an operator-wide env var.

{freesolo_flash_dev-0.2.55 → freesolo_flash_dev-0.2.57}/.github/workflows/bake-kernel-cache.yml

@@ -4,8 +4,8 @@ name: flash worker kernel-cache (per-arch)
 # first-use JIT on a cold worker (the regression #194 reintroduced). Each matrix leg:
 #   1. offloads the warmup to a RunPod GPU of that arch (GH runners have none) -> build/kernel_cache,
 #   2. docker build --build-arg BUILD_KERNEL_CACHE=true -> ghcr.io/freesolo-co/flash-worker:cu128-<sm>,
-#   3. pushes it. Activate by setting FLASH_WORKER_IMAGE_PER_SM=1 on the control plane so the
-#      allocator selects the cu128-<sm> tag per GPU class (see flash/providers/runpod/train/deps.py).
+#   3. pushes it. The allocator then selects the cu128-<sm> tag per GPU class for baked arches
+#      (see flash/providers/runpod/train/deps.py).
 #
 # Depends on the base ghcr.io/freesolo-co/flash-worker:cu128 image existing (worker-image.yml) — the
 # warmup runs INSIDE it so the cache matches the image's pinned torch/triton/fla/liger toolchain.

{freesolo_flash_dev-0.2.55 → freesolo_flash_dev-0.2.57}/Dockerfile

@@ -24,17 +24,12 @@ RUN pip install --no-cache-dir ".[server]"
 VOLUME /root/.flash
 EXPOSE 8080
 
-# Use the per-arch baked worker images (ghcr.io/.../flash-worker:cu128-<sm>) so cold workers skip the
-# ~10-15 min first-use JIT; the allocator maps each GPU class to its matching -smXX tag. All validated
-# SMs (sm80/86/89/90/120) are published. Rebakes are MANUAL -- after a Dockerfile.worker/deps change
-# rebuilds :cu128, re-run bake-kernel-cache.yml so the -smXX tags don't ship stale deps (the
-# worker-image build posts a reminder). Override at runtime with `-e FLASH_WORKER_IMAGE_PER_SM=0`.
-#
-# NOTE: this ENV is the default for BARE (non-Infisical) `flash-server` deploys. Under the Infisical
-# entrypoint below, `infisical run` overrides the container env, so for the Infisical-managed deploy
-# set FLASH_WORKER_IMAGE_PER_SM in the vault (path /flash) or add it to INFISICAL_KEEP -- otherwise
-# this default may not reach the server.
-ENV FLASH_WORKER_IMAGE_PER_SM=1
+# The allocator uses the per-arch baked worker images (ghcr.io/.../flash-worker:cu128-<sm>) so cold
+# workers skip the ~10-15 min first-use JIT; each GPU class maps to its matching -smXX tag. All
+# validated SMs (sm80/86/89/90/120) are published; unbaked arches fall back to the base :cu128 image.
+# Rebakes are MANUAL -- after a Dockerfile.worker/deps change rebuilds :cu128, re-run
+# bake-kernel-cache.yml so the -smXX tags don't ship stale deps (the worker-image build posts a
+# reminder).
 
 # secret injection wrapper: no-op passthrough unless INFISICAL_CLIENT_ID is set, else
 # `infisical login` (universal-auth) then `infisical run --path /flash` before the server.

{freesolo_flash_dev-0.2.55 → freesolo_flash_dev-0.2.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 0.2.55
+Version: 0.2.57
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-0.2.55 → freesolo_flash_dev-0.2.57}/flash/cli/commands.py

@@ -23,12 +23,21 @@ from flash.client.config import load_credentials
 from flash.client.runtime_secrets import runtime_secrets_from_local_env
 from flash.client.specs import spec_payload
 from flash.cost.spec import runconfig_from_spec
-from flash.runner import TERMINAL_STATES, new_run_id
+from flash.runner import TERMINAL_STATES
 from flash.schema import ConfigError, spec_from_file
 
 from . import render
 from ._tty import TtyStatusLine
-from .training_doc import TRAINING_MD
+from .env_setup import (  # noqa: F401
+    _REASONING_OPTIONS,
+    _STARTER_DATASET_JSONL,
+    _STARTER_DATASET_MULTITURN_JSONL,
+    _STARTER_ENV_MULTITURN_PY,
+    _STARTER_ENV_PY,
+    _TURN_OPTIONS,
+    _setup_interactive,
+    cmd_env_setup,
+)
 
 logger = get_logger("flash.cli")
 
@@ -98,7 +107,7 @@ def cmd_login(args) -> int:
         print(render.login_failed(str(exc)), file=sys.stderr)
         return 1
     api_url = args.api_url or load_credentials()[0]
-    _ = save_credentials(api_key, api_url=api_url)
+    save_credentials(api_key, api_url=api_url)
     if args.api_key and env_api_key and env_api_key != args.api_key:
         msg = (
             "FREESOLO_API_KEY is set and will override this saved login for future "
@@ -129,414 +138,6 @@ def cmd_whoami(args) -> int:
     return 0
 
 
-_STARTER_ENV_PY = '''\
-"""Starter Freesolo environment.
-
-Edit dataset/train.jsonl and the reward code, then upload with
-`flash env push --name my-env .`.
-
-A managed run should use the returned [environment] id from
-`flash env push --name my-env .`.
-
-This starter keeps a tiny smoke-test dataset in dataset/train.jsonl. Replace it
-with your real training rows before a real run.
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-from freesolo.datasets.types import TaskExample
-from freesolo.environments import EnvironmentSingleTurn, RewardResult
-
-
-DEFAULT_DATASET_PATH = Path(__file__).parent / "dataset" / "train.jsonl"
-
-
-def load_jsonl(path: str | Path):
-    rows = []
-    with Path(path).open() as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                rows.append(json.loads(line))
-    return rows
-
-
-def exact_match_reward(example: TaskExample, response_text: str) -> RewardResult:
-    expected = str(example.output or "").strip()
-    score = 1.0 if expected and expected in response_text else 0.0
-    return RewardResult(score=score, threshold=1.0)
-
-
-class StarterEnv(EnvironmentSingleTurn):
-    dataset = load_jsonl(DEFAULT_DATASET_PATH)
-
-    def build_prompt_messages(self, example: TaskExample, prompt_text: str):
-        return [{"role": "user", "content": example.input}]
-
-    def score_response(self, example: TaskExample, response_text: str) -> RewardResult:
-        return exact_match_reward(example, response_text)
-
-
-def load_environment(dataset_path: str | None = None, **kwargs) -> StarterEnv:
-    env = StarterEnv()
-    if dataset_path:
-        env.dataset = load_jsonl(dataset_path)
-    return env
-'''
-
-_STARTER_DATASET_JSONL = """\
-{"input":"What is 2 + 2?","output":"4"}
-{"input":"What is 3 + 5?","output":"8"}
-"""
-
-
-_STARTER_ENV_MULTITURN_PY = '''\
-"""Starter Freesolo multi-turn environment.
-
-A multi-turn environment runs a bounded episode: the model produces an assistant
-action, `step_episode` advances the world (optionally appending an observation
-message), and the loop repeats until `done` or `max_episode_turns`. The finished
-transcript is graded by `score_episode`.
-
-Edit dataset/train.jsonl and the episode logic, then upload with
-`flash env push --name my-env .`.
-
-A managed run should use the returned [environment] id from
-`flash env push --name my-env .`.
-
-This starter implements a tiny "guess the secret number" game so you can see the
-episode hooks wired end-to-end. Replace it with your real task before a real run.
-
-All three algorithms train off this file:
-- GRPO (configs/rl.toml) rolls out full episodes and optimizes `score_episode`.
-- SFT (configs/sft.toml) learns the gold trajectory. Provide it per row as
-  `output = {"messages": [...]}` (a full assistant/tool trajectory) or a scalar
-  `output` for a single gold assistant turn.
-- OPD (configs/opd.toml) rolls out each episode and distils EVERY assistant turn against
-  the Fireworks GLM teacher, conditioned on the transcript so far — the multi-turn
-  on-policy-distillation objective. The teacher key is platform-managed (nothing to set).
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-from freesolo.datasets.types import TaskExample
-from freesolo.environments import (
-    EnvironmentEpisode,
-    EnvironmentMultiTurn,
-    EnvironmentStepResult,
-    RewardResult,
-)
-
-
-DEFAULT_DATASET_PATH = Path(__file__).parent / "dataset" / "train.jsonl"
-
-# How many assistant guesses the model gets before the episode is forced terminal.
-MAX_TURNS = 5
-
-
-def load_jsonl(path: str | Path):
-    rows = []
-    with Path(path).open() as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                rows.append(json.loads(line))
-    return rows
-
-
-def _secret(example: TaskExample) -> int:
-    return int(str(example.output).strip())
-
-
-class StarterMultiTurnEnv(EnvironmentMultiTurn):
-    dataset = load_jsonl(DEFAULT_DATASET_PATH)
-
-    def start_episode(self, example: TaskExample, prompt_text: str):
-        # The opening prompt shown to the model. `example.input` describes the range.
-        return [
-            {
-                "role": "user",
-                "content": (
-                    f"{example.input}\\n"
-                    f"Reply with a single integer per turn. I will say 'higher', "
-                    f"'lower', or 'correct'. You have {MAX_TURNS} guesses."
-                ),
-            }
-        ]
-
-    def max_episode_turns(self, example: TaskExample) -> int:
-        return MAX_TURNS
-
-    def step_episode(
-        self,
-        example: TaskExample,
-        messages: list,
-        assistant_response: str,
-    ) -> EnvironmentStepResult:
-        # Advance the world after one assistant action. Return done=True to end the
-        # episode, or append an observation message and keep going.
-        try:
-            guess = int(assistant_response.strip().split()[0])
-        except (ValueError, IndexError):
-            return EnvironmentStepResult(
-                done=False,
-                messages=[{"role": "user", "content": "Please reply with a single integer."}],
-            )
-        secret = _secret(example)
-        if guess == secret:
-            return EnvironmentStepResult(done=True, final_response_text=str(guess))
-        hint = "higher" if guess < secret else "lower"
-        return EnvironmentStepResult(
-            done=False,
-            messages=[{"role": "user", "content": hint}],
-        )
-
-    def score_episode(
-        self,
-        example: TaskExample,
-        episode: EnvironmentEpisode,
-    ) -> RewardResult:
-        # Grade the finished transcript. Reward a correct final guess; give partial
-        # credit for getting close so GRPO has a usable gradient.
-        secret = _secret(example)
-        try:
-            final = int(str(episode.response_text).strip())
-        except ValueError:
-            return RewardResult(score=0.0, threshold=1.0)
-        if final == secret:
-            return RewardResult(score=1.0, threshold=1.0, success=True)
-        # Closeness in [0, 1): further guesses score lower.
-        closeness = max(0.0, 1.0 - abs(final - secret) / 100.0)
-        return RewardResult(score=closeness * 0.5, threshold=1.0, success=False)
-
-
-def load_environment(dataset_path: str | None = None, **kwargs) -> StarterMultiTurnEnv:
-    env = StarterMultiTurnEnv()
-    if dataset_path:
-        env.dataset = load_jsonl(dataset_path)
-    return env
-'''
-
-
-# Multi-turn rows: `input` sets up the episode, `output` is the secret number.
-# The scalar `output` also serves as the gold single-turn SFT target; swap in
-# `{"messages": [...]}` to teach a full gold trajectory.
-_STARTER_DATASET_MULTITURN_JSONL = """\
-{"input":"I picked a secret whole number between 1 and 100.","output":"42"}
-{"input":"I picked a secret whole number between 1 and 100.","output":"73"}
-"""
-
-
-_TURN_OPTIONS = [
-    ("single", "single-turn", "one prompt -> one response"),
-    ("multi", "multi-turn", "bounded episode: step_episode / score_episode"),
-]
-_REASONING_OPTIONS = [
-    ("off", "no reasoning", "the model answers directly"),
-    ("on", "reasoning", "thinking = true; spends more tokens per answer"),
-]
-
-
-def _setup_interactive(args) -> bool:
-    """Whether to ask the setup questions interactively.
-
-    Prompt only on a real terminal a human can answer from. Fall back to defaults (no prompt) when
-    --yes is passed, under CI, or when stdin is closed/redirected, so automation never blocks on an
-    unanswered prompt. A pseudo-TTY in CI can report isatty()=True, so CI is checked explicitly; a
-    closed fd 0 can make sys.stdin None, so that is guarded before .isatty()."""
-    if getattr(args, "yes", False):
-        return False
-    if os.environ.get("CI", "").strip().lower() not in ("", "0", "false", "no"):
-        return False
-    stdin = sys.stdin
-    if stdin is None or not stdin.isatty():
-        return False
-    return render.styled()
-
-
-def cmd_env_setup(args) -> int:
-    starter_env = Path("environment.py")
-    dataset = Path("dataset/train.jsonl")
-    # An existing environment.py is the authoritative signal for which turn mode this
-    # scaffold already uses (the dataset is plain JSONL with no reliable mode marker).
-    # Anchor to it so a re-run never leaves a single-turn env beside a multi-turn
-    # dataset (or vice versa); the flag/answer only decides the mode when starting fresh.
-    existing_multi: bool | None = None
-    anchor = "environment.py"
-    if starter_env.exists():
-        existing_multi = "EnvironmentMultiTurn" in starter_env.read_text(encoding="utf-8")
-    elif dataset.exists():
-        # No env.py to anchor on, but the starter multi-turn dataset carries a
-        # distinctive prompt; use it so we don't drop a single-turn env beside it.
-        existing_multi = "secret whole number" in dataset.read_text(encoding="utf-8")
-        anchor = "dataset/train.jsonl"
-
-    # Resolve the turn mode. An existing scaffold wins (warn if a flag disagrees); otherwise an
-    # explicit --single-turn/--multi-turn flag wins; otherwise ask on a terminal, else single-turn.
-    flag_mode = getattr(args, "turn_mode", None)
-    if existing_multi is not None:
-        if flag_mode is not None and (flag_mode == "multi") != existing_multi:
-            have = "multi-turn" if existing_multi else "single-turn"
-            want = "multi-turn" if flag_mode == "multi" else "single-turn"
-            msg = (
-                f"existing {anchor} is {have}; keeping it and ignoring --{want}. "
-                f"Delete environment.py and dataset/train.jsonl first to re-scaffold as {want}."
-            )
-            print(render.warn(msg) if render.styled() else f"warning: {msg}", file=sys.stderr)
-        multi_turn = existing_multi
-    elif flag_mode is not None:
-        multi_turn = flag_mode == "multi"
-    elif _setup_interactive(args):
-        multi_turn = (
-            render.select("How does the model interact with your task?", _TURN_OPTIONS) == "multi"
-        )
-    else:
-        multi_turn = False
-
-    # Resolve reasoning. Like the turn mode, an existing config is authoritative: configs are only
-    # written when absent, so applying a reasoning flag to an already-scaffolded project would
-    # silently no-op (or write one config with reasoning and leave the other without). Anchor to the
-    # existing config's `thinking` state and warn if a flag disagrees; otherwise the flag wins;
-    # otherwise ask on a terminal; else off.
-    rl = Path("configs/rl.toml")
-    sft = Path("configs/sft.toml")
-    existing_reasoning: bool | None = None
-    for cfg in (rl, sft):
-        if cfg.exists():
-            existing_reasoning = "thinking = true" in cfg.read_text(encoding="utf-8")
-            break
-    flag_reason = getattr(args, "reasoning", None)
-    if existing_reasoning is not None:
-        if flag_reason is not None and flag_reason != existing_reasoning:
-            have = "reasoning" if existing_reasoning else "no reasoning"
-            want = "reasoning" if flag_reason else "no-reasoning"
-            msg = (
-                f"existing configs are {have}; keeping them and ignoring --{want}. "
-                f"Delete configs/rl.toml and configs/sft.toml first to re-scaffold with --{want}."
-            )
-            print(render.warn(msg) if render.styled() else f"warning: {msg}", file=sys.stderr)
-        reasoning = existing_reasoning
-    elif flag_reason is not None:
-        reasoning = flag_reason
-    elif _setup_interactive(args):
-        reasoning = render.select("Train with reasoning (thinking)?", _REASONING_OPTIONS) == "on"
-    else:
-        reasoning = False
-
-    env_py = _STARTER_ENV_MULTITURN_PY if multi_turn else _STARTER_ENV_PY
-    dataset_jsonl = _STARTER_DATASET_MULTITURN_JSONL if multi_turn else _STARTER_DATASET_JSONL
-    Path("configs").mkdir(exist_ok=True)
-    Path("dataset").mkdir(exist_ok=True)
-    if not dataset.exists():
-        dataset.write_text(dataset_jsonl)
-    if not starter_env.exists():
-        starter_env.write_text(env_py)
-    env_comment = (
-        "# Environment: upload this project folder with\n"
-        "# `flash env push --name my-env .`, then paste the returned id below.\n"
-        "# If the environment reads secrets with os.environ, list only the env var names here.\n"
-        "# Values are read from your shell or .env at submit time and are not stored in the spec.\n"
-        "[environment]\n"
-        'id = ""\n\n'
-        '# secrets = ["SERPAPI_API_KEY"]\n\n'
-    )
-    # `thinking = true` opts the run into reasoning mode. Reasoning shares the generation budget with
-    # the answer, so GRPO also gets a raised completion budget. These strings are empty when reasoning
-    # is off, keeping the default scaffold byte-for-byte identical.
-    thinking_line = "thinking = true\n" if reasoning else ""
-    rl_reasoning_train = (
-        "max_completion_tokens = 2048  # reasoning shares this budget with the answer; raised so it isn't truncated\n"
-        if reasoning
-        else ""
-    )
-    sft_reasoning_note = (
-        "# reasoning is on (thinking = true): each gold `output` must contain a <think>...</think>\n"
-        "# block; validate locally with freesolo.datasets.warn_missing_think_tags before a real run.\n"
-        if reasoning
-        else ""
-    )
-    if not rl.exists():
-        rl.write_text(
-            'model = "Qwen/Qwen3.5-4B"\n'
-            'algorithm = "grpo"\n'
-            f"{thinking_line}"
-            "\n"
-            f"{env_comment}"
-            "[train]\n"
-            "epochs = 1\n"
-            "max_examples = 2  # rows to train on; the starter dataset has 2 (raise as your dataset grows)\n"
-            f"{rl_reasoning_train}"
-            "lora_rank = 32\n"
-            "# GPU and HF artifacts are managed automatically by the platform: the GPU is\n"
-            "# the cheapest fitting managed class, and artifacts live in a private environment-scoped repo.\n"
-        )
-    if not sft.exists():
-        sft.write_text(
-            'model = "Qwen/Qwen3.5-4B"\n'
-            'algorithm = "sft"\n'
-            f"{thinking_line}"
-            "\n"
-            f"{env_comment}"
-            "[train]\n"
-            "epochs = 1\n"
-            "max_examples = 2  # rows to train on; the starter dataset has 2 (raise as your dataset grows)\n"
-            "lora_rank = 32\n"
-            f"{sft_reasoning_note}"
-            "# GPU and HF artifacts are managed automatically by the platform: the GPU is\n"
-            "# the cheapest fitting managed class, and artifacts live in a private environment-scoped repo.\n"
-        )
-    opd = Path("configs/opd.toml")
-    if not opd.exists():
-        # opd (on-policy distillation) works in BOTH modes: single-turn distils one sampled completion
-        # per prompt; multi-turn rolls out each episode and distils every assistant turn against the
-        # transcript so far. The scaffold differs only by a one-line note pointing at the mode.
-        opd_multiturn_note = (
-            "# NOTE: opd rolls out each episode and distils EVERY assistant turn (conditioned on the\n"
-            "# transcript so far) against the Fireworks GLM teacher — the multi-turn distillation path.\n\n"
-            if multi_turn
-            else ""
-        )
-        opd.write_text(
-            f"{opd_multiturn_note}"
-            'model = "Qwen/Qwen3.5-4B"\n'
-            'algorithm = "opd"   # on-policy distillation from the managed GLM 5.2 teacher\n\n'
-            "# Environment: upload this project folder with\n"
-            "# `flash env push --name my-env .`, then paste the returned id below.\n"
-            "# The GLM 5.2 teacher and key are platform-managed — nothing to set up or export.\n"
-            "[environment]\n"
-            'id = ""\n\n'
-            "[train]\n"
-            "epochs = 1\n"
-            "max_examples = 2  # rows to train on; the starter dataset has 2 (raise as your dataset grows)\n"
-            "lora_rank = 32\n"
-            "# GPU and HF artifacts are managed automatically by the platform: the GPU is\n"
-            "# the cheapest fitting managed class, and artifacts live in a private environment-scoped repo.\n"
-        )
-    training = Path("TRAINING.md")
-    if not training.exists():
-        # Explicit UTF-8: TRAINING_MD has non-ASCII chars that raise UnicodeEncodeError under a non-UTF-8 locale.
-        training.write_text(TRAINING_MD, encoding="utf-8")
-    scaffolded = [
-        "environment.py",
-        "dataset/train.jsonl",
-        "configs/sft.toml",
-        "configs/rl.toml",
-        "configs/opd.toml",
-        "TRAINING.md",
-    ]
-    if render.styled():
-        print(render.env_setup(scaffolded))
-        return 0
-    print(f"ensured {', '.join(scaffolded)}")
-    return 0
-
-
 def cmd_models(args) -> int:
     rows = public_model_rows()
     if render.styled():
@@ -631,20 +232,29 @@ def cmd_train(args) -> int:
         return _cmd_train_cost(args)
     spec = spec_from_file(
         args.config,
-        run_id=new_run_id() if args.dry_run else None,
+        run_id=None,
         overrides=args.overrides,
         extra_configs=args.extra_configs,
     )
+    client = client_from_config()
     if args.dry_run:
-        payload = {"run_id": spec.run_id, "state": "dry_run", "spec": spec.to_dict()}
+        # Dry-run is a faithful server-side preview: the control plane runs the SAME config
+        # validation and warm-start/serving preflights it would at real submit (serving rank/context
+        # caps, the continued warm-start rank match, cost quote) and records a state=dry_run run, but
+        # allocates no GPU and charges nothing. Local runtime secrets aren't sent — the server
+        # relaxes the required-[environment].secrets check for a preview, so `--dry-run` works before
+        # prod secrets are wired up. A rejection surfaces as the server's `error: ...` (exit 1),
+        # exactly as a real submit would.
+        status = client.create_run(spec_payload(spec), dry_run=True)
         if render.styled():
             print(
-                render.object_panel("train", payload, "dry run — validated locally, not submitted")
+                render.object_panel(
+                    "train", status, "dry run — validated by the server, not submitted"
+                )
             )
         else:
-            print(json.dumps(payload, indent=2))
+            print(json.dumps(status, indent=2))
         return 0
-    client = client_from_config()
     status = client.create_run(
         spec_payload(spec),
         runtime_secrets=runtime_secrets_from_local_env(args.config, keys=spec.environment.secrets),
@@ -931,7 +541,6 @@ def cmd_deploy(args) -> int:
             print(
                 render.arrow(url_note) if render.styled() else f"note: {url_note}", file=sys.stderr
             )
-    if dep.get("state") != "dry_run":
         state = dep.get("state", "deploying")
         if state == "failed":
             detail = str(dep.get("error") or dep.get("detail") or "unknown error")