freesolo-flash-dev 0.2.58__tar.gz → 1.0.1__tar.gz

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 0.2.58
+Version: 1.0.1
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/catalog.py

@@ -111,6 +111,8 @@ class ModelInfo:
     # multimodal-nested config, so the curated values are what actually engage the gate.
     num_layers: int = 0
     hidden_size: int = 0
+    # vllm hybrid-mamba cache block size in tokens. 0 means the model has no catalogued mamba floor.
+    mamba_block_size: int = 0
 
     @property
     def is_moe(self) -> bool:
@@ -124,6 +126,8 @@ class ModelInfo:
 
     def to_dict(self) -> dict[str, Any]:
         data = asdict(self)
+        if not data["mamba_block_size"]:
+            del data["mamba_block_size"]
         serving = data["serving"]
         if serving is None:
             del data["serving"]
@@ -170,7 +174,7 @@ MODELS: dict[str, ModelInfo] = {
             serve_model_id=SERVING_FP8_MODEL_REPOS["openbmb/MiniCPM5-1B"],
             max_loras=16,
             max_lora_rank=128,
-            max_model_len=8192,
+            max_model_len=32768,
         ),
         thinking="hybrid",
         notes="On-device class SLM (131k ctx); standard Llama architecture.",
@@ -189,7 +193,7 @@ MODELS: dict[str, ModelInfo] = {
             serve_model_id=SERVING_FP8_MODEL_REPOS["Qwen/Qwen3.5-0.8B"],
             max_loras=16,
             max_lora_rank=128,
-            max_model_len=8192,
+            max_model_len=32768,
         ),
         thinking="hybrid",
         notes="Smallest Qwen3.5; cheap smoke/dev runs with the modern arch.",
@@ -208,7 +212,7 @@ MODELS: dict[str, ModelInfo] = {
             serve_model_id=SERVING_FP8_MODEL_REPOS["Qwen/Qwen3.5-2B"],
             max_loras=16,
             max_lora_rank=128,
-            max_model_len=8192,
+            max_model_len=32768,
         ),
         thinking="hybrid",
     ),
@@ -226,7 +230,7 @@ MODELS: dict[str, ModelInfo] = {
             serve_model_id=SERVING_FP8_MODEL_REPOS["Qwen/Qwen3.5-4B"],
             max_loras=16,
             max_lora_rank=64,
-            max_model_len=8192,
+            max_model_len=32768,
             max_num_seqs=8,
             gpu_memory_utilization=0.98,
         ),
@@ -247,11 +251,11 @@ MODELS: dict[str, ModelInfo] = {
         quant="bf16",
         recommended_gpu="A100 PCIe",
         serving=ServingCapacity(
-            gpu="L4",
+            gpu="H100",
             serve_model_id=SERVING_FP8_MODEL_REPOS["Qwen/Qwen3.5-9B"],
             max_loras=16,
             max_lora_rank=64,
-            max_model_len=8192,
+            max_model_len=32768,
             max_num_seqs=8,
             gpu_memory_utilization=0.98,
         ),
@@ -271,6 +275,10 @@ MODELS: dict[str, ModelInfo] = {
         # layers x 2048 hidden (hybrid GatedDeltaNet + full-attention, 256 experts / 8 active).
         num_layers=40,
         hidden_size=2048,
+        # vllm 0.19.1 model_executor/models/config.py derives this from the qwen config via
+        # mamba_utils.py: the 1,097,728-byte gdn state page needs 1072 fp8 attention tokens
+        # after the 16-token backend alignment applied by hybridattentionmambamodelconfig.
+        mamba_block_size=1072,
         vocab_size=248_320,
         algos=ALGORITHMS,
         min_vram_gb=141,
@@ -312,6 +320,17 @@ MODELS: dict[str, ModelInfo] = {
 }
 
 
+def opd_mamba_batched_token_floor(
+    model_id: str, seq_cap: int, max_num_seqs: int | None
+) -> int | None:
+    """return a mamba floor only when vllm's derived opd scheduler budget is too small."""
+    info = MODELS.get(model_id)
+    if info is None or info.mamba_block_size <= 0 or max_num_seqs is None:
+        return None
+    derived_budget = int(max_num_seqs) * int(seq_cap)
+    return info.mamba_block_size if derived_budget < info.mamba_block_size else None
+
+
 def list_models() -> list[ModelInfo]:
     return sorted(MODELS.values(), key=lambda m: (m.min_vram_gb, m.id))
 

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/cli/commands.py

@@ -214,6 +214,7 @@ def _cmd_train_cost(args) -> int:
     Catalog-only and deterministic. SFT cost never imports the environment; it requires a positive
     [train].max_examples row count instead of guessing or locally counting a dataset."""
     from flash.cost import estimate_cost
+    from flash.lora_rank import preflight_train_context_within_serving
 
     spec = spec_from_file(
         args.config,
@@ -221,6 +222,7 @@ def _cmd_train_cost(args) -> int:
         overrides=args.overrides,
         extra_configs=args.extra_configs,
     )
+    preflight_train_context_within_serving(spec)
     if spec.train.init_from_adapter:
         # --cost is offline/catalog-only and cannot read the source adapter, so the rank stays at the
         # local default. Warm starts train and are priced at the SOURCE adapter's authoritative rank
@@ -302,23 +304,24 @@ def cmd_train(args) -> int:
     )
     payload = spec_payload(spec, authored_train_keys=authored_train_keys)
     client = client_from_config()
+    client_train_schema = {
+        "version": __version__,
+        "fields": train_schema_metadata(),
+        "authored_keys": sorted(authored_train_keys),
+    }
+    runtime_secrets = (
+        runtime_secrets_from_local_env(args.config, keys=spec.environment.secrets) or None
+    )
     if args.dry_run:
-        # Dry-run is a faithful server-side preview: the control plane runs the SAME config
-        # validation and warm-start/serving preflights it would at real submit (serving rank/context
-        # caps, the continued warm-start rank match, cost quote) and records a state=dry_run run, but
-        # allocates no GPU and charges nothing. Local runtime secrets aren't sent — the server
-        # relaxes the required-[environment].secrets check for a preview, so `--dry-run` works before
-        # prod secrets are wired up. A rejection surfaces as the server's `error: ...` (exit 1),
-        # exactly as a real submit would.
+        # dry-run is a faithful server-side preview: it sends the same declared secrets and runs the
+        # same config, warm-start, serving, and cost preflights as a real submit, but allocates no gpu
+        # and charges nothing. a rejection surfaces as the server's error with exit status 1.
         try:
             status = client.create_run(
                 payload,
+                runtime_secrets=runtime_secrets,
                 dry_run=True,
-                client_train_schema={
-                    "version": __version__,
-                    "fields": train_schema_metadata(),
-                    "authored_keys": sorted(authored_train_keys),
-                },
+                client_train_schema=client_train_schema,
             )
         except ApiError as exc:
             detail = _legacy_train_key_rejection_detail(exc, authored_train_keys)
@@ -338,7 +341,8 @@ def cmd_train(args) -> int:
         return 0
     status = client.create_run(
         payload,
-        runtime_secrets=runtime_secrets_from_local_env(args.config, keys=spec.environment.secrets),
+        runtime_secrets=runtime_secrets,
+        client_train_schema=client_train_schema,
     )
     run_id = status["run_id"]
     logger.info(

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/client/http.py

@@ -343,8 +343,8 @@ class ApiClient:
             # Server-side preview: runs the same validation/preflights as a real submit and records a
             # state=dry_run run, but allocates no GPU and charges nothing. Returns that status.
             body["dry_run"] = True
-            if client_train_schema is not None:
-                body["client_train_schema"] = client_train_schema
+        if client_train_schema is not None:
+            body["client_train_schema"] = client_train_schema
         return self._request("POST", "/v1/runs", body=body)
 
     def list_runs(self) -> list[dict]:

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/cost/analytical.py

@@ -40,6 +40,22 @@ MFU_TRAIN = 0.35  # GRPO policy/reference update
 MFU_SFT_TRAIN = 0.25  # SFT fwd/bwd (smaller effective batch, long sequences)
 MFU_DECODE = 0.12  # batched vLLM rollout (decode is memory-bandwidth-bound)
 
+# --- MoE (mixture-of-experts) per-step correction ----------------------------------------------
+# For an MoE model (active params << total, e.g. Qwen3.6-35B-A3B: ~3B active / 35B total) the wall
+# time per step is NOT the tiny active-param FLOPs the dense model predicts. Routing, all-expert
+# coordination, and grouped-GEMM under-utilization at small batch make the step scale with TOTAL
+# params. Pricing 35B-A3B on active params under-quoted real RunPod runs by 13-27x (SFT ~1.2s vs
+# ~42s realized/step; GRPO ~1.4s vs ~24s). So an MoE prices on TOTAL params at a reduced effective
+# MFU + a per-step overhead + a one-time compile. DENSE models (active == total) are unaffected --
+# they keep the original active-param path and the MFUs above.
+MFU_SFT_TRAIN_MOE = 0.10  # MoE SFT fwd/bwd priced on total params
+MFU_TRAIN_MOE = 0.09  # MoE GRPO/OPD policy update priced on total params
+MOE_STEP_OVERHEAD_S = 2.0  # routing/dispatch/kernel-launch overhead an MoE pays every step
+# One-time kernel/graph compile amortized into the first training step (fused Triton kernels;
+# + vLLM cudagraph capture for rollout methods). MoE-only; the prior model omitted it entirely.
+COMPILE_MOE_SFT_S = 35.0
+COMPILE_MOE_ROLLOUT_S = 48.0  # GRPO / OPD (adds vLLM cudagraph capture)
+
 # Reward grading is CONCURRENT: a step's completions score in parallel slots, so the reward
 # wall is ceil(completions / slots) waves x latency, not completions x latency.
 REWARD_CONCURRENCY = 16.0
@@ -116,13 +132,32 @@ def required_save_overhead_seconds(config: RunConfig) -> float:
     return len(n.save_at_steps) * per_save
 
 
+def _is_moe(model_id: str) -> bool:
+    """True when the model routes each token through a subset of experts (active < total params)."""
+    return active_params_b(model_id) < total_params_b(model_id)
+
+
+def compile_seconds(config: RunConfig, gpu: str) -> float:
+    """One-time kernel/graph compile folded into the first training step. MoE-only; 0 for dense
+    (whose original timing did not model it). Larger for rollout methods (add vLLM cudagraph
+    capture)."""
+    _ = gpu
+    if not _is_moe(config.model_id):
+        return 0.0
+    return COMPILE_MOE_SFT_S if config.method == "sft" else COMPILE_MOE_ROLLOUT_S
+
+
 def seconds_per_step(config: RunConfig, gpu: str) -> float:
     """Steady-state wall time for one optimizer step on ``gpu``."""
     n = config.normalized()
-    # Per-token FLOPs scale with the ACTIVE params (an MoE token routes through only a subset of
-    # experts); for a dense model this equals the total. Memory/size terms below keep total_params_b.
-    params = active_params_b(n.model_id) * 1e9
     peak = gpu_tflops(gpu) * 1e12  # FLOP/s
+    # An MoE's per-step wall scales with TOTAL params (routing + all-expert coordination + grouped
+    # GEMM under-utilization), not the tiny active-param FLOPs; dense models keep active (== total).
+    moe = _is_moe(n.model_id)
+    params = (total_params_b(n.model_id) if moe else active_params_b(n.model_id)) * 1e9
+    overhead = MOE_STEP_OVERHEAD_S if moe else 0.0
+    sft_mfu = MFU_SFT_TRAIN_MOE if moe else MFU_SFT_TRAIN
+    update_mfu = MFU_TRAIN_MOE if moe else MFU_TRAIN
 
     if n.is_opd:
         # OPD step = on-policy student rollout (like GRPO) + remote teacher scoring (CONCURRENT
@@ -131,7 +166,7 @@ def seconds_per_step(config: RunConfig, gpu: str) -> float:
         # sequence (see _opd_step_shape), not completion-only, or long-prompt opd is underquoted.
         completions, seq_tokens = _opd_step_shape(n)
         gen_s = (GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * MFU_DECODE)
-        update_s = (OPD_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * MFU_TRAIN)
+        update_s = (OPD_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * update_mfu)
         teacher_lat = teacher_seconds_per_completion()
         # run_opd's primary path scores a step's completions CONCURRENTLY over Fireworks with a fan-out
         # cap of the step's OWN completion count (prompts_per_step * group_size, opd.py Phase 2), so
@@ -139,30 +174,33 @@ def seconds_per_step(config: RunConfig, gpu: str) -> float:
         # latency, NOT the full serial sum (that describes only the CPU-test fallback that can't
         # batch-generate). The teacher endpoint's rate limit is the real ceiling on this fan-out.
         teacher_s = teacher_lat
-        return gen_s + teacher_s + update_s
+        return overhead + gen_s + teacher_s + update_s
 
     if not n.is_grpo:
         flops = SFT_FLOPS_PER_TOKEN_PER_PARAM * params * (n.batch_size * n.seq_len)
-        return flops / (peak * MFU_SFT_TRAIN)
+        return overhead + flops / (peak * sft_mfu)
 
     # GRPO step = rollout (G completions/prompt) + concurrent reward grading + policy/ref update.
     completions = n.batch_size * n.group_size
     gen_tokens = completions * n.completion_len
     gen_s = (GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_DECODE)
-    update_s = (GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_TRAIN)
+    update_s = (GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * update_mfu)
     latency = reward_seconds_per_completion(n.reward_seconds_per_completion)
     reward_s = (
         math.ceil(completions / REWARD_CONCURRENCY) * latency
     )  # ceil: a partial wave still costs one latency
-    return gen_s + reward_s + update_s
+    return overhead + gen_s + reward_s + update_s
 
 def sft_seconds_for_tokens(config: RunConfig, gpu: str, train_tokens: float) -> float:
     """SFT steady-state wall time for an actual token count on ``gpu``."""
     n = config.normalized()
-    params = active_params_b(n.model_id) * 1e9
+    # MoE prices on total params at a reduced MFU (see seconds_per_step); dense keeps active.
+    moe = _is_moe(n.model_id)
+    params = (total_params_b(n.model_id) if moe else active_params_b(n.model_id)) * 1e9
+    mfu = MFU_SFT_TRAIN_MOE if moe else MFU_SFT_TRAIN
     peak = gpu_tflops(gpu) * 1e12
     flops = SFT_FLOPS_PER_TOKEN_PER_PARAM * params * train_tokens
-    return flops / (peak * MFU_SFT_TRAIN)
+    return flops / (peak * mfu)
 
 
 def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str, int]:
@@ -221,6 +259,11 @@ def _notes(
             f"{len(n.save_at_steps)} synchronous required save(s) add "
             f"~{_fmt_duration(required_save_s)}"
         )
+    if _is_moe(n.model_id):
+        notes.append(
+            "MoE model: per-step time priced on total params (routing + all-expert coordination), "
+            "not just active-param FLOPs, plus a one-time kernel compile"
+        )
     notes.append(f"GPU sized with {vram_headroom() - 1:.0%} VRAM headroom; static GPU $/hr")
     if wall_capped:
         notes.append(
@@ -283,9 +326,12 @@ def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S)
     setup = setup_seconds(config)
     sps = seconds_per_step(config, gpu)
     required_save_s = required_save_overhead_seconds(config)
-    raw_train = config.steps * sps + required_save_s
+    # A one-time kernel/graph compile is paid once on the first step (MoE-only; 0 for dense). It is
+    # training GPU time, so it belongs in the (billed) train term, not setup.
+    compile_s = compile_seconds(config, gpu)
+    raw_train = compile_s + config.steps * sps + required_save_s
     if not config.is_grpo and config.train_tokens is not None:
-        raw_train = sft_seconds_for_tokens(config, gpu, config.train_tokens) + required_save_s
+        raw_train = compile_s + sft_seconds_for_tokens(config, gpu, config.train_tokens) + required_save_s
     sps = raw_train / config.steps
 
     # The cap is on total elapsed wall; setup is reported but not billed, so only training

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/engine/worker/opd_vllm.py

@@ -127,7 +127,8 @@ def opd_vllm_kwargs(model_id: str, knobs: Any, seq_cap: int) -> dict[str, Any]:
 
     fp8_kv = bool(cc >= (8, 9))
     kwargs["kv_cache_dtype"] = "fp8" if fp8_kv else None
-    kwargs["max_num_batched_tokens"] = max(8192, int(seq_cap)) if card_gb >= 140 else None
+    if card_gb >= 140:
+        kwargs["max_num_batched_tokens"] = max(8192, int(seq_cap))
 
     if card_gb > 0:
         try:
@@ -189,6 +190,12 @@ def opd_vllm_kwargs(model_id: str, knobs: Any, seq_cap: int) -> dict[str, Any]:
             kwargs["rollout_batch_size"] = rollout_concurrency
         except Exception as exc:
             print(f"[opd] vLLM memory-util sizing failed; using 0.10: {exc}")
+    if card_gb < 140:
+        from flash.catalog import opd_mamba_batched_token_floor
+
+        kwargs["max_num_batched_tokens"] = opd_mamba_batched_token_floor(
+            model_id, seq_cap, kwargs["max_num_seqs"]
+        )
     if startup_oom is not None:
         raise startup_oom
 

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/envs/loader.py

@@ -36,7 +36,7 @@ from flash.envs.archive_policy import (
 _DEFAULT_GITHUB_REF = "main"
 _DEFAULT_ENVIRONMENT_PATH = "environment.py"
 _DEFAULT_MANAGED_ENV_REPO = "freesolo-co/environment-hub"
-_CACHE_ROOT = Path(os.environ.get("FLASH_ENV_CACHE_DIR", "/tmp/flash-env-cache"))
+_CACHE_ROOT = Path("/tmp/flash-env-cache")
 # bound the on-disk env cache so it cannot grow without limit (one subdir per env
 # content-sha, ~30-80 MB each). evicted LRU by dir mtime, which we bump on cache hit.
 _CACHE_MAX_ENTRIES = 32

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/runner/__init__.py

@@ -691,7 +691,7 @@ def _prepare_init_from_adapter(
     owner_key_id: int | None = None,
     token: str | None = None,
 ) -> tuple[JobSpec, JobSpec, dict | None]:
-    """Preserve the public request while applying source metadata only to the worker spec."""
+    """prepare public and worker specs with source-authoritative adapter metadata."""
     _require_supported_adapter_continuation(spec)
     ref = spec.train.init_from_adapter
     if not ref:
@@ -776,7 +776,7 @@ def _prepare_init_from_adapter(
     )
     assert metadata is not None
     identity = adapter_artifact_identity(storage, config, token, revision).to_dict()
-    public_spec = spec
+    public_spec = replace(spec, train=replace(spec.train, lora_alpha=metadata.alpha))
     worker_spec = replace(
         worker_spec,
         train=replace(worker_spec.train, lora_rank=metadata.rank, lora_alpha=metadata.alpha),

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/schema/__init__.py

@@ -546,7 +546,12 @@ def _validate_grpo(spec: JobSpec) -> None:
 
 
 def _validate_opd(spec: JobSpec) -> None:
-    """validate that the opd context budget leaves room for a prompt."""
+    """validate opd-specific training constraints."""
+    if spec.train.kl_penalty_coef == 0.0:
+        raise ConfigError(
+            "[train] kl_penalty_coef must be > 0 for opd because it scales the gkd "
+            "distillation objective; omit it to use the default 1.0 or set a positive value"
+        )
     if spec.train.max_context_tokens:
         # Mirror run_opd's prompt-budget guard at PARSE time: a context budget that leaves no room
         # for any prompt after the completion budget is rejected here, BEFORE a paid worker is
@@ -564,6 +569,9 @@ def _validate_opd(spec: JobSpec) -> None:
                 f"invalid budget fails before a GPU worker is provisioned."
             )
 
+        # short hybrid-mamba contexts are safe because the worker compares vllm's derived scheduler
+        # budget with the catalogued block size and supplies an explicit floor only when required.
+
 
 # Each algorithm's spec-level contract lives in ONE validator, dispatched by name so a new algorithm
 # adds a function + a map entry rather than another scattered ``if spec.algorithm == ...`` block.

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.1}/flash/serve/deploy.py

@@ -26,6 +26,7 @@ READBACK_DELAY_SECONDS = 2.0
 REVISION_READY_BUDGET_SECONDS = 5 * 60.0
 ACTIVATION_READBACK_ATTEMPTS = 3
 THINKING_STRUCTURED_OUTPUTS_CAPABILITY = "thinking_structured_outputs_deferred_v1"
+REVISION_PROVENANCE_CAPABILITY = "revision_provenance"
 _RETRYABLE_SMOKE_503_CODES = frozenset({"adapter_loading", "engine_unavailable"})
 
 
@@ -382,7 +383,10 @@ def deploy_adapter(
     dep.adapter_revision = revision
     checkpoint = f"{run_id}/step-{checkpoint_step}" if checkpoint_step is not None else run_id
     so_default = parse_structured_outputs(structured_outputs) if structured_outputs else None
-    _require_serving_capabilities(thinking_structured_outputs=thinking and so_default is not None)
+    advertised = _require_serving_capabilities(
+        thinking_structured_outputs=thinking and so_default is not None
+    )
+    require_provenance = REVISION_PROVENANCE_CAPABILITY in advertised
 
     body = {
         "adapter_id": revision,
@@ -391,7 +395,9 @@ def deploy_adapter(
         "subfolder": subfolder,
         "repo_type": "dataset",
         "checkpoint": checkpoint,
-        "status": "ready",
+        # NOTE: do NOT send a client-chosen "status" -- the serving backend sets the record status
+        # itself on registration and rejects an incoming "status" as an extra field (HTTP 422
+        # extra_forbidden). Sending it blocked every deploy against the deployed serving backend.
         "metadata": {
             "record_type": "revision",
             "run_id": run_id,
@@ -419,14 +425,18 @@ def deploy_adapter(
             record = _registered_adapter(revision)
         except ServingError as read_exc:
             raise exc from read_exc
-        if record is None or not _matches_revision_identity(record, body):
+        if record is None or not _matches_revision_identity(
+            record, body, require_provenance=require_provenance
+        ):
             raise exc
         logger.warning(
             "adapter registration response was ambiguous; revision %s exists with matching identity",
             revision,
         )
 
-    _wait_revision_ready(revision, subfolder, expected_identity=body)
+    _wait_revision_ready(
+        revision, subfolder, expected_identity=body, require_provenance=require_provenance
+    )
     if before_activate is not None:
         before_activate(revision, checkpoint)
     activation = _activate_revision(
@@ -469,9 +479,9 @@ def _registered_adapter(adapter_id: str, *, timeout_s: float | None = None) -> d
     return record
 
 
-def _matches_revision_identity(record: dict, expected: dict) -> bool:
-    metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
-    expected_metadata = expected["metadata"]
+def _matches_revision_identity(
+    record: dict, expected: dict, *, require_provenance: bool = True
+) -> bool:
     scalar_fields = (
         "adapter_id",
         "repo_id",
@@ -487,19 +497,35 @@ def _matches_revision_identity(record: dict, expected: dict) -> bool:
         return False
     if (record.get("structured_outputs") or None) != (expected.get("structured_outputs") or None):
         return False
+    if not require_provenance:
+        # backends without revision_provenance do not echo provenance metadata; the immutable
+        # adapter_id already pins the artifact, so this cross-check is best effort here.
+        return True
+    metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
+    expected_metadata = expected["metadata"]
     return all(
         metadata.get(field) == expected_metadata.get(field)
         for field in ("record_type", "run_id", "checkpoint_step", "hf_revision")
     )
 
 
-def _require_serving_capabilities(*, thinking_structured_outputs: bool = False) -> None:
+def _require_serving_capabilities(*, thinking_structured_outputs: bool = False) -> set[str]:
+    # SAFETY-CRITICAL capabilities the deploy correctness contract genuinely depends on: immutable
+    # revisions (a revision id always maps to one artifact) and an atomic alias compare-and-swap
+    # (the alias flip can't race). These are hard-required.
     required = {
         "immutable_adapter_revisions",
         "alias_compare_and_swap",
-        "revision_provenance",
     }
+    # PREFERRED (not required): `revision_provenance` only lets the serving backend echo back the
+    # run/checkpoint/hf_revision metadata, which is used ONLY on the rare 5xx-during-registration
+    # recovery path (`_matches_revision_identity`). Hard-requiring it blocked EVERY deploy org-wide
+    # whenever the serving build lagged on advertising it, even though the happy path never uses it.
+    # So its absence is a logged warning, not a deploy-blocking error.
+    preferred = {REVISION_PROVENANCE_CAPABILITY}
     if thinking_structured_outputs:
+        # Genuinely required for this run: thinking + structured outputs needs the serving backend's
+        # deferred-constraint support (grammar applied after </think>) or served output is invalid.
         required.add(THINKING_STRUCTURED_OUTPUTS_CAPABILITY)
     url = f"{serving_base_url()}/healthz"
     response = _serving_request("GET", url)
@@ -524,12 +550,24 @@ def _require_serving_capabilities(*, thinking_structured_outputs: bool = False)
             f"serving_contract_unsupported: serving health check at {url} capabilities must be "
             "strings"
         )
-    missing = sorted(required - set(capabilities))
+    advertised = set(capabilities)
+    missing = sorted(required - advertised)
     if missing:
         raise ServingError(
             "serving_contract_unsupported: serving is missing required capabilities "
             + ", ".join(missing)
         )
+    missing_preferred = sorted(preferred - advertised)
+    if missing_preferred:
+        # Not fatal: the happy-path deploy does not use these; only the ambiguous-registration
+        # recovery path degrades (best-effort identity check). Surface it so an operator can ship
+        # the serving build that advertises them, without blocking customers meanwhile.
+        logger.warning(
+            "serving backend does not advertise preferred capabilities %s; deploying anyway "
+            "(ambiguous-registration recovery will be best-effort)",
+            ", ".join(missing_preferred),
+        )
+    return advertised
 
 
 def _wait_revision_ready(
@@ -537,6 +575,7 @@ def _wait_revision_ready(
     subfolder: str,
     *,
     expected_identity: dict | None = None,
+    require_provenance: bool = True,
     budget_s: float = REVISION_READY_BUDGET_SECONDS,
 ) -> dict:
     deadline = time.monotonic() + max(0.0, float(budget_s))
@@ -566,7 +605,7 @@ def _wait_revision_ready(
             continue
         last_read_error = None
         if expected_identity is not None and not _matches_revision_identity(
-            record, expected_identity
+            record, expected_identity, require_provenance=require_provenance
         ):
             raise ServingError(
                 f"adapter revision {revision} resolved to a different immutable identity"