freesolo-flash-dev 1.2.129__tar.gz → 1.2.130__tar.gz

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.129
+Version: 1.2.130
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/core/catalog.py

@@ -258,7 +258,7 @@ MODELS: dict[str, ModelInfo] = {
         quant="bf16",
         recommended_gpu="A100 PCIe",
         serving=ServingCapacity(
-            gpu="L40S",
+            gpu="B200",
             serve_model_id=SERVING_MODEL_REPOS["Qwen/Qwen3.5-9B"],
             max_loras=16,
             max_cpu_loras=16,
@@ -318,7 +318,7 @@ MODELS: dict[str, ModelInfo] = {
         recommended_gpu="A100 PCIe",
         min_disk_gb=160,
         serving=ServingCapacity(
-            gpu="H100",
+            gpu="B200",
             serve_model_id=SERVING_MODEL_REPOS["Qwen/Qwen3.8-27B"],
             max_loras=16,
             max_cpu_loras=16,
@@ -332,7 +332,7 @@ MODELS: dict[str, ModelInfo] = {
         thinking="hybrid",
         notes="Dense 27.781427952B multimodal VL checkpoint with image-capable bf16 LoRA training. "
         "SFT fits the 80GB A100 (~55.6GB weights); colocated GRPO needs the B200 (trainer + vLLM "
-        "rollout = two copies). Hosted serving remains pending the exact official FP8 H100 canary.",
+        "rollout = two copies). Hosted serving runs the official FP8 checkpoint on the B200.",
     ),
     "Qwen/Qwen3.6-35B-A3B": ModelInfo(
         id="Qwen/Qwen3.6-35B-A3B",
@@ -393,10 +393,10 @@ MODELS: dict[str, ModelInfo] = {
         quant="bf16",
         recommended_gpu="H200",
         serving=ServingCapacity(
-            gpu="H200",
+            gpu="B200",
             serve_model_id=SERVING_MODEL_REPOS["Qwen/Qwen3.6-35B-A3B"],
-            # bf16 on h200 is the validated full-expert lora path. six hot rank-64 adapters plus 32k
-            # fit with cuda graphs and a 679,701-token kv cache; eight hot adapters overflow the card.
+            # bf16 is the validated full-expert lora path. six hot rank-64 adapters plus 32k fit
+            # with cuda graphs; the b200 measured 49.38 GiB of kv (3,678,952 tokens) for this shape.
             max_loras=6,
             max_cpu_loras=6,
             max_lora_rank=64,

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/serve/deployment/profiles.py

@@ -385,9 +385,15 @@ def _require_catalog_agreement(profile: ServingProfile) -> None:
                 f"{profile.model_id} serving profile {name}={actual!r} disagrees with the "
                 f"catalog capacity {expected!r}"
             )
+    # NB: modal_gpu is deliberately NOT checked against serving.gpu. They describe different
+    # planes: serving.gpu is the card the freesolo-owned hosted plane runs on, while modal_gpu is
+    # the card THIS customer-owned profile was live-qualified on (modal_live_qualified). The hosted
+    # plane moved to B200; the customer profiles stay on their qualified cards until a customer-owned
+    # B200 qualification runs. Engine SHAPE still has to agree -- see _CATALOG_CHECKED_FIELDS, which
+    # is what stops a profile serving a longer context or higher lora rank than the catalog
+    # advertises. The card is placement, not shape.
     for label, expected, actual in (
         ("served_model", serving.serve_model_id, profile.served_model),
-        ("modal gpu", serving.gpu, profile.modal_gpu),
     ):
         if expected != actual:
             raise ProfileError(

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/serving/app/modal_app.py

@@ -93,10 +93,11 @@ HOSTING_CACHE_MOUNT = "/vol/hosting-cache"
 # below — HF/internal key, backend + Supabase URLs — stays env-configured because it is per-deploy
 # credentials, not tuning.)
 TIMEOUT_SECONDS = 600
-# 2700s (45 min): the 35B bf16/H200 cold boot takes ~1010s of engine init alone (67 GiB load +
-# ~377s torch.compile + graph capture + warmup) on top of image pull, which blows past the old 1200s
-# ceiling — Modal SIGTERMs the container mid-warmup and it cold-cycles forever. A higher ceiling is
-# harmless for the fast tiers (they still boot in a minute; this is only a kill-if-stuck bound).
+# 2700s (45 min): the slowest MEASURED cold boot on the shipped B200 tiers is the 27B at 1821s
+# (a cold torch.compile cache alone costs ~354s of Dynamo), on top of image pull. The old 1200s
+# ceiling was already blown by the 35B's ~1010s of engine init — Modal SIGTERMs the container
+# mid-warmup and it cold-cycles forever. 2700s leaves ~880s of margin over the worst measurement,
+# and a higher ceiling is harmless for the fast tiers: this is only a kill-if-stuck bound.
 STARTUP_TIMEOUT_SECONDS = 2700
 # The router awaits the engine call, which on a cold base model includes vLLM startup, so its
 # timeout must cover startup + a request (else it's killed mid cold-start).
@@ -126,10 +127,14 @@ SCALEDOWN_WINDOW_SECONDS_BY_GPU: dict[str, int] = {
     # ~1010s cold boot (35B bf16, 67 GiB + ~377s torch.compile). Break-even AND a ~17-min
     # user-visible stall on a miss both argue for keeping the full window here.
     "H200": 1800,
-    # Blackwell. No cold-boot canary has run on either card yet, so both hold the H200's window as a
-    # placeholder: it is the longest we ship, and a too-long hold overpays for idle rather than
-    # cold-cycling a user request. Replace each with its measured boot time when the canary runs.
-    "B200": 1800,
+    # B200 now carries all three tiers. Measured cold boots (2026-08-30/31, this card): 9B 723s,
+    # 35B 488s, 27B 1821s. The 27B dominates because a cold torch.compile cache costs ~354s of
+    # Dynamo alone; warm-cache boots of the same tiers run 153-473s. Sized to the slowest MEASURED
+    # cold boot rather than the warm case, so a scale-out replacement is not cycled mid-boot.
+    "B200": 2100,
+    # B300 has no shipped tier. It also wedges intermittently serving above max_num_seqs (sm 10.3;
+    # B200 served 20/20 on the same workload), so it is not a deployment target yet -- the window
+    # stays a placeholder rather than a measurement.
     "B300": 1800,
 }
 # The tiers this app is allowed to run an engine on. `gpu` in the serving catalog is a plain string,
@@ -275,7 +280,7 @@ image = (
             # artifacts to VLLM_CACHE_ROOT/torch_compile_cache/<hash>/ (vllm/compilation/backends.py),
             # plus modelinfos/ and the GPU p2p cache. With MIN_CONTAINERS = 0 and a 30-minute
             # scaledown window that cost is paid on every cold start — it is the torch.compile +
-            # graph-capture portion of the ~1010s 35B engine init measured for STARTUP_TIMEOUT_SECONDS.
+            # graph-capture portion of the cold boots measured for STARTUP_TIMEOUT_SECONDS.
             # vLLM's own startup benchmark DEFINES a cold start this way (benchmarks/startup.py
             # cold_startup() points VLLM_CACHE_ROOT at a fresh mkdtemp), so leaving it unset on
             # ephemeral storage reproduces that worst case involuntarily.

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/serving/src/accounting/usage.py

@@ -41,9 +41,16 @@ _USD_PER_MTOK_DIVISOR = Decimal("1000000")
 @dataclass(frozen=True)
 class AuthorizedTraffic:
     principal: ServingTrafficPrincipal
-    openrouter_request_id: str | None = None
-    openrouter_generation_id: str | None = None
-    upstream_id: str | None = None
+
+
+@dataclass(frozen=True)
+class TrustedInternalAuthorization:
+    """A trusted server-to-server caller that has not yet been attributed to an organization."""
+
+    org_id: str | None = None
+
+
+InferenceAuthorization = AuthorizedTraffic | TrustedInternalAuthorization
 
 
 @dataclass(frozen=True)
@@ -104,7 +111,6 @@ def new_request_identity(
     request: Request,
     *,
     openai_completion_id: str | None = None,
-    traffic: AuthorizedTraffic | None = None,
 ) -> RequestIdentity:
     request_id = new_generation_id()
     supplied_correlation = _bounded_header(request, "X-Correlation-ID")
@@ -113,9 +119,6 @@ def new_request_identity(
         request_id=request_id,
         correlation_id=correlation_id,
         openai_completion_id=openai_completion_id,
-        openrouter_request_id=traffic.openrouter_request_id if traffic else None,
-        openrouter_generation_id=traffic.openrouter_generation_id if traffic else None,
-        upstream_id=traffic.upstream_id if traffic else None,
     )
 
 
@@ -131,29 +134,13 @@ def build_usage_session(
     serving_release: str,
     captured_at: datetime,
 ) -> UsageSession:
-    public_model_id = (
-        principal.publicModelId if principal.kind == "openrouter" else requested.adapter_id
-    )
     immutable_target = ImmutableTarget(
-        public_model_id=public_model_id,
+        public_model_id=requested.adapter_id,
         base_model=target.base_model,
         checkpoint_id=target.adapter_id if target.is_checkpoint else None,
         artifact_fingerprint=target.artifact_fingerprint,
     )
-    price = (
-        CapturedPrice(
-            source="openrouter_admission",
-            version=principal.providerCatalogDigest,
-            snapshot=principal.acceptedPriceSnapshot.model_dump(mode="json"),
-        )
-        if principal.kind == "openrouter"
-        else freesolo_price(target.base_model)
-    )
-    if principal.kind == "trusted_internal" and target.org_id is not None:
-        principal = TrustedInternalTrafficPrincipal(
-            orgId=target.org_id,
-            billingAttributionExplicit=True,
-        )
+    price = freesolo_price(target.base_model)
     return UsageSession(
         store=store,
         identity=identity,
@@ -191,11 +178,8 @@ def principal_for_external_org(org_id: str) -> FreesoloOrgTrafficPrincipal:
     return FreesoloOrgTrafficPrincipal(orgId=org_id)
 
 
-def principal_for_trusted_internal(org_id: str | None = None) -> TrustedInternalTrafficPrincipal:
-    return TrustedInternalTrafficPrincipal(
-        orgId=org_id,
-        billingAttributionExplicit=org_id is not None,
-    )
+def principal_for_trusted_internal(org_id: str) -> TrustedInternalTrafficPrincipal:
+    return TrustedInternalTrafficPrincipal(orgId=org_id)
 
 
 def captured_now() -> datetime:

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/serving/src/accounting/usage_outbox.py

@@ -10,17 +10,16 @@ import uuid
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import UTC, date, datetime, timedelta
-from decimal import Decimal
 from typing import Annotated, Any, Literal, Protocol
 
 import httpx
-from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints
 
+from flash.serving.src.accounting.usage_retry import is_transient_rpc_code
 from flash.serving.src.store.settings import Settings
 from flash.serving.src.store.supabase_rest import supabase_headers
 
 StableId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=512)]
-DecimalString = Annotated[str, StringConstraints(pattern=r"^(0|[1-9][0-9]*)(\.[0-9]+)?$")]
 _GENERATION_ID_RE = re.compile(r"^fsgen-[0-9a-f]{32}$")
 _HEARTBEAT_BATCH_SIZE = 128
 _RECOVERY_BATCH_SIZE = 500
@@ -41,24 +40,6 @@ _RESPONSE_LOSS_SAFE_RPCS = frozenset(
 )
 
 
-class AcceptedPriceSnapshot(BaseModel):
-    model_config = ConfigDict(frozen=True, extra="forbid")
-
-    promptTokenUsd: DecimalString
-    cachedPromptTokenUsd: DecimalString | None = None
-    completionTokenUsd: DecimalString
-    requestUsd: DecimalString | None = None
-
-
-class OpenRouterTrafficPrincipal(BaseModel):
-    model_config = ConfigDict(frozen=True, extra="forbid")
-
-    kind: Literal["openrouter"] = "openrouter"
-    publicModelId: StableId
-    providerCatalogDigest: StableId
-    acceptedPriceSnapshot: AcceptedPriceSnapshot
-
-
 class FreesoloOrgTrafficPrincipal(BaseModel):
     model_config = ConfigDict(frozen=True, extra="forbid")
 
@@ -70,18 +51,12 @@ class TrustedInternalTrafficPrincipal(BaseModel):
     model_config = ConfigDict(frozen=True, extra="forbid")
 
     kind: Literal["trusted_internal"] = "trusted_internal"
-    orgId: StableId | None = None
-    billingAttributionExplicit: bool = False
-
-    @model_validator(mode="after")
-    def validate_attribution(self) -> TrustedInternalTrafficPrincipal:
-        if self.orgId is not None and not self.billingAttributionExplicit:
-            raise ValueError("trusted_internal org billing requires explicit attribution")
-        return self
+    orgId: StableId
+    billingAttributionExplicit: Literal[True] = True
 
 
 ServingTrafficPrincipal = Annotated[
-    OpenRouterTrafficPrincipal | FreesoloOrgTrafficPrincipal | TrustedInternalTrafficPrincipal,
+    FreesoloOrgTrafficPrincipal | TrustedInternalTrafficPrincipal,
     Field(discriminator="kind"),
 ]
 
@@ -91,9 +66,6 @@ class RequestIdentity:
     request_id: str
     correlation_id: str
     openai_completion_id: str | None = None
-    openrouter_request_id: str | None = None
-    openrouter_generation_id: str | None = None
-    upstream_id: str | None = None
 
     def __post_init__(self) -> None:
         if _GENERATION_ID_RE.fullmatch(self.request_id) is None:
@@ -103,9 +75,6 @@ class RequestIdentity:
             for value in (
                 self.correlation_id,
                 self.openai_completion_id,
-                self.openrouter_request_id,
-                self.openrouter_generation_id,
-                self.upstream_id,
             )
             if value is not None
         }
@@ -126,7 +95,6 @@ class CapturedPrice:
     source: str
     version: str
     snapshot: Mapping[str, Any]
-    quoted_provider_amount_micro_usd: int | None = None
 
 
 @dataclass(frozen=True)
@@ -161,28 +129,16 @@ class UsageEvent:
 
     def rpc_payload(self) -> dict[str, Any]:
         principal = self.principal
-        openrouter = principal if principal.kind == "openrouter" else None
-        org_id = None if principal.kind == "openrouter" else principal.orgId
-        explicit = (
-            principal.billingAttributionExplicit if principal.kind == "trusted_internal" else False
-        )
-        accepted = openrouter.acceptedPriceSnapshot.model_dump(mode="json") if openrouter else None
         return {
             "request_id": self.identity.request_id,
             "correlation_id": self.identity.correlation_id,
             "traffic_principal_kind": principal.kind,
-            "billing_attribution_explicit": explicit,
-            "org_id": org_id,
-            "traffic_source": "openrouter" if openrouter else "freesolo",
-            "openrouter_request_id": self.identity.openrouter_request_id,
-            "openrouter_generation_id": self.identity.openrouter_generation_id,
+            "billing_attribution_explicit": principal.kind == "trusted_internal",
+            "org_id": principal.orgId,
             "openai_completion_id": self.identity.openai_completion_id,
-            "upstream_id": self.identity.upstream_id,
-            "public_model_id": (
-                openrouter.publicModelId if openrouter else self.target.public_model_id
-            ),
+            "public_model_id": self.target.public_model_id,
             "base_model": self.target.base_model,
-            "checkpoint_id": None if openrouter else self.target.checkpoint_id,
+            "checkpoint_id": self.target.checkpoint_id,
             "artifact_fingerprint": self.target.artifact_fingerprint,
             "prompt_tokens": self.facts.prompt_tokens,
             "completion_tokens": self.facts.completion_tokens,
@@ -214,9 +170,6 @@ class UsageEvent:
             "pricing_source": self.price.source,
             "price_version": self.price.version,
             "price_snapshot": dict(self.price.snapshot),
-            "quoted_provider_amount_micro_usd": self.price.quoted_provider_amount_micro_usd,
-            "provider_catalog_digest": (openrouter.providerCatalogDigest if openrouter else None),
-            "accepted_price_snapshot": accepted,
         }
 
 
@@ -249,6 +202,7 @@ class ReconciliationUnavailable(UsageOutboxError):
 class UsageStore(Protocol):
     enabled: bool
 
+    def assert_healthy(self) -> None: ...
     async def start(self) -> None: ...
     async def capture(self, event: UsageEvent) -> None: ...
     async def finalize(self, event: UsageEvent) -> None: ...
@@ -264,6 +218,9 @@ class OfflineUsageStore:
 
     enabled = False
 
+    def assert_healthy(self) -> None:
+        return None
+
     async def start(self) -> None:
         return None
 
@@ -438,6 +395,15 @@ class DurableUsageOutbox:
         self._active_leases: set[str] = set()
         self._background_error: BaseException | None = None
 
+    def assert_healthy(self) -> None:
+        self._raise_background_error()
+        if self._stopping.is_set():
+            raise UsageOutboxError("usage_outbox_not_accepting_requests")
+        if self._worker is not None and self._worker.done():
+            raise UsageOutboxError("usage_outbox_worker_stopped")
+        if self._heartbeat_worker is not None and self._heartbeat_worker.done():
+            raise UsageOutboxError("usage_outbox_heartbeat_stopped")
+
     async def start(self) -> None:
         if self._worker is not None:
             return
@@ -470,7 +436,9 @@ class DurableUsageOutbox:
     async def _terminal_rpc(
         self, name: str, event: UsageEvent, *, failure_code: str | None = None
     ) -> None:
-        self._raise_background_error()
+        # a background delivery failure refuses NEW work at admission; it must not refuse to
+        # settle work already performed. the terminal rpcs are idempotent, so attempting one after
+        # the worker died is safe, while skipping it drops the charge for a served request.
         request_id = event.identity.request_id
         self._terminal_generations.add(request_id)
         try:
@@ -655,7 +623,15 @@ class DurableUsageOutbox:
                     self._wake.set()
             except asyncio.CancelledError:
                 raise
-            except Exception:
+            except Exception as exc:
+                # a non-transient delivery failure is a settlement defect, not a blip. keep it
+                # instead of looping forever so the health gate can refuse new chargeable traffic.
+                if not _is_transient_rpc_error(exc):
+                    self._background_error = exc
+                    self._stopping.set()
+                    self._wake.set()
+                    self._heartbeat_wake.set()
+                    return
                 await self._sleep(self._poll_seconds)
             if self._wake.is_set():
                 continue
@@ -704,7 +680,7 @@ class DurableUsageOutbox:
             except asyncio.CancelledError:
                 raise
             except Exception as exc:
-                if not _is_transient_heartbeat_error(exc):
+                if not _is_transient_rpc_error(exc):
                     raise
                 active_deadlines = [
                     deadline
@@ -922,17 +898,8 @@ class DurableUsageOutbox:
             raise UsageOutboxError("usage_outbox_shutdown_failed") from errors[0]
 
 
-def _is_transient_heartbeat_error(exc: BaseException) -> bool:
-    if not isinstance(exc, UsageOutboxError):
-        return False
-    code = str(exc)
-    if code == "supabase_transport_failure":
-        return True
-    match = re.fullmatch(r"supabase_rpc_([0-9]{3})", code)
-    if match is None:
-        return False
-    status_code = int(match.group(1))
-    return status_code in {408, 429} or status_code >= 500
+def _is_transient_rpc_error(exc: BaseException) -> bool:
+    return isinstance(exc, UsageOutboxError) and is_transient_rpc_code(str(exc))
 
 
 def _generation_capture_result(data: Any) -> dict[str, Any]:
@@ -963,15 +930,16 @@ def _generation_capture_result(data: Any) -> dict[str, Any]:
 
 def _settlement_principal(row: Mapping[str, Any]) -> dict[str, Any]:
     kind = row.get("traffic_principal_kind")
-    if kind == "openrouter":
-        return {"kind": "openrouter", "publicModelId": row.get("public_model_id")}
+    org_id = row.get("org_id")
+    if not isinstance(org_id, str) or not org_id.strip():
+        raise UsageOutboxError("usage_principal_invalid")
     if kind == "freesolo_org":
-        return {"kind": "freesolo_org", "orgId": row.get("org_id")}
-    if kind == "trusted_internal":
+        return {"kind": "freesolo_org", "orgId": org_id}
+    if kind == "trusted_internal" and row.get("billing_attribution_explicit") is True:
         return {
             "kind": "trusted_internal",
-            "orgId": row.get("org_id"),
-            "billingAttributionExplicit": bool(row.get("billing_attribution_explicit")),
+            "orgId": org_id,
+            "billingAttributionExplicit": True,
         }
     raise UsageOutboxError("usage_principal_invalid")
 
@@ -986,13 +954,3 @@ def _parse_datetime(value: Any) -> datetime:
     if parsed.tzinfo is None:
         raise UsageOutboxError("usage_snapshot_timestamp_invalid")
     return parsed
-
-
-def accepted_price_micro_usd(snapshot: AcceptedPriceSnapshot, facts: UsageFacts) -> int:
-    uncached = facts.prompt_tokens - facts.cached_tokens
-    cost = Decimal(snapshot.promptTokenUsd) * uncached
-    cost += Decimal(snapshot.cachedPromptTokenUsd or snapshot.promptTokenUsd) * facts.cached_tokens
-    cost += Decimal(snapshot.completionTokenUsd) * facts.completion_tokens
-    if snapshot.requestUsd is not None:
-        cost += Decimal(snapshot.requestUsd)
-    return int((cost * 1_000_000).quantize(Decimal("1")))

freesolo_flash_dev-1.2.130/flash/serving/src/accounting/usage_retry.py

@@ -0,0 +1,13 @@
+"""Retry classification for durable serving accounting RPCs."""
+
+import re
+
+
+def is_transient_rpc_code(code: str) -> bool:
+    if code == "supabase_transport_failure":
+        return True
+    match = re.fullmatch(r"supabase_rpc_([0-9]{3})", code)
+    if match is None:
+        return False
+    status_code = int(match.group(1))
+    return status_code in {408, 429} or status_code >= 500

{freesolo_flash_dev-1.2.129 → freesolo_flash_dev-1.2.130}/flash/serving/src/engine/model_config.py

@@ -34,10 +34,10 @@ from flash.serving.src.engine.prequant_config import (
 # checkpoints published to the operator HF org.
 #
 # sizing rationale. preallocated lora buffers and loaded checkpoint weights dominate engine vram.
-# the active 9b uses 16 rank-128 slots on l40s, the active 27b uses 16 rank-64 slots on h100, and the
-# active 35b moe uses 6 rank-64 slots on h200.
-#   - Qwen3.6-35B-A3B (vision-language MoE; arch ``Qwen3_5MoeForConditionalGeneration``) -> H200
-#     (141 GiB) with the base bf16 weights, 6 x 64 LoRA at 32k. bf16 (not FP8) is the one path giving
+# all three active tiers serve on b200: the 9b uses 16 rank-128 slots, the 27b 16 rank-64 slots, and
+# the 35b moe 6 rank-64 slots.
+#   - Qwen3.6-35B-A3B (vision-language MoE; arch ``Qwen3_5MoeForConditionalGeneration``) -> B200
+#     (178 GiB) with the base bf16 weights, 6 x 64 LoRA at 32k. bf16 (not FP8) is the one path giving
 #     full-expert LoRA + CUDA graphs because the fused-MoE LoRA path won't compile on fp8e4nv. see the
 #     detailed 35B block below.
 #
@@ -49,11 +49,13 @@ SERVING_MODELS: list[dict[str, Any]] = [
     {
         "base_model": "Qwen/Qwen3.5-9B",
         "image_input_limit": 4,
-        "gpu": "L40S",
+        "gpu": "B200",
         "engine": {
-            # the L40S (48 GiB, Ada sm89) is the cheapest Modal card that fits rank-128 x 16 LoRA
-            # at 32k; L4 and 2xL4 OOMed in the real-GPU sweep. keep CUDA graphs on because eager is
-            # about 10x slower for this hybrid GatedDeltaNet model, and keep 0.90 for graph-capture headroom.
+            # measured on B200 (2026-08-31): 132.87 GiB of KV, 7,109,793 tokens, 217x concurrency at
+            # 32k with rank-128 x 16 LoRA. this tier previously ran on the L40S (48 GiB, Ada sm89),
+            # the cheapest card that fit the LoRA buffer; L4 and 2xL4 OOMed in the real-GPU sweep.
+            # keep CUDA graphs on because eager is about 10x slower for this hybrid GatedDeltaNet
+            # model, and keep 0.90 for graph-capture headroom.
             "gpu_memory_utilization": 0.90,
             "max_loras": 16,
             "max_lora_rank": 128,  # rank-128 / 16 hot LoRAs (cheap on the 9 GiB FP8 9B); 32k context.
@@ -63,22 +65,23 @@ SERVING_MODELS: list[dict[str, Any]] = [
             "reasoning_parser": "qwen3",
         },
     },
-    # 35B-A3B MoE: bf16 on an H200 (141 GiB) is the one serving path that gives a flash adapter its
+    # 35B-A3B MoE: bf16 on a B200 (178 GiB) is the one serving path that gives a flash adapter its
     # full all-expert LoRA and CUDA graphs at speed. it gets rank 64 at 6 hot slots (6 x 64).
-    # why bf16/H200 and not the FP8 checkpoint used by every other tier:
+    # why bf16 and not the FP8 checkpoint used by every other tier:
     #   * FP8 on A100 materializes the FP8 experts back to bf16 in the fused-MoE LoRA path, leaving no
     #     room for CUDA-graph capture and forcing eager at about 4-10 tok/s.
     #   * FP8 on H200/B200 fails the fused-MoE LoRA kernel with "Unsupported lhs dtype fp8e4nv"; only
     #     the A100's Marlin kernel runs this MoE's full-expert LoRA.
-    #   * bf16 on H200 sidesteps the FP8 kernel. the real-GPU canary found that 8 x 64 LoRA plus 32k
-    #     overflows the 141 GiB card, with only about 19k context fitting. 6 x 64 plus 32k fits cleanly
-    #     with a 679,701-token KV cache, about 20x concurrency at 32k.
-    #   * cold boot is about 17 min (67 gibibytes of weights plus compile, graph capture, and warmup),
-    #     so it needs the raised startup_timeout in modal_app. inference or adapter registration starts it.
+    #   * bf16 sidesteps the FP8 kernel. on the 141 GiB H200 this tier used to run on, 8 x 64 LoRA
+    #     plus 32k overflowed the card (only about 19k context fit) and 6 x 64 measured about 20x
+    #     concurrency; the 178 GiB B200 measured 112x at the same 6 x 64 shape.
+    #   * cold boot measured 488s on the B200 (67 gibibytes of weights plus compile, graph capture
+    #     and warmup), so it needs the raised startup_timeout in modal_app. inference or adapter
+    #     registration starts it.
     {
         "base_model": "Qwen/Qwen3.6-35B-A3B",
         "image_input_limit": 4,
-        "gpu": "H200",
+        "gpu": "B200",
         "engine": {
             # Load the BASE bf16 weights, NOT the FP8 checkpoint — bf16 is what lets full-expert LoRA
             # + graphs coexist (see the block comment above). serve_model_id overrides the FP8 default
@@ -91,12 +94,13 @@ SERVING_MODELS: list[dict[str, Any]] = [
             "max_loras": 6,
             "max_lora_rank": 64,
             "pin_loras": False,
-            # 32k context at 6 hot rank-64 LoRAs. the real-GPU canary produced a healthy 679,701-token
-            # KV cache, about 20x concurrency at 32k; 8 hot LoRAs overflowed and only fit about 19k.
+            # 32k context at 6 hot rank-64 LoRAs. measured on B200 (2026-08-30): 49.38 GiB of KV,
+            # 3,678,952 tokens, 112x concurrency at 32k -- the 178 GiB card leaves far more room than
+            # the 141 GiB H200 this tier previously ran on (which measured about 20x).
             "max_model_len": 32768,
             "max_num_batched_tokens": 4096,
-            # CUDA graphs ON — the whole point. On bf16/H200 the graph capture fits (~0.2-0.8 GiB) and
-            # is LoRA-specialized, so adapters serve under graphs too.
+            # CUDA graphs ON — the whole point. On bf16/B200 the graph capture fits (measured 0.38 GiB)
+            # and is LoRA-specialized, so adapters serve under graphs too.
             "enforce_eager": False,
             # Startup memory-profiling runs max_num_seqs sequences; cap low so the 248k-vocab logits +
             # all-expert MoE activations don't spike the profiling peak.
@@ -105,20 +109,20 @@ SERVING_MODELS: list[dict[str, Any]] = [
             # NB: the vision encoder is now LOADED (no language_model_only) — flash adapters adapt the
             # full multimodal tree, so their vision-tower LoRA keys must have real modules to bind to.
             # this adds the vision encoder's weights on top of the already weight-bound 6 x 64 LoRA buffer.
-            # the complete model and LoRA load is about 108 GiB on the 141 GiB H200.
+            # the complete model and LoRA load is about 108 GiB, on the 178 GiB B200.
         },
     },
-    # 27B dense on an H100 (80 GiB). the real-GPU canary measured the load at 44.25 GiB -- above the
-    # FP8 weight size alone, because the vision tower and the non-quantized tensors stay bf16 -- which
-    # still leaves 23.07 GiB of KV cache (350,981 tokens, 10.71x concurrency at 32k) after the 16
-    # rank-64 LoRA buffers and a 0.35 GiB graph capture. every repository here is pinned to an
-    # immutable revision so a served engine cannot silently follow an upstream retag; note the model
-    # pin names a commit in the -FP8 repo while the tokenizer/processor pins name one in the base
-    # repo, which are separate sha namespaces.
+    # 27B dense on a B200 (178 GiB). the load is 44.25 GiB -- above the FP8 weight size alone, because
+    # the vision tower and the non-quantized tensors stay bf16. measured on B200 (2026-08-30):
+    # 112.66 GiB of KV cache, 2,856,884 tokens, 87x concurrency at 32k, after the 16 rank-64 LoRA
+    # buffers and graph capture; the 80 GiB H100 this tier previously ran on measured 10.71x. every
+    # repository here is pinned to an immutable revision so a served engine cannot silently follow an
+    # upstream retag; note the model pin names a commit in the -FP8 repo while the tokenizer/processor
+    # pins name one in the base repo, which are separate sha namespaces.
     {
         "base_model": "Qwen/Qwen3.8-27B",
         "image_input_limit": 4,
-        "gpu": "H100",
+        "gpu": "B200",
         "engine": {
             "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
             "tokenizer_model": "Qwen/Qwen3.8-27B",