freesolo-flash-dev 1.2.121__tar.gz → 1.2.123__tar.gz

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.121
+Version: 1.2.123
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/docs/serving-contract.md

@@ -10,7 +10,7 @@ adapters. A different base model requires a separate deployment. Use
 Customer-owned Modal is live-qualified for `Qwen/Qwen3.5-9B`, `Qwen/Qwen3.8-27B`, and
 `Qwen/Qwen3.6-35B-A3B`. The 27B and 35B-A3B qualifications are bound to the exact certified serving
 image digest; another digest remains available for offline planning but cannot allocate Modal.
-Hosted Qwen3.8 27B remains inactive. Its customer-owned engine runs on `H100!` and serves the pinned
+Hosted Qwen3.8 27B is active on H100. Its customer-owned engine runs on `H100!` and serves the pinned
 `Qwen/Qwen3.8-27B-FP8` checkpoint while preserving `Qwen/Qwen3.8-27B` as the distinct logical base
 and tokenizer provenance. The 35B-A3B engine serves BF16 weights on one H200 with FP8 KV cache, a
 32K context, eight sequences, a 4096 batched-token cap, and six rank-64 LoRA slots.

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/core/catalog.py

@@ -208,7 +208,7 @@ class ModelInfo:
 DEFAULT_MODEL = "Qwen/Qwen3.5-9B"
 
 # the checkpoint each public catalog model loads for customer-owned serving. hosted activation is
-# a separate allowlist in flash.serving.src.engine.model_config, where qwen3.8-27b remains inactive.
+# a separate allowlist in flash.serving.src.engine.model_config.
 SERVING_MODEL_REPOS: dict[str, str] = {
     "Qwen/Qwen3.5-9B": "Freesolo-Co/Qwen3.5-9B-FP8",
     "Qwen/Qwen3.8-27B": "Qwen/Qwen3.8-27B-FP8",

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/app/README.md

@@ -40,9 +40,13 @@ Modal environment `dev` and that production mode does not. The mode does not cha
 engine behavior.
 
 Both environments keep `MIN_CONTAINERS = 0`, so every GPU engine scales to zero and starts on demand.
-The CPU router remains warm. The shared hardcoded settings remain `MAX_CONTAINERS = None`,
-`SCALEDOWN_WINDOW_SECONDS = 1800`, and `MAX_INPUTS = 64` with
-`TARGET_INPUTS = 48`.
+The CPU router remains warm. The hardcoded settings remain `MAX_CONTAINERS = None`,
+`BUFFER_CONTAINERS = 1` (on the router AND every engine), `SCALEDOWN_WINDOW_SECONDS = 1800`, and
+`MAX_INPUTS = 64` with `TARGET_INPUTS = 48`. `MAX_CONTAINERS` is deliberately uncapped: `base_model` is a
+`modal.parameter()`, so each model has its own container pool and a fixed cap would ceiling one
+model's capacity rather than bounding total spend. Bound spend with workspace quotas and billing
+alerts instead. `BUFFER_CONTAINERS` only provisions while a Function is active, so engines still
+scale to zero when idle.
 
 ### Production
 
@@ -150,13 +154,12 @@ GPUs, so there's nothing to tune at deploy time:
   checkpoints are VL-preserving (each VL-unified base's vision tower + MoE router gates stay in
   original precision) and already published to the operator HF org. Native FP8 tensor cores need
   compute capability ≥ 8.9; every dense FP8 tier runs on an L4, L40S, or H100 (all native). No serving
-  tier runs on A100 now that the 35B is bf16 on H200. The pending Qwen3.8 candidate allocates no
-  hosted tier until its exact canary passes.
+  tier runs on A100 now that the 35B is bf16 on H200. The 27B serves the official native FP8
+  checkpoint on an H100.
 
 - **The real memory lever — LoRA buffers:** vLLM PRE-ALLOCATES the GPU LoRA buffers at `max_loras ×
 max_lora_rank`, regardless of how many adapters load. Both are linear levers. The qualified 9B runs
-  **16 × 128** on L40S. Qwen3.8-27B's H100 LoRA capacity remains pending exact-checkpoint
-  qualification. The 35B MoE runs **6 × 64** bf16 on H200. Adapters trained above the effective rank are rejected at load; a base regularly serving
+  **16 × 128** on L40S and the 27B runs **16 × 64** on H100. The 35B MoE runs **6 × 64** bf16 on H200. Adapters trained above the effective rank are rejected at load; a base regularly serving
   more distinct adapters than its hot limit pays a swap latency from the CPU pool unless its hot
   limit equals `MAX_CPU_LORAS`.
 
@@ -174,17 +177,19 @@ cold-boot smoke test first.
 | Base model                | Checkpoint                                   | GPU       | Hot LoRA shape | Context |
 | ------------------------- | -------------------------------------------- | --------- | -------------- | ------- |
 | Qwen3.5-9B                | owned FP8 (`Freesolo-Co/*-FP8`)              | **L40S**§ | 16 × 128       | 32768   |
-| Qwen3.8-27B candidate     | official native FP8 (`Qwen/Qwen3.8-27B-FP8`) | inactive‡ | pending        | pending |
+| Qwen3.8-27B               | official native FP8 (`Qwen/Qwen3.8-27B-FP8`) | **H100**‡ | 16 × 64        | 32768   |
 | Qwen3.6-35B-A3B (MoE, VL) | official **bf16** (`Qwen/Qwen3.6-35B-A3B`)   | **H200**† | 6 × 64         | 32768   |
 
 § 9B (L40S) runs **16 × 128** at 32k context with `max_num_seqs=8`, CUDA graphs
 (`enforce_eager=False`), and `gpu_memory_utilization=0.90`. Its rank-128 × 16 buffer OOMed an L4 and
 2×L4 in the real-GPU sweep, so it runs on the 48 GiB L40S, the cheapest Ada card that fits it at 32k.
 
-‡ Qwen3.8-27B is not in active hosted routing, registration, base-model health, or engine deployment.
-Its proposed H100 tier, hot LoRA capacity, context limit, CUDA-graph behavior, throughput, and latency
-remain pending qualification on the exact immutable official FP8 checkpoint. Qwen3.6-27B measurements
-do not qualify or predict the Qwen3.8 checkpoint.
+‡ 27B (H100, 80 GiB) runs **16 × 64** at 32k with `max_num_seqs=8`, CUDA graphs
+(`enforce_eager=False`), and `gpu_memory_utilization=0.90`, qualified by a real-GPU canary on the exact
+immutable official FP8 checkpoint: weights loaded at 44.25 GiB, leaving 23.07 GiB of KV cache
+(350,981 tokens, 10.71x concurrency at 32k) after a 0.35 GiB graph capture. Cold boot is about 20 min
+(load + 137 s torch.compile + 442 s warmup dominated by the FlashInfer GDN JIT), all cached on the
+serving volume. Qwen3.6-27B measurements do not qualify or predict the Qwen3.8 checkpoint.
 
 † 35B-A3B runs **bf16 on an H200** (141 GiB) at **6 × 64**, the one serving path that gives a flash
 adapter its full all-expert LoRA AND CUDA graphs at speed. It gets rank 64 like the 27B

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/app/modal_app.py

@@ -138,8 +138,17 @@ def scaledown_window_for(gpu: str) -> int:
 # gpu model engines scale to zero. inference and adapter registration remote calls start the matching
 # parameter-bound engine on demand.
 MIN_CONTAINERS = 0
-# no autoscaling cap per base-model engine; modal adds capacity as concurrency demands.
+# No autoscaling cap per base-model engine; modal adds capacity as concurrency demands. A fixed cap
+# here is a hard ceiling on a SINGLE model's capacity: `base_model` is a modal.parameter() and Modal
+# gives each value its own container pool, so sustained load on one model cannot borrow headroom from
+# an idle one. Bound spend with workspace quotas and billing alerts, which do not silently convert
+# demand into queueing on the hot tier.
 MAX_CONTAINERS = None
+# One spare warm container for each autoscaling pool -- every base-model engine AND the cpu router --
+# so a burst past `TARGET_INPUTS` does not pay a full cold boot (420s on L40S, 900s on H100, 1010s on
+# H200) before it can be served. Modal only provisions the buffer while a Function is ACTIVE, so this
+# does not defeat `MIN_CONTAINERS = 0`: idle engines still scale to zero and bill nothing.
+BUFFER_CONTAINERS = 1
 # Concurrent requests packed onto one base-model GPU before Modal autoscales a new (costly) one.
 # A real-GPU sweep (scripts/gpu_canary.py::sweep_concurrency on A10G/Qwen2.5-1.5B) showed vLLM
 # throughput scaling near-linearly with no saturation through 128 concurrent, while TTFT stayed
@@ -424,6 +433,7 @@ def _build_engine(
         timeout=TIMEOUT_SECONDS,
         min_containers=MIN_CONTAINERS,
         max_containers=MAX_CONTAINERS,
+        buffer_containers=BUFFER_CONTAINERS,
     )(modal.concurrent(max_inputs=max_inputs, target_inputs=target_inputs)(_Engine))
     # Rebind the module name to the decorated handle, matching the normal ``@app.cls class X`` pattern
     # where the module attribute ends up referring to the decorated class.
@@ -552,6 +562,66 @@ _AUTH_CACHE_TTL_SECONDS = 60.0
 _AUTH_CACHE_MAX_ENTRIES = 4096
 
 
+def _authorize_response_org(resp: Any) -> "str | None":
+    """Translate the backend's authorize response into a billing org, or raise.
+
+    Extracted from _build_chat_authorizer only to keep that closure under the repo's
+    function-size limit; it reads nothing but the response, so hoisting it changes no
+    behaviour and makes each status mapping testable on its own.
+    """
+    from fastapi import HTTPException, status
+
+    if resp.status_code == 200:
+        # Return the authorized billing org so the router can meter a base-model serve to the
+        # caller (a base model has no adapter owner). A malformed response cannot authorize an
+        # external request because that would let base-model usage escape billing.
+        try:
+            org_id = resp.json().get("orgId")
+        except Exception as exc:
+            raise HTTPException(
+                status.HTTP_503_SERVICE_UNAVAILABLE,
+                "serving auth backend returned malformed response",
+            ) from exc
+        if not isinstance(org_id, str) or not org_id:
+            raise HTTPException(
+                status.HTTP_503_SERVICE_UNAVAILABLE,
+                "serving auth backend returned malformed response",
+            )
+        return org_id
+    if resp.status_code == 401:
+        # The backend returns 401 for BOTH an invalid *user* key (our authorize route, body
+        # code "invalid_api_key") AND a rejected serving *machine* bearer (require_internal_token,
+        # decided before the user key is even read). Only the former is the caller's fault; the
+        # latter is a serving misconfiguration we must not report as the user's bad key.
+        code = ""
+        try:
+            detail = resp.json().get("detail")
+            if isinstance(detail, dict):
+                code = detail.get("code") or ""
+        except Exception:  # a non-JSON 401 is an internal-auth failure (below)
+            code = ""
+        if code == "invalid_api_key":
+            raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid Freesolo API key")
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is misconfigured")
+    # Org has no budget to serve (zero balance and no card+auto-topup). The backend gates this
+    # before generation so a broke org can't serve for free; surface it as a clean 402 rather
+    # than letting it fall through to the retryable 503 below.
+    if resp.status_code == 402:
+        raise HTTPException(
+            status.HTTP_402_PAYMENT_REQUIRED,
+            "insufficient balance: top up or enable auto-topup to serve this model",
+        )
+    # Unknown adapter (404) and org mismatch (403) both collapse to 403 so we don't leak
+    # which adapters exist to an unauthorized caller.
+    if resp.status_code in (403, 404):
+        raise HTTPException(status.HTTP_403_FORBIDDEN, "API key is not authorized for this adapter")
+    # Any backend 5xx (500/502/503/504) or other unexpected status is a transient auth-lookup
+    # infra failure (Supabase down, backend overloaded). Surface it as a RETRYABLE 503, never a
+    # 502: a client/load-balancer must be free to retry rather than treat it as a permanent
+    # upstream error. The cache+single-flight above is what makes this failure rare under load.
+    raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unavailable")
+
+
 def _build_chat_authorizer(settings: Any) -> Any:
     """Build the serving->backend chat authorizer, or None when not configured.
 
@@ -585,70 +655,30 @@ def _build_chat_authorizer(settings: Any) -> Any:
 
     # (api_key_sha256, adapter_id) -> (expires_at_monotonic, org_id); populated only on a successful
     # authorization. Per-router-container in-memory (each container reduces its own backend load).
-    _cache: dict[tuple[str, str], tuple[float, str | None]] = {}
+    _cache: dict[tuple[str, str, str], tuple[float, str | None]] = {}
     # In-flight single-flight tasks so concurrent identical misses share ONE backend call.
-    _inflight: dict[tuple[str, str], asyncio.Task[str | None]] = {}
+    _inflight: dict[tuple[str, str, str], asyncio.Task[str | None]] = {}
 
-    async def _authorize_backend(api_key: str, adapter_id: str) -> "str | None":
+    async def _authorize_backend(
+        api_key: str, adapter_id: str, scope: "dict[str, str]"
+    ) -> "str | None":
         try:
-            resp = await _client.post(url, json={"apiKey": api_key, "modelId": adapter_id})
+            resp = await _client.post(
+                url,
+                json={"apiKey": api_key, "modelId": adapter_id},
+                # A training container authenticates with the platform internal key rather
+                # than a customer key, so the backend cannot resolve it via
+                # authenticate_api_key. It identifies the calling job from these headers and
+                # re-checks them against the live job row. Dropping them here is
+                # indistinguishable, to the backend, from a request that never had them: it
+                # refuses with missing_training_context and catalog sampling fails closed.
+                headers=scope or None,
+            )
         except Exception as exc:  # backend unreachable -> fail closed, never serve unauthorized
             raise HTTPException(
                 status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unreachable"
             ) from exc
-        if resp.status_code == 200:
-            # Return the authorized billing org so the router can meter a base-model serve to the
-            # caller (a base model has no adapter owner). A malformed response cannot authorize an
-            # external request because that would let base-model usage escape billing.
-            try:
-                org_id = resp.json().get("orgId")
-            except Exception as exc:
-                raise HTTPException(
-                    status.HTTP_503_SERVICE_UNAVAILABLE,
-                    "serving auth backend returned malformed response",
-                ) from exc
-            if not isinstance(org_id, str) or not org_id:
-                raise HTTPException(
-                    status.HTTP_503_SERVICE_UNAVAILABLE,
-                    "serving auth backend returned malformed response",
-                )
-            return org_id
-        if resp.status_code == 401:
-            # The backend returns 401 for BOTH an invalid *user* key (our authorize route, body
-            # code "invalid_api_key") AND a rejected serving *machine* bearer (require_internal_token,
-            # decided before the user key is even read). Only the former is the caller's fault; the
-            # latter is a serving misconfiguration we must not report as the user's bad key.
-            code = ""
-            try:
-                detail = resp.json().get("detail")
-                if isinstance(detail, dict):
-                    code = detail.get("code") or ""
-            except Exception:  # a non-JSON 401 is an internal-auth failure (below)
-                code = ""
-            if code == "invalid_api_key":
-                raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid Freesolo API key")
-            raise HTTPException(
-                status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is misconfigured"
-            )
-        # Org has no budget to serve (zero balance and no card+auto-topup). The backend gates this
-        # before generation so a broke org can't serve for free; surface it as a clean 402 rather
-        # than letting it fall through to the retryable 503 below.
-        if resp.status_code == 402:
-            raise HTTPException(
-                status.HTTP_402_PAYMENT_REQUIRED,
-                "insufficient balance: top up or enable auto-topup to serve this model",
-            )
-        # Unknown adapter (404) and org mismatch (403) both collapse to 403 so we don't leak
-        # which adapters exist to an unauthorized caller.
-        if resp.status_code in (403, 404):
-            raise HTTPException(
-                status.HTTP_403_FORBIDDEN, "API key is not authorized for this adapter"
-            )
-        # Any backend 5xx (500/502/503/504) or other unexpected status is a transient auth-lookup
-        # infra failure (Supabase down, backend overloaded). Surface it as a RETRYABLE 503, never a
-        # 502: a client/load-balancer must be free to retry rather than treat it as a permanent
-        # upstream error. The cache+single-flight above is what makes this failure rare under load.
-        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unavailable")
+        return _authorize_response_org(resp)
 
     def _prune(now: float) -> None:
         for expired in [k for k, (exp, _) in _cache.items() if exp <= now]:
@@ -659,10 +689,10 @@ def _build_chat_authorizer(settings: Any) -> Any:
                 _cache.pop(k, None)
 
     async def _authorize_and_cache(
-        ck: tuple[str, str], api_key: str, adapter_id: str
+        ck: tuple[str, str, str], api_key: str, adapter_id: str, scope: "dict[str, str]"
     ) -> "str | None":
         try:
-            org = await _authorize_backend(api_key, adapter_id)
+            org = await _authorize_backend(api_key, adapter_id, scope)
         finally:
             # Drop the single-flight slot regardless of outcome so a failed lookup re-checks next
             # time (failures are never cached) and a success can be re-driven once the cache expires.
@@ -673,9 +703,19 @@ def _build_chat_authorizer(settings: Any) -> Any:
         _prune(now)
         return org
 
-    async def authorize(api_key: str, adapter_id: str) -> "str | None":
+    async def authorize(
+        api_key: str, adapter_id: str, scope: "dict[str, str] | None" = None
+    ) -> "str | None":
         # do not retain raw user credentials as live dict keys beyond the request that supplied them.
-        ck = (hashlib.sha256(api_key.encode("utf-8")).hexdigest(), adapter_id)
+        scope = scope or {}
+        # The scope is part of the key, not just the payload. Every training container presents
+        # the SAME platform internal key, so keying on (key, model) alone would let one job's
+        # cached authorization answer another job's request -- and the cached value is the
+        # billing org, so the second job's usage would be metered to the first job's tenant.
+        scope_digest = hashlib.sha256(
+            "\x00".join(f"{k}={scope[k]}" for k in sorted(scope)).encode("utf-8")
+        ).hexdigest()
+        ck = (hashlib.sha256(api_key.encode("utf-8")).hexdigest(), adapter_id, scope_digest)
         cached = _cache.get(ck)
         now = time.monotonic()
         if cached is not None:
@@ -684,7 +724,7 @@ def _build_chat_authorizer(settings: Any) -> Any:
             _cache.pop(ck, None)
         task = _inflight.get(ck)
         if task is None:
-            task = asyncio.ensure_future(_authorize_and_cache(ck, api_key, adapter_id))
+            task = asyncio.ensure_future(_authorize_and_cache(ck, api_key, adapter_id, scope))
             # Retrieve the task's outcome even if every awaiter is cancelled (all identical clients
             # disconnect), so an orphaned single-flight failure doesn't warn "exception never retrieved".
             task.add_done_callback(lambda t: t.cancelled() or t.exception())
@@ -723,6 +763,7 @@ def _base_model_records() -> list:
 @app.function(
     secrets=runtime_secrets,
     min_containers=1,  # one warm CPU front door (hardcoded; no deploy-time knob)
+    buffer_containers=BUFFER_CONTAINERS,  # scales out with the engines; see BUFFER_CONTAINERS
     timeout=ROUTER_TIMEOUT_SECONDS,  # must cover engine cold start (see ROUTER_TIMEOUT_SECONDS)
 )
 @modal.concurrent(max_inputs=MAX_INPUTS, target_inputs=TARGET_INPUTS)

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/accounting/usage.py

@@ -24,16 +24,17 @@ from flash.serving.src.accounting.usage_outbox import (
 from flash.serving.src.io.schemas import AdapterRecord
 
 FREESOLO_PRICING_SOURCE = "freesolo_backend_catalog"
-FREESOLO_PRICING_VERSION = "2026-08-24.1"
+FREESOLO_PRICING_VERSION = "2026-08-27.1"
+# (prompt, completion, cached prompt) usd per million tokens, one entry per active hosted model.
+# these are the charged customer rates, set 5% below the lowest healthy comparable openrouter fp8
+# listing for each model and token category. they are NOT cost-plus: no markup is applied, and they
+# currently sit below the recorded cost basis, so do not reintroduce a multiplier here until measured
+# b200 economics establish a lower truthful cost basis.
 _FREESOLO_USD_PER_MTOK: dict[str, tuple[str, str, str]] = {
-    "Qwen/Qwen3.5-0.8B": ("0.01", "0.05", "0.002"),
-    "Qwen/Qwen3.5-2B": ("0.02", "0.10", "0.004"),
-    "Qwen/Qwen3.5-4B": ("0.03", "0.15", "0.006"),
-    "Qwen/Qwen3.5-9B": ("0.114", "0.19", "0.023"),
-    "Qwen/Qwen3.6-27B": ("0.4254", "3.055", "0.14"),
-    "Qwen/Qwen3.6-35B-A3B": ("0.198", "1.265", "0.066"),
+    "Qwen/Qwen3.5-9B": ("0.095", "0.1425", "0.0276"),
+    "Qwen/Qwen3.8-27B": ("0.3325", "2.4225", "0.03325"),
+    "Qwen/Qwen3.6-35B-A3B": ("0.095", "0.9025", "0.0475"),
 }
-_FREESOLO_MARKUP = Decimal("1.2")
 _USD_PER_MTOK_DIVISOR = Decimal("1000000")
 
 
@@ -173,7 +174,7 @@ def freesolo_price(base_model: str) -> CapturedPrice:
         raise ValueError(f"no durable serving price for base model {base_model!r}") from exc
 
     def per_token(rate: str) -> str:
-        return format(Decimal(rate) * _FREESOLO_MARKUP / _USD_PER_MTOK_DIVISOR, "f")
+        return format(Decimal(rate) / _USD_PER_MTOK_DIVISOR, "f")
 
     return CapturedPrice(
         source=FREESOLO_PRICING_SOURCE,

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/engine/model_config.py

@@ -3,9 +3,9 @@
 The serving app runs one vLLM GPU engine per base model, each on the Modal GPU class set by its
 catalog ``gpu`` (see below). Adapters and routing key off the logical ``base_model``; the engine
 loads its configured checkpoint at that checkpoint's own dtype. The active dense 9B uses Freesolo
-compressed-tensors FP8. Qwen3.8-27B retains pinned candidate metadata but is excluded from active
-lookups until its exact canary passes. The active 35B-A3B MoE serves base bf16 weights because its
-fused-MoE LoRA path will not compile on FP8, as detailed below.
+compressed-tensors FP8 and the active dense 27B uses the official Qwen FP8 checkpoint. The active
+35B-A3B MoE serves base bf16 weights because its fused-MoE LoRA path will not compile on FP8, as
+detailed below.
 """
 
 from __future__ import annotations
@@ -27,15 +27,14 @@ from flash.serving.src.engine.prequant_config import (
 # `engine` (optional): per-model vLLM engine-arg overrides (LoRA buffer shape, scheduler/memory caps,
 # language_model_only, …). The engine's LOADED checkpoint is NOT in here — it is resolved centrally by
 # ``serve_model_for`` from ``src.prequant_config`` and injected into the overrides as ``serve_model_id``.
-# active models resolve through prequant_config unless an exact validated override is present. the
-# pending 27b candidate carries its own immutable checkpoint pin without entering active resolution.
+# active models resolve through prequant_config unless an exact validated override is present.
 # ⚠ serve_model_id is pointed only at checkpoints VERIFIED to exist (a missing repo 404-crash-loops
 # the engine — the reason this mechanism was removed once); the owned repos are VL-preserving FP8
 # checkpoints published to the operator HF org.
 #
 # sizing rationale. preallocated lora buffers and loaded checkpoint weights dominate engine vram.
-# the active 9b uses 16 rank-128 slots on l40s and the active 35b moe uses 6 rank-64 slots on h200.
-# the pending 27b candidate retains its proposed h100 shape only for the exact canary.
+# the active 9b uses 16 rank-128 slots on l40s, the active 27b uses 16 rank-64 slots on h100, and the
+# active 35b moe uses 6 rank-64 slots on h200.
 #   - Qwen3.6-35B-A3B (vision-language MoE; arch ``Qwen3_5MoeForConditionalGeneration``) -> H200
 #     (141 GiB) with the base bf16 weights, 6 x 64 LoRA at 32k. bf16 (not FP8) is the one path giving
 #     full-expert LoRA + CUDA graphs because the fused-MoE LoRA path won't compile on fp8e4nv. see the
@@ -108,32 +107,35 @@ SERVING_MODELS: list[dict[str, Any]] = [
             # the complete model and LoRA load is about 108 GiB on the 141 GiB H200.
         },
     },
+    # 27B dense on an H100 (80 GiB). the real-GPU canary measured the load at 44.25 GiB -- above the
+    # FP8 weight size alone, because the vision tower and the non-quantized tensors stay bf16 -- which
+    # still leaves 23.07 GiB of KV cache (350,981 tokens, 10.71x concurrency at 32k) after the 16
+    # rank-64 LoRA buffers and a 0.35 GiB graph capture. every repository here is pinned to an
+    # immutable revision so a served engine cannot silently follow an upstream retag; note the model
+    # pin names a commit in the -FP8 repo while the tokenizer/processor pins name one in the base
+    # repo, which are separate sha namespaces.
+    {
+        "base_model": "Qwen/Qwen3.8-27B",
+        "image_input_limit": 4,
+        "gpu": "H100",
+        "engine": {
+            "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
+            "tokenizer_model": "Qwen/Qwen3.8-27B",
+            "tokenizer_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
+            "processor_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
+            "gpu_memory_utilization": 0.90,
+            "max_loras": 16,
+            "max_lora_rank": 64,
+            "max_model_len": 32768,
+            "max_num_seqs": 8,
+            "enforce_eager": False,
+            "reasoning_parser": "qwen3",
+        },
+    },
 ]
 
 _BY_MODEL: dict[str, dict[str, Any]] = {m["base_model"]: m for m in SERVING_MODELS}
 
-# inert descriptor for the exact pending canary. activation means moving this descriptor into
-# SERVING_MODELS, not consulting a second runtime allowlist.
-_QWEN38_HOSTED_CANDIDATE: dict[str, Any] = {
-    "base_model": "Qwen/Qwen3.8-27B",
-    "image_input_limit": 4,
-    "gpu": "H100",
-    "engine": {
-        "serve_model_id": "Qwen/Qwen3.8-27B-FP8",
-        "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
-        "tokenizer_model": "Qwen/Qwen3.8-27B",
-        "tokenizer_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
-        "processor_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
-        "gpu_memory_utilization": 0.90,
-        "max_loras": 16,
-        "max_lora_rank": 64,
-        "max_model_len": 32768,
-        "max_num_seqs": 8,
-        "enforce_eager": False,
-        "reasoning_parser": "qwen3",
-    },
-}
-
 
 def base_models() -> list[str]:
     return [model["base_model"] for model in SERVING_MODELS]

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/engine/prequant_config.py

@@ -1,9 +1,8 @@
 """Prequantized checkpoint mapping for active hosted serving.
 
-The 9B loads a Freesolo-owned compressed-tensors FP8 checkpoint. The 35B MoE mapping remains its
-official FP8 default, although the validated engine override serves the base bf16 weights so
-full-expert LoRA and CUDA graphs coexist. Pending hosted candidates keep their pins in model_config
-without entering this active mapping.
+The 9B loads a Freesolo-owned compressed-tensors FP8 checkpoint. The 27B loads its official FP8
+checkpoint. The 35B MoE mapping remains its official FP8 default, although the validated engine
+override serves the base bf16 weights so full-expert LoRA and CUDA graphs coexist.
 """
 
 from __future__ import annotations
@@ -15,6 +14,7 @@ OWNED_FP8_MODEL_REPOS: dict[str, str] = {
 
 # official qwen checkpoints active in hosted serving.
 OFFICIAL_FP8_MODEL_REPOS: dict[str, str] = {
+    "Qwen/Qwen3.8-27B": "Qwen/Qwen3.8-27B-FP8",
     "Qwen/Qwen3.6-35B-A3B": "Qwen/Qwen3.6-35B-A3B-FP8",
 }
 

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/context.py

@@ -20,6 +20,7 @@ from flash.serving.src.http.headers import (
     assert_internal,
     internal_org_id,
     is_trusted_internal,
+    training_scope_headers,
 )
 from flash.serving.src.http.routing import AdapterRouter, EnginePool
 from flash.serving.src.io.schemas import AdapterRecord
@@ -42,7 +43,10 @@ class ServingContext:
         serving_release: str,
         reload_records: Callable[[], list[AdapterRecord]] | None,
         lookup_record: Callable[[str, str], AdapterRecord | None] | None,
-        chat_authorizer: Callable[[str, str], Awaitable[str | AuthorizedTraffic | None]] | None,
+        chat_authorizer: Callable[
+            [str, str, dict[str, str]], Awaitable[str | AuthorizedTraffic | None]
+        ]
+        | None,
     ) -> None:
         self.pool = pool
         self.router = router
@@ -83,7 +87,7 @@ class ServingContext:
             raise HTTPException(
                 status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is not configured"
             )
-        authorized = await self.chat_authorizer(token, adapter_id)
+        authorized = await self.chat_authorizer(token, adapter_id, training_scope_headers(request))
         if isinstance(authorized, AuthorizedTraffic):
             if authorized.principal.kind == "trusted_internal":
                 raise HTTPException(

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/headers.py

@@ -50,6 +50,34 @@ def is_trusted_internal(request: Request, trusted_keys: tuple[str, ...]) -> bool
     return any([hmac.compare_digest(presented, k) for k in trusted_keys])  # noqa: C419
 
 
+# The job-scope headers a training container sends with a catalog-base sample. The backend
+# re-checks all five against the live training job row before authorizing, so they have to
+# survive the hop through this service -- it forwards the caller's key but is otherwise the
+# only thing standing between the container and /api/serving/authorize.
+TRAINING_SCOPE_HEADERS = (
+    "x-freesolo-org-id",
+    "x-freesolo-training-job-id",
+    "x-freesolo-project-id",
+    "x-freesolo-worker-id",
+    "x-freesolo-training-job-attempt",
+)
+
+
+def training_scope_headers(request: Request) -> dict[str, str]:
+    """The training job-scope headers present on this request, lowercased.
+
+    Absent headers are omitted rather than sent empty: a normal customer chat request
+    carries none of these, and forwarding five blanks would make every such request look
+    like a malformed training call to the backend.
+    """
+    found = {}
+    for name in TRAINING_SCOPE_HEADERS:
+        value = (request.headers.get(name) or "").strip()
+        if value:
+            found[name] = value
+    return found
+
+
 def _bearer_token(request: Request) -> str | None:
     header = request.headers.get("Authorization") or ""
     scheme, _, value = header.partition(" ")

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/router.py

@@ -56,7 +56,7 @@ def build_serving_app(
     lookup_record: Callable[[str, str], AdapterRecord | None] | None = None,
     reload_interval_seconds: float = 30.0,
     usage_store: UsageStore,
-    chat_authorizer: Callable[[str, str], Awaitable["str | None"]] | None = None,
+    chat_authorizer: Callable[[str, str, dict[str, str]], Awaitable["str | None"]] | None = None,
 ):
     """Front-door FastAPI app. ``reload_records`` re-reads persisted ready adapters so a router
     that missed a (un)registration on another container still resolves it: reload once on a miss

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.121"
+version = "1.2.123"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -196,7 +196,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.121"
+version = "1.2.123"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/tests/serving/test_base_model_serving.py

@@ -86,7 +86,7 @@ class FakeAuthorizer:
         self.calls = []
         self._org = org
 
-    async def __call__(self, token, adapter_id):
+    async def __call__(self, token, adapter_id, scope=None):
         self.calls.append((token, adapter_id))
         return self._org
 

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/tests/serving/test_chat_auth.py

@@ -81,7 +81,7 @@ class FakeAuthorizer:
         self.calls: list[tuple[str, str]] = []
         self._raises = raises
 
-    async def __call__(self, token: str, adapter_id: str) -> str:
+    async def __call__(self, token: str, adapter_id: str, scope: dict | None = None) -> str:
         self.calls.append((token, adapter_id))
         if self._raises is not None:
             raise self._raises

{freesolo_flash_dev-1.2.121 → freesolo_flash_dev-1.2.123}/tests/serving/test_chat_authorizer.py

@@ -82,7 +82,7 @@ def _authorizer(modal_app_module, monkeypatch, response: Any):
         def __init__(self, *_a: Any, **_k: Any) -> None:
             pass
 
-        async def post(self, _url: str, json: Any) -> _FakeResp:
+        async def post(self, _url: str, json: Any, headers: Any = None) -> _FakeResp:
             if isinstance(response, Exception):
                 raise response
             return response
@@ -194,7 +194,7 @@ def _counting_client(monkeypatch, responder, *, delay: float = 0.0):
         def __init__(self, *_a: Any, **_k: Any) -> None:
             pass
 
-        async def post(self, _url: str, json: Any) -> _FakeResp:
+        async def post(self, _url: str, json: Any, headers: Any = None) -> _FakeResp:
             calls["n"] += 1
             n = calls["n"]
             if delay:
@@ -413,6 +413,9 @@ def test_failed_authorization_is_not_cached(modal_app_module, monkeypatch):
     assert calls["n"] == 2
 
 
+_EMPTY_SCOPE_DIGEST = hashlib.sha256(b"").hexdigest()
+
+
 def test_expired_key_is_evicted_before_reauthorization(modal_app_module, monkeypatch):
     import asyncio
     import inspect
@@ -426,7 +429,11 @@ def test_expired_key_is_evicted_before_reauthorization(modal_app_module, monkeyp
     authorize = _new_authorizer(modal_app_module)
     cache = inspect.getclosurevars(authorize).nonlocals["_cache"]
     request_key = ("synthetic-expired-key", "adapter-1")
-    cache_key = (hashlib.sha256(request_key[0].encode("utf-8")).hexdigest(), request_key[1])
+    cache_key = (
+        hashlib.sha256(request_key[0].encode("utf-8")).hexdigest(),
+        request_key[1],
+        _EMPTY_SCOPE_DIGEST,
+    )
 
     async def run() -> None:
         await authorize(*request_key)
@@ -455,10 +462,12 @@ def test_prune_removes_expired_keys_below_capacity(modal_app_module, monkeypatch
     expired_key = (
         hashlib.sha256(expired_request[0].encode("utf-8")).hexdigest(),
         expired_request[1],
+        _EMPTY_SCOPE_DIGEST,
     )
     current_key = (
         hashlib.sha256(current_request[0].encode("utf-8")).hexdigest(),
         current_request[1],
+        _EMPTY_SCOPE_DIGEST,
     )
 
     async def run() -> None: