freesolo-flash-dev 1.2.122__tar.gz → 1.2.123__tar.gz

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.122
+Version: 1.2.123
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/flash/serving/app/README.md

@@ -40,9 +40,13 @@ Modal environment `dev` and that production mode does not. The mode does not cha
 engine behavior.
 
 Both environments keep `MIN_CONTAINERS = 0`, so every GPU engine scales to zero and starts on demand.
-The CPU router remains warm. The shared hardcoded settings remain `MAX_CONTAINERS = None`,
-`SCALEDOWN_WINDOW_SECONDS = 1800`, and `MAX_INPUTS = 64` with
-`TARGET_INPUTS = 48`.
+The CPU router remains warm. The hardcoded settings remain `MAX_CONTAINERS = None`,
+`BUFFER_CONTAINERS = 1` (on the router AND every engine), `SCALEDOWN_WINDOW_SECONDS = 1800`, and
+`MAX_INPUTS = 64` with `TARGET_INPUTS = 48`. `MAX_CONTAINERS` is deliberately uncapped: `base_model` is a
+`modal.parameter()`, so each model has its own container pool and a fixed cap would ceiling one
+model's capacity rather than bounding total spend. Bound spend with workspace quotas and billing
+alerts instead. `BUFFER_CONTAINERS` only provisions while a Function is active, so engines still
+scale to zero when idle.
 
 ### Production
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/flash/serving/app/modal_app.py

@@ -138,12 +138,17 @@ def scaledown_window_for(gpu: str) -> int:
 # gpu model engines scale to zero. inference and adapter registration remote calls start the matching
 # parameter-bound engine on demand.
 MIN_CONTAINERS = 0
-# At most 3 GPUs per base model. `base_model` is a modal.parameter(), and Modal gives each distinct
-# parameter value its own container pool with its own autoscaling accounting, so this bound applies
-# PER MODEL rather than across the catalog. Engines are TENSOR_PARALLEL_SIZE 1 on a single-card `gpu=`
-# request, so one container is exactly one GPU and 3 containers is exactly 3 GPUs. Raising this bounds
-# spend per model, not total spend: N models can still reach 3N GPUs.
-MAX_CONTAINERS = 3
+# No autoscaling cap per base-model engine; modal adds capacity as concurrency demands. A fixed cap
+# here is a hard ceiling on a SINGLE model's capacity: `base_model` is a modal.parameter() and Modal
+# gives each value its own container pool, so sustained load on one model cannot borrow headroom from
+# an idle one. Bound spend with workspace quotas and billing alerts, which do not silently convert
+# demand into queueing on the hot tier.
+MAX_CONTAINERS = None
+# One spare warm container for each autoscaling pool -- every base-model engine AND the cpu router --
+# so a burst past `TARGET_INPUTS` does not pay a full cold boot (420s on L40S, 900s on H100, 1010s on
+# H200) before it can be served. Modal only provisions the buffer while a Function is ACTIVE, so this
+# does not defeat `MIN_CONTAINERS = 0`: idle engines still scale to zero and bill nothing.
+BUFFER_CONTAINERS = 1
 # Concurrent requests packed onto one base-model GPU before Modal autoscales a new (costly) one.
 # A real-GPU sweep (scripts/gpu_canary.py::sweep_concurrency on A10G/Qwen2.5-1.5B) showed vLLM
 # throughput scaling near-linearly with no saturation through 128 concurrent, while TTFT stayed
@@ -428,6 +433,7 @@ def _build_engine(
         timeout=TIMEOUT_SECONDS,
         min_containers=MIN_CONTAINERS,
         max_containers=MAX_CONTAINERS,
+        buffer_containers=BUFFER_CONTAINERS,
     )(modal.concurrent(max_inputs=max_inputs, target_inputs=target_inputs)(_Engine))
     # Rebind the module name to the decorated handle, matching the normal ``@app.cls class X`` pattern
     # where the module attribute ends up referring to the decorated class.
@@ -556,6 +562,66 @@ _AUTH_CACHE_TTL_SECONDS = 60.0
 _AUTH_CACHE_MAX_ENTRIES = 4096
 
 
+def _authorize_response_org(resp: Any) -> "str | None":
+    """Translate the backend's authorize response into a billing org, or raise.
+
+    Extracted from _build_chat_authorizer only to keep that closure under the repo's
+    function-size limit; it reads nothing but the response, so hoisting it changes no
+    behaviour and makes each status mapping testable on its own.
+    """
+    from fastapi import HTTPException, status
+
+    if resp.status_code == 200:
+        # Return the authorized billing org so the router can meter a base-model serve to the
+        # caller (a base model has no adapter owner). A malformed response cannot authorize an
+        # external request because that would let base-model usage escape billing.
+        try:
+            org_id = resp.json().get("orgId")
+        except Exception as exc:
+            raise HTTPException(
+                status.HTTP_503_SERVICE_UNAVAILABLE,
+                "serving auth backend returned malformed response",
+            ) from exc
+        if not isinstance(org_id, str) or not org_id:
+            raise HTTPException(
+                status.HTTP_503_SERVICE_UNAVAILABLE,
+                "serving auth backend returned malformed response",
+            )
+        return org_id
+    if resp.status_code == 401:
+        # The backend returns 401 for BOTH an invalid *user* key (our authorize route, body
+        # code "invalid_api_key") AND a rejected serving *machine* bearer (require_internal_token,
+        # decided before the user key is even read). Only the former is the caller's fault; the
+        # latter is a serving misconfiguration we must not report as the user's bad key.
+        code = ""
+        try:
+            detail = resp.json().get("detail")
+            if isinstance(detail, dict):
+                code = detail.get("code") or ""
+        except Exception:  # a non-JSON 401 is an internal-auth failure (below)
+            code = ""
+        if code == "invalid_api_key":
+            raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid Freesolo API key")
+        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is misconfigured")
+    # Org has no budget to serve (zero balance and no card+auto-topup). The backend gates this
+    # before generation so a broke org can't serve for free; surface it as a clean 402 rather
+    # than letting it fall through to the retryable 503 below.
+    if resp.status_code == 402:
+        raise HTTPException(
+            status.HTTP_402_PAYMENT_REQUIRED,
+            "insufficient balance: top up or enable auto-topup to serve this model",
+        )
+    # Unknown adapter (404) and org mismatch (403) both collapse to 403 so we don't leak
+    # which adapters exist to an unauthorized caller.
+    if resp.status_code in (403, 404):
+        raise HTTPException(status.HTTP_403_FORBIDDEN, "API key is not authorized for this adapter")
+    # Any backend 5xx (500/502/503/504) or other unexpected status is a transient auth-lookup
+    # infra failure (Supabase down, backend overloaded). Surface it as a RETRYABLE 503, never a
+    # 502: a client/load-balancer must be free to retry rather than treat it as a permanent
+    # upstream error. The cache+single-flight above is what makes this failure rare under load.
+    raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unavailable")
+
+
 def _build_chat_authorizer(settings: Any) -> Any:
     """Build the serving->backend chat authorizer, or None when not configured.
 
@@ -589,70 +655,30 @@ def _build_chat_authorizer(settings: Any) -> Any:
 
     # (api_key_sha256, adapter_id) -> (expires_at_monotonic, org_id); populated only on a successful
     # authorization. Per-router-container in-memory (each container reduces its own backend load).
-    _cache: dict[tuple[str, str], tuple[float, str | None]] = {}
+    _cache: dict[tuple[str, str, str], tuple[float, str | None]] = {}
     # In-flight single-flight tasks so concurrent identical misses share ONE backend call.
-    _inflight: dict[tuple[str, str], asyncio.Task[str | None]] = {}
+    _inflight: dict[tuple[str, str, str], asyncio.Task[str | None]] = {}
 
-    async def _authorize_backend(api_key: str, adapter_id: str) -> "str | None":
+    async def _authorize_backend(
+        api_key: str, adapter_id: str, scope: "dict[str, str]"
+    ) -> "str | None":
         try:
-            resp = await _client.post(url, json={"apiKey": api_key, "modelId": adapter_id})
+            resp = await _client.post(
+                url,
+                json={"apiKey": api_key, "modelId": adapter_id},
+                # A training container authenticates with the platform internal key rather
+                # than a customer key, so the backend cannot resolve it via
+                # authenticate_api_key. It identifies the calling job from these headers and
+                # re-checks them against the live job row. Dropping them here is
+                # indistinguishable, to the backend, from a request that never had them: it
+                # refuses with missing_training_context and catalog sampling fails closed.
+                headers=scope or None,
+            )
         except Exception as exc:  # backend unreachable -> fail closed, never serve unauthorized
             raise HTTPException(
                 status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unreachable"
             ) from exc
-        if resp.status_code == 200:
-            # Return the authorized billing org so the router can meter a base-model serve to the
-            # caller (a base model has no adapter owner). A malformed response cannot authorize an
-            # external request because that would let base-model usage escape billing.
-            try:
-                org_id = resp.json().get("orgId")
-            except Exception as exc:
-                raise HTTPException(
-                    status.HTTP_503_SERVICE_UNAVAILABLE,
-                    "serving auth backend returned malformed response",
-                ) from exc
-            if not isinstance(org_id, str) or not org_id:
-                raise HTTPException(
-                    status.HTTP_503_SERVICE_UNAVAILABLE,
-                    "serving auth backend returned malformed response",
-                )
-            return org_id
-        if resp.status_code == 401:
-            # The backend returns 401 for BOTH an invalid *user* key (our authorize route, body
-            # code "invalid_api_key") AND a rejected serving *machine* bearer (require_internal_token,
-            # decided before the user key is even read). Only the former is the caller's fault; the
-            # latter is a serving misconfiguration we must not report as the user's bad key.
-            code = ""
-            try:
-                detail = resp.json().get("detail")
-                if isinstance(detail, dict):
-                    code = detail.get("code") or ""
-            except Exception:  # a non-JSON 401 is an internal-auth failure (below)
-                code = ""
-            if code == "invalid_api_key":
-                raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid Freesolo API key")
-            raise HTTPException(
-                status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is misconfigured"
-            )
-        # Org has no budget to serve (zero balance and no card+auto-topup). The backend gates this
-        # before generation so a broke org can't serve for free; surface it as a clean 402 rather
-        # than letting it fall through to the retryable 503 below.
-        if resp.status_code == 402:
-            raise HTTPException(
-                status.HTTP_402_PAYMENT_REQUIRED,
-                "insufficient balance: top up or enable auto-topup to serve this model",
-            )
-        # Unknown adapter (404) and org mismatch (403) both collapse to 403 so we don't leak
-        # which adapters exist to an unauthorized caller.
-        if resp.status_code in (403, 404):
-            raise HTTPException(
-                status.HTTP_403_FORBIDDEN, "API key is not authorized for this adapter"
-            )
-        # Any backend 5xx (500/502/503/504) or other unexpected status is a transient auth-lookup
-        # infra failure (Supabase down, backend overloaded). Surface it as a RETRYABLE 503, never a
-        # 502: a client/load-balancer must be free to retry rather than treat it as a permanent
-        # upstream error. The cache+single-flight above is what makes this failure rare under load.
-        raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unavailable")
+        return _authorize_response_org(resp)
 
     def _prune(now: float) -> None:
         for expired in [k for k, (exp, _) in _cache.items() if exp <= now]:
@@ -663,10 +689,10 @@ def _build_chat_authorizer(settings: Any) -> Any:
                 _cache.pop(k, None)
 
     async def _authorize_and_cache(
-        ck: tuple[str, str], api_key: str, adapter_id: str
+        ck: tuple[str, str, str], api_key: str, adapter_id: str, scope: "dict[str, str]"
     ) -> "str | None":
         try:
-            org = await _authorize_backend(api_key, adapter_id)
+            org = await _authorize_backend(api_key, adapter_id, scope)
         finally:
             # Drop the single-flight slot regardless of outcome so a failed lookup re-checks next
             # time (failures are never cached) and a success can be re-driven once the cache expires.
@@ -677,9 +703,19 @@ def _build_chat_authorizer(settings: Any) -> Any:
         _prune(now)
         return org
 
-    async def authorize(api_key: str, adapter_id: str) -> "str | None":
+    async def authorize(
+        api_key: str, adapter_id: str, scope: "dict[str, str] | None" = None
+    ) -> "str | None":
         # do not retain raw user credentials as live dict keys beyond the request that supplied them.
-        ck = (hashlib.sha256(api_key.encode("utf-8")).hexdigest(), adapter_id)
+        scope = scope or {}
+        # The scope is part of the key, not just the payload. Every training container presents
+        # the SAME platform internal key, so keying on (key, model) alone would let one job's
+        # cached authorization answer another job's request -- and the cached value is the
+        # billing org, so the second job's usage would be metered to the first job's tenant.
+        scope_digest = hashlib.sha256(
+            "\x00".join(f"{k}={scope[k]}" for k in sorted(scope)).encode("utf-8")
+        ).hexdigest()
+        ck = (hashlib.sha256(api_key.encode("utf-8")).hexdigest(), adapter_id, scope_digest)
         cached = _cache.get(ck)
         now = time.monotonic()
         if cached is not None:
@@ -688,7 +724,7 @@ def _build_chat_authorizer(settings: Any) -> Any:
             _cache.pop(ck, None)
         task = _inflight.get(ck)
         if task is None:
-            task = asyncio.ensure_future(_authorize_and_cache(ck, api_key, adapter_id))
+            task = asyncio.ensure_future(_authorize_and_cache(ck, api_key, adapter_id, scope))
             # Retrieve the task's outcome even if every awaiter is cancelled (all identical clients
             # disconnect), so an orphaned single-flight failure doesn't warn "exception never retrieved".
             task.add_done_callback(lambda t: t.cancelled() or t.exception())
@@ -727,6 +763,7 @@ def _base_model_records() -> list:
 @app.function(
     secrets=runtime_secrets,
     min_containers=1,  # one warm CPU front door (hardcoded; no deploy-time knob)
+    buffer_containers=BUFFER_CONTAINERS,  # scales out with the engines; see BUFFER_CONTAINERS
     timeout=ROUTER_TIMEOUT_SECONDS,  # must cover engine cold start (see ROUTER_TIMEOUT_SECONDS)
 )
 @modal.concurrent(max_inputs=MAX_INPUTS, target_inputs=TARGET_INPUTS)

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/context.py

@@ -20,6 +20,7 @@ from flash.serving.src.http.headers import (
     assert_internal,
     internal_org_id,
     is_trusted_internal,
+    training_scope_headers,
 )
 from flash.serving.src.http.routing import AdapterRouter, EnginePool
 from flash.serving.src.io.schemas import AdapterRecord
@@ -42,7 +43,10 @@ class ServingContext:
         serving_release: str,
         reload_records: Callable[[], list[AdapterRecord]] | None,
         lookup_record: Callable[[str, str], AdapterRecord | None] | None,
-        chat_authorizer: Callable[[str, str], Awaitable[str | AuthorizedTraffic | None]] | None,
+        chat_authorizer: Callable[
+            [str, str, dict[str, str]], Awaitable[str | AuthorizedTraffic | None]
+        ]
+        | None,
     ) -> None:
         self.pool = pool
         self.router = router
@@ -83,7 +87,7 @@ class ServingContext:
             raise HTTPException(
                 status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth is not configured"
             )
-        authorized = await self.chat_authorizer(token, adapter_id)
+        authorized = await self.chat_authorizer(token, adapter_id, training_scope_headers(request))
         if isinstance(authorized, AuthorizedTraffic):
             if authorized.principal.kind == "trusted_internal":
                 raise HTTPException(

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/headers.py

@@ -50,6 +50,34 @@ def is_trusted_internal(request: Request, trusted_keys: tuple[str, ...]) -> bool
     return any([hmac.compare_digest(presented, k) for k in trusted_keys])  # noqa: C419
 
 
+# The job-scope headers a training container sends with a catalog-base sample. The backend
+# re-checks all five against the live training job row before authorizing, so they have to
+# survive the hop through this service -- it forwards the caller's key but is otherwise the
+# only thing standing between the container and /api/serving/authorize.
+TRAINING_SCOPE_HEADERS = (
+    "x-freesolo-org-id",
+    "x-freesolo-training-job-id",
+    "x-freesolo-project-id",
+    "x-freesolo-worker-id",
+    "x-freesolo-training-job-attempt",
+)
+
+
+def training_scope_headers(request: Request) -> dict[str, str]:
+    """The training job-scope headers present on this request, lowercased.
+
+    Absent headers are omitted rather than sent empty: a normal customer chat request
+    carries none of these, and forwarding five blanks would make every such request look
+    like a malformed training call to the backend.
+    """
+    found = {}
+    for name in TRAINING_SCOPE_HEADERS:
+        value = (request.headers.get(name) or "").strip()
+        if value:
+            found[name] = value
+    return found
+
+
 def _bearer_token(request: Request) -> str | None:
     header = request.headers.get("Authorization") or ""
     scheme, _, value = header.partition(" ")

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/flash/serving/src/http/router.py

@@ -56,7 +56,7 @@ def build_serving_app(
     lookup_record: Callable[[str, str], AdapterRecord | None] | None = None,
     reload_interval_seconds: float = 30.0,
     usage_store: UsageStore,
-    chat_authorizer: Callable[[str, str], Awaitable["str | None"]] | None = None,
+    chat_authorizer: Callable[[str, str, dict[str, str]], Awaitable["str | None"]] | None = None,
 ):
     """Front-door FastAPI app. ``reload_records`` re-reads persisted ready adapters so a router
     that missed a (un)registration on another container still resolves it: reload once on a miss

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.122"
+version = "1.2.123"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -196,7 +196,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.122"
+version = "1.2.123"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_base_model_serving.py

@@ -86,7 +86,7 @@ class FakeAuthorizer:
         self.calls = []
         self._org = org
 
-    async def __call__(self, token, adapter_id):
+    async def __call__(self, token, adapter_id, scope=None):
         self.calls.append((token, adapter_id))
         return self._org
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_chat_auth.py

@@ -81,7 +81,7 @@ class FakeAuthorizer:
         self.calls: list[tuple[str, str]] = []
         self._raises = raises
 
-    async def __call__(self, token: str, adapter_id: str) -> str:
+    async def __call__(self, token: str, adapter_id: str, scope: dict | None = None) -> str:
         self.calls.append((token, adapter_id))
         if self._raises is not None:
             raise self._raises

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_chat_authorizer.py

@@ -82,7 +82,7 @@ def _authorizer(modal_app_module, monkeypatch, response: Any):
         def __init__(self, *_a: Any, **_k: Any) -> None:
             pass
 
-        async def post(self, _url: str, json: Any) -> _FakeResp:
+        async def post(self, _url: str, json: Any, headers: Any = None) -> _FakeResp:
             if isinstance(response, Exception):
                 raise response
             return response
@@ -194,7 +194,7 @@ def _counting_client(monkeypatch, responder, *, delay: float = 0.0):
         def __init__(self, *_a: Any, **_k: Any) -> None:
             pass
 
-        async def post(self, _url: str, json: Any) -> _FakeResp:
+        async def post(self, _url: str, json: Any, headers: Any = None) -> _FakeResp:
             calls["n"] += 1
             n = calls["n"]
             if delay:
@@ -413,6 +413,9 @@ def test_failed_authorization_is_not_cached(modal_app_module, monkeypatch):
     assert calls["n"] == 2
 
 
+_EMPTY_SCOPE_DIGEST = hashlib.sha256(b"").hexdigest()
+
+
 def test_expired_key_is_evicted_before_reauthorization(modal_app_module, monkeypatch):
     import asyncio
     import inspect
@@ -426,7 +429,11 @@ def test_expired_key_is_evicted_before_reauthorization(modal_app_module, monkeyp
     authorize = _new_authorizer(modal_app_module)
     cache = inspect.getclosurevars(authorize).nonlocals["_cache"]
     request_key = ("synthetic-expired-key", "adapter-1")
-    cache_key = (hashlib.sha256(request_key[0].encode("utf-8")).hexdigest(), request_key[1])
+    cache_key = (
+        hashlib.sha256(request_key[0].encode("utf-8")).hexdigest(),
+        request_key[1],
+        _EMPTY_SCOPE_DIGEST,
+    )
 
     async def run() -> None:
         await authorize(*request_key)
@@ -455,10 +462,12 @@ def test_prune_removes_expired_keys_below_capacity(modal_app_module, monkeypatch
     expired_key = (
         hashlib.sha256(expired_request[0].encode("utf-8")).hexdigest(),
         expired_request[1],
+        _EMPTY_SCOPE_DIGEST,
     )
     current_key = (
         hashlib.sha256(current_request[0].encode("utf-8")).hexdigest(),
         current_request[1],
+        _EMPTY_SCOPE_DIGEST,
     )
 
     async def run() -> None:

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_checkpoint_identity_repairs.py

@@ -264,7 +264,7 @@ def test_hosted_authorizer_uses_freesolo_model_id_schema(monkeypatch) -> None:
         def __init__(self, *_args, **_kwargs):
             pass
 
-        async def post(self, _url, *, json):
+        async def post(self, _url, *, json, headers=None):
             seen.update(json)
             return _Response()
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_checkpoint_model_validation.py

@@ -72,7 +72,7 @@ class FakeAuthorizer:
         self.calls: list[tuple[str, str]] = []
         self.error = error
 
-    async def __call__(self, token: str, adapter_id: str) -> str:
+    async def __call__(self, token: str, adapter_id: str, scope: dict | None = None) -> str:
         self.calls.append((token, adapter_id))
         if self.error is not None:
             raise self.error

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_multimodal.py

@@ -744,7 +744,7 @@ class _Pool:
         return None
 
 
-async def _allow(_token: str, _adapter_id: str) -> str:
+async def _allow(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
     return "org-1"
 
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_router.py

@@ -28,7 +28,7 @@ from tests.serving.checkpoint_fixtures import (
 from tests.serving.conftest import attest
 
 
-async def _allow(_token: str, _adapter_id: str) -> str:
+async def _allow(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
     """Permissive attributed authorizer for routing tests."""
     return "org-1"
 
@@ -894,7 +894,7 @@ def test_openai_chat_stream_sets_anti_buffering_headers(app_setup):
 def test_openai_chat_rejects_non_boolean_stream_after_authorization(stream):
     authorizations: list[str] = []
 
-    async def _authorize(_token: str, adapter_id: str) -> str:
+    async def _authorize(_token: str, adapter_id: str, _scope: dict | None = None) -> str:
         authorizations.append(adapter_id)
         return "org-1"
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_router_coverage.py

@@ -30,7 +30,7 @@ from flash.serving.src.io.structured_outputs import StructuredOutputsError
 QWEN = "Qwen/Qwen3.5-9B"
 
 
-async def _allow(_token: str, _adapter_id: str) -> str:
+async def _allow(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
     return "org-1"
 
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_router_edge_coverage.py

@@ -19,7 +19,7 @@ from tests.serving.conftest import attest
 QWEN = "Qwen/Qwen3.5-9B"
 
 
-async def _allow(_token: str, _adapter_id: str) -> str:
+async def _allow(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
     return "org-1"
 
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_serve_reasoning_split.py

@@ -26,7 +26,7 @@ from tests.serving.conftest import attest
 QWEN = "Qwen/Qwen3.5-9B"
 
 
-async def _allow(_token: str, _adapter_id: str) -> str:
+async def _allow(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
     return "org-1"
 
 

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_usage_outbox.py

@@ -295,7 +295,7 @@ def test_nonstream_capture_is_awaited_before_successful_response() -> None:
     pool = _Pool()
     store = _Store()
 
-    async def authorize(_token: str, _adapter_id: str) -> str:
+    async def authorize(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
         return "org-1"
 
     app = build_serving_app(
@@ -330,7 +330,7 @@ def test_stream_captures_in_progress_before_response_and_finalizes_same_id() ->
     pool = _Pool()
     store = _Store()
 
-    async def authorize(_token: str, _adapter_id: str) -> str:
+    async def authorize(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
         return "org-1"
 
     app = build_serving_app(
@@ -412,7 +412,7 @@ def test_stream_finalization_preserves_first_event_attestation() -> None:
 def test_stream_capture_failure_before_headers_returns_controlled_503() -> None:
     record = _revision()
 
-    async def authorize(_token: str, _adapter_id: str) -> str:
+    async def authorize(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
         return "org-1"
 
     app = build_serving_app(
@@ -439,7 +439,7 @@ def test_thinking_request_is_rejected_before_engine_dispatch() -> None:
     record = _revision().model_copy(update={"thinking": True})
     pool = _Pool()
 
-    async def authorize(_token: str, _adapter_id: str) -> str:
+    async def authorize(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
         return "org-1"
 
     app = build_serving_app(
@@ -461,7 +461,7 @@ def test_thinking_request_is_rejected_before_engine_dispatch() -> None:
 def test_nonstream_capture_failure_returns_controlled_503() -> None:
     record = _revision()
 
-    async def authorize(_token: str, _adapter_id: str) -> str:
+    async def authorize(_token: str, _adapter_id: str, _scope: dict | None = None) -> str:
         return "org-1"
 
     app = build_serving_app(

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/tests/serving/test_usage_reporter.py

@@ -124,21 +124,31 @@ def test_lora_engine_scales_to_zero_by_default(modal_app_module):
     assert modal_app_module.app.function.call_args.kwargs["min_containers"] == 1
 
 
-def test_every_gpu_engine_is_capped_at_three_containers(modal_app_module):
-    # At most 3 GPUs per base model. `base_model` is a modal.parameter(), so Modal gives each distinct
-    # value its own container pool with its own autoscaling accounting -- the cap therefore binds PER
-    # MODEL, not across the catalog. Engines are tensor-parallel 1 on a single-card `gpu=` request, so
-    # one container is exactly one GPU. Asserted on the kwargs modal is actually CALLED with, because a
-    # ceiling that never reaches `app.cls` does not bound anything.
-    assert modal_app_module.MAX_CONTAINERS == 3
+def test_no_gpu_engine_is_capped_and_each_keeps_one_buffer(modal_app_module):
+    # `base_model` is a modal.parameter(), so Modal gives each distinct value its own container pool
+    # with its own autoscaling accounting. A fixed cap therefore ceilings a SINGLE model's capacity
+    # rather than bounding total spend -- sustained load on one model cannot borrow headroom from an
+    # idle one, so the cap converts demand into queueing on the hot tier. Spend is bounded by workspace
+    # quotas and billing alerts instead. Asserted on the kwargs modal is actually CALLED with, because
+    # a module constant that never reaches `app.cls` governs nothing.
+    assert modal_app_module.MAX_CONTAINERS is None
+    assert modal_app_module.BUFFER_CONTAINERS == 1
 
     cls_calls = [call.kwargs for call in modal_app_module.app.cls.call_args_list]
     assert len(cls_calls) == len(modal_app_module.ENGINE_BY_KEY)
-    assert all(kwargs["max_containers"] == 3 for kwargs in cls_calls)
-
-    # The cpu router is deliberately UNCAPPED: it is the front door that triggers cold engines, and
-    # capping it would throttle every model at once rather than bounding gpu spend per model.
-    assert "max_containers" not in modal_app_module.app.function.call_args.kwargs
+    assert all(kwargs["max_containers"] is None for kwargs in cls_calls)
+    # One spare warm container per engine absorbs a burst past TARGET_INPUTS without a cold boot.
+    # `buffer_containers` only provisions while the Function is ACTIVE, so this preserves scale-to-zero
+    # (MIN_CONTAINERS stays 0) rather than paying for an idle gpu.
+    assert all(kwargs["buffer_containers"] == 1 for kwargs in cls_calls)
+    assert all(kwargs["min_containers"] == 0 for kwargs in cls_calls)
+
+    # The cpu router is likewise uncapped: it is the front door that triggers cold engines, and
+    # capping it would throttle every model at once rather than bounding gpu spend per model. It
+    # carries the same buffer, so a burst does not queue behind the front door it just cleared.
+    router_kwargs = modal_app_module.app.function.call_args.kwargs
+    assert "max_containers" not in router_kwargs
+    assert router_kwargs["buffer_containers"] == 1
 
 
 def test_scaledown_window_is_per_tier_and_cheaper_tiers_release_sooner(modal_app_module):

{freesolo_flash_dev-1.2.122 → freesolo_flash_dev-1.2.123}/uv.lock

@@ -1480,7 +1480,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.2.122"
+version = "1.2.123"
 source = { editable = "." }
 
 [package.optional-dependencies]