freesolo-flash-dev 1.4.15__tar.gz → 1.4.16__tar.gz

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.15
+Version: 1.4.16
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/core/catalog.py

@@ -382,7 +382,7 @@ MODELS: dict[str, ModelInfo] = {
             max_cpu_loras=16,
             max_lora_rank=128,
             max_model_len=262144,
-            max_num_seqs=8,
+            max_num_seqs=16,
             tensor_parallel_size=1,
             gpu_memory_utilization=0.90,
             image_limit=MAX_IMAGES_PER_CONVERSATION,
@@ -549,7 +549,7 @@ MODELS["Qwen/Qwen3.5-9B-Base"] = replace(
         max_cpu_loras=16,
         max_lora_rank=128,
         max_model_len=262144,
-        max_num_seqs=8,
+        max_num_seqs=32,
         tensor_parallel_size=1,
         gpu_memory_utilization=0.90,
         image_limit=MAX_IMAGES_PER_CONVERSATION,

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/custom/controller.py

@@ -58,9 +58,28 @@ def submit_custom(api, payload, *, client_train_schema=None):
         ) from error
 
 
-def connect(api, run_id, *, timeout=1200):
-    deadline = time.monotonic() + timeout
-    while time.monotonic() < deadline:
+_ATTACH_RETRY_MAX_SECONDS = 30.0
+
+
+def _transient_attach_error(exc):
+    """Whether a failed attach lookup can be retried rather than ending the run.
+
+    425 is the documented "GPU still provisioning" answer. A 5xx or a transport failure (no
+    status) means the control plane could not answer this time -- for example a slow Modal
+    tunnel lookup or an overloaded plane -- not that the run is gone. Treating those as fatal made
+    run_custom cancel healthy runs. Every other 4xx (ended run, auth, ownership) stays fatal.
+    """
+    if isinstance(exc, ApiError):
+        status = exc.status or 0
+        return status == 425 or status >= 500
+    return isinstance(exc, ClientError)
+
+
+def connect(api, run_id, *, timeout=1200, sleep=time.sleep, clock=time.monotonic):
+    deadline = clock() + timeout
+    failures = 0
+    last = None
+    while clock() < deadline:
         try:
             connection = api._request("GET", f"/v1/runs/{run_id}/custom")
             try:
@@ -70,13 +89,26 @@ def connect(api, run_id, *, timeout=1200):
                 client._http.timeout = 300
                 client.run_id = run_id
                 return client
-            except ClientError:
-                pass  # tunnel can exist before model loading finishes.
-        except ApiError as exc:
-            if exc.status != 425:
+            except ClientError as exc:
+                last = exc  # tunnel can exist before model loading finishes.
+            delay = 2.0
+        except ClientError as exc:
+            if not _transient_attach_error(exc):
                 raise
-        time.sleep(2)
-    raise TimeoutError(f"custom run {run_id} did not become ready")
+            last = exc
+            if isinstance(exc, ApiError) and exc.status == 425:
+                delay = 2.0
+            else:
+                failures += 1
+                delay = min(_ATTACH_RETRY_MAX_SECONDS, 2.0 * 2 ** min(failures, 4))
+                if failures == 1:
+                    print(
+                        f"control plane could not attach {run_id} yet ({exc}); retrying",
+                        file=sys.stderr,
+                        flush=True,
+                    )
+        sleep(max(0.0, min(delay, deadline - clock())))
+    raise TimeoutError(f"custom run {run_id} did not become ready ({last})")
 
 
 def run_custom(args, spec, authored_train_keys):

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/custom/session.py

@@ -13,6 +13,41 @@ from flash._internal.diagnostics import sanitize_diagnostic
 from flash.custom.types import integer
 
 
+def _reset_cuda_peak():
+    """Start a fresh per-operation peak; best effort, a no-op without CUDA."""
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            torch.cuda.reset_peak_memory_stats()
+    except Exception:
+        pass
+
+
+def _cuda_memory():
+    """GPU memory at the end of an operation (GiB), or None without CUDA.
+
+    Logged with every operation event so an out-of-memory failure shows whether memory crept up
+    over earlier operations or spiked inside the failing one. Never raises.
+    """
+    try:
+        import torch
+
+        if not torch.cuda.is_available():
+            return None
+        gib = 1024**3
+        free, total = torch.cuda.mem_get_info()
+        return {
+            "allocated_gib": round(torch.cuda.memory_allocated() / gib, 2),
+            "reserved_gib": round(torch.cuda.memory_reserved() / gib, 2),
+            "peak_allocated_gib": round(torch.cuda.max_memory_allocated() / gib, 2),
+            "free_gib": round(free / gib, 2),
+            "total_gib": round(total / gib, 2),
+        }
+    except Exception:
+        return None
+
+
 class SessionConflict(ValueError):
     """The caller must inspect/reconcile session state before doing more work."""
 
@@ -61,6 +96,9 @@ class TrainingSession:
         }
         if result.get("error"):
             event["error"] = result["error"]
+        memory = _cuda_memory()
+        if memory is not None:
+            event["cuda_memory"] = memory
         self.events.append(event)
         # prompts, token ids and credentials never enter worker stdout.
         print(json.dumps({"custom_training": event}, allow_nan=False), flush=True)
@@ -116,6 +154,7 @@ class TrainingSession:
                     "step": self.step,
                 }
             start = time.monotonic()
+            _reset_cuda_peak()
             try:
                 output = self.backend.execute(operation, arguments)
                 # reject invalid numerical results before committing a success receipt.

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/providers/core/base.py

@@ -596,7 +596,9 @@ class JobHandle:
 class PollResult:
     ok: bool
     metrics: dict | None = None
-    # failure: job_failed, oom, job_preempted, no_capacity, stalled, or poll_error.
+    # failure: job_failed, oom, job_preempted, no_capacity, stalled, poll_error, or
+    # checkpoint_upload_failed (training finished; the final CPU checkpoint upload failed after
+    # its own retries, and the committed handoff is preserved).
     failure: str | None = None
     detail: str | None = None
 

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/providers/modal/checkpoint_upload.py

@@ -59,8 +59,20 @@ def cleanup_run(run_id: str) -> None:
         api.call(remove, timeout=api.TERMINATION_TIMEOUT_SECONDS)
 
 
-def _uploader_tags(handle):
-    return {
+# Training is finished and the GPU released by the time the CPU uploader runs, and the handoff
+# Volume keeps the committed checkpoint, so a failed upload is retried on a fresh uploader instead
+# of failing the run (`cpu_handoff` re-validates and re-uploads idempotently). Try 0 keeps the
+# original name and tags so uploads launched by an older control plane still reconcile.
+UPLOAD_TRIES = 3
+_UPLOAD_RETRY_DELAYS_S = (15.0, 60.0)
+
+
+class CheckpointUploadError(api.ModalApiError):
+    """Every CPU uploader try failed; the committed handoff Volume is preserved."""
+
+
+def _uploader_tags(handle, try_index: int = 0):
+    tags = {
         "flash_owner": "flash-training",
         "flash_run": handle.run_id,
         "flash_attempt": str(handle.attempt),
@@ -68,28 +80,51 @@ def _uploader_tags(handle):
         "flash_role": "checkpoint-upload",
         "flash_scope": handle.scope,
     }
+    if try_index:
+        tags["flash_upload_try"] = str(try_index)
+    return tags
 
 
-def _uploader_name(handle):
-    return "upload-" + handle.instance_id
+def _uploader_name(handle, try_index: int = 0):
+    name = "upload-" + handle.instance_id
+    return f"{name}-r{try_index}" if try_index else name
 
 
-async def _find_owned_uploader(modal, client, handle):
+async def _find_owned_uploader(modal, client, handle, try_index: int = 0):
     try:
         sandbox = await modal.Sandbox.from_name.aio(
             api.APP_NAME,
-            _uploader_name(handle),
+            _uploader_name(handle, try_index),
             environment_name=api.environment(),
             client=client,
         )
     except modal.exception.NotFoundError:
         return None
-    if await sandbox.get_tags.aio() != _uploader_tags(handle):
+    if await sandbox.get_tags.aio() != _uploader_tags(handle, try_index):
         raise UnreconciledCreateError("CPU uploader name belongs to another attempt")
     return sandbox
 
 
-def finish(handle, spec, *, deadline_at):
+async def _launched_spec_sha256(modal, client, handle):
+    """Bind publication to the original launch, not a reconstructed polling spec."""
+    sandbox = await modal.Sandbox.from_id.aio(handle.instance_id, client=client)
+    tags = await sandbox.get_tags.aio()
+    expected = {
+        "flash_owner": "flash-training",
+        "flash_run": handle.run_id,
+        "flash_attempt": str(handle.attempt),
+        "flash_name": handle.name,
+        "flash_handoff": volume_name(handle.scope, handle.run_id, handle.attempt),
+    }
+    if any(tags.get(key) != value for key, value in expected.items()):
+        raise api.ModalApiError("CPU checkpoint parent launch identity mismatch")
+    digest = tags.get("flash_spec")
+    if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+        raise api.ModalApiError("CPU checkpoint parent launch has no valid specification digest")
+    return digest
+
+
+def finish(handle, *, deadline_at):
     """The GPU must be stopped and its Volume committed before attaching a CPU reader."""
     from flash.providers._lifecycle.net.worker import worker_image_for_gpu
 
@@ -103,7 +138,6 @@ def finish(handle, spec, *, deadline_at):
     environment = {
         **storage_environment,
         "FLASH_JOB_SPEC_PATH": f"{MOUNT}/spec.json",
-        "FLASH_EXPECTED_SPEC_SHA256": hashlib.sha256(spec.to_json().encode()).hexdigest(),
         "HF_REPO": handle.hf_repo,
         "RUN_ID": handle.run_id,
         "PHASE": handle.phase,
@@ -113,9 +147,22 @@ def finish(handle, spec, *, deadline_at):
         "PYTHONPATH": f"{MOUNT}/source",
     }
 
-    async def upload(modal, client):
-        sandbox = await _find_owned_uploader(modal, client, handle)
+    def uploader(try_index: int):
+        async def upload(modal, client):
+            await _run_uploader(modal, client, try_index)
+
+        return upload
+
+    async def _run_uploader(modal, client, try_index: int):
+        # a finished sandbox from an earlier try (or an earlier control plane) is not relaunched:
+        # its exit code is the result for that try.
+        sandbox = await _find_owned_uploader(modal, client, handle, try_index)
         if sandbox is None:
+            # Recovery reduces gpu.max_wall_seconds on the poll spec. Hashing that spec
+            # rejects the untouched handoff from the original GPU launch on every retry.
+            environment["FLASH_EXPECTED_SPEC_SHA256"] = await _launched_spec_sha256(
+                modal, client, handle
+            )
             app = await modal.App.lookup.aio(
                 api.APP_NAME,
                 environment_name=api.environment(),
@@ -134,8 +181,8 @@ def finish(handle, spec, *, deadline_at):
                     "-m",
                     "flash.engine.worker.io.cpu_handoff",
                     app=app,
-                    name=_uploader_name(handle),
-                    tags=_uploader_tags(handle),
+                    name=_uploader_name(handle, try_index),
+                    tags=_uploader_tags(handle, try_index),
                     image=modal.Image.from_registry(worker_image_for_gpu(handle.gpu)).pip_install(
                         "azure-storage-blob>=12.24,<13", "azure-identity>=1.19,<2"
                     ),
@@ -147,19 +194,42 @@ def finish(handle, spec, *, deadline_at):
                     client=client,
                 )
             except Exception:
-                sandbox = await _find_owned_uploader(modal, client, handle)
+                sandbox = await _find_owned_uploader(modal, client, handle, try_index)
                 if sandbox is None:
                     raise UnreconciledCreateError(
                         "CPU upload launch could not be reconciled"
                     ) from None
         await sandbox.wait.aio()
-        if await sandbox.poll.aio() != 0:
-            raise api.ModalApiError("CPU checkpoint upload failed; prior durable save is preserved")
+        returncode = await sandbox.poll.aio()
+        if returncode != 0:
+            raise api.ModalApiError(
+                f"CPU checkpoint uploader {sandbox.object_id} exited with code {returncode}"
+            )
 
-    api.call(upload, deadline_at=deadline_at, timeout=1900)
+    tries = 0
+    for try_index in range(UPLOAD_TRIES):
+        tries = try_index + 1
+        try:
+            api.call(uploader(try_index), deadline_at=deadline_at, timeout=1900)
+            break
+        except (api.ModalApiError, TimeoutError) as exc:
+            remaining = deadline_at - time.time()
+            if tries >= UPLOAD_TRIES or remaining <= 0:
+                raise CheckpointUploadError(
+                    f"CPU checkpoint upload failed after {tries} "
+                    f"{'try' if tries == 1 else 'tries'}; prior durable save is preserved; {exc}"
+                ) from exc
+            time.sleep(
+                min(
+                    _UPLOAD_RETRY_DELAYS_S[min(try_index, len(_UPLOAD_RETRY_DELAYS_S) - 1)],
+                    remaining,
+                )
+            )
     seconds = time.time() - started
     return {
         "cpu_upload_wall_seconds": seconds,
+        "cpu_upload_tries": tries,
+        # tries run one after another, so wall time already covers every uploader
         "cpu_upload_cost_upper_estimate_usd": seconds
         / 3600
         * (4 * CPU_HOURLY + 8 * MEMORY_GIB_HOURLY),
@@ -170,11 +240,12 @@ def cleanup(handle):
     """Cancel an owned CPU child before deleting its per-attempt storage."""
 
     async def remove(modal, client):
-        sandbox = await _find_owned_uploader(modal, client, handle)
-        if sandbox is not None:
-            await sandbox.terminate.aio(wait=True)
-            if await sandbox.poll.aio() is None:
-                raise api.ModalApiError("CPU uploader termination was not confirmed")
+        for try_index in range(UPLOAD_TRIES):
+            sandbox = await _find_owned_uploader(modal, client, handle, try_index)
+            if sandbox is not None:
+                await sandbox.terminate.aio(wait=True)
+                if await sandbox.poll.aio() is None:
+                    raise api.ModalApiError("CPU uploader termination was not confirmed")
         with contextlib.suppress(modal.exception.NotFoundError):
             await modal.Volume.objects.delete.aio(
                 volume_name(handle.scope, handle.run_id, handle.attempt),

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/providers/modal/jobs.py

@@ -146,11 +146,16 @@ def poll(handle, spec, *, log=None, deadline_at=None):
         from flash.providers.modal import checkpoint_upload
 
         try:
-            notes = checkpoint_upload.finish(h, spec, deadline_at=deadline_at)
-        except (api.ModalApiError, TimeoutError):
-            return PollResult(
-                False, failure="job_preempted", detail="Final CPU checkpoint publication failed"
+            notes = checkpoint_upload.finish(h, deadline_at=deadline_at)
+        except (api.ModalApiError, TimeoutError) as exc:
+            # training completed and the handoff Volume is preserved; this is an upload failure
+            # (finish already retried it), not a preemption -- report it as its own category.
+            detail = (
+                str(exc)
+                if isinstance(exc, checkpoint_upload.CheckpointUploadError)
+                else "Final CPU checkpoint publication failed"
             )
+            return PollResult(False, failure="checkpoint_upload_failed", detail=detail)
         result.metrics.setdefault("notes", {}).update(notes)
     return result
 

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/runner/results/checkpoints.py

@@ -102,7 +102,7 @@ def _checkpoints_from_files(spec: JobSpec, files: list[str]) -> list[dict]:
     return out
 
 
-def list_checkpoints(spec: JobSpec) -> list[dict]:
+def list_checkpoints(spec: JobSpec, *, strict: bool = False) -> list[dict]:
     """Deployable per-step adapter snapshots for ``spec``, ascending by step.
 
     A step is included only if its adapter folder carries BOTH ``adapter_config.json`` AND a
@@ -110,8 +110,12 @@ def list_checkpoints(spec: JobSpec) -> list[dict]:
     entry: ``{"step", "adapter_prefix", "subfolder", "repo_id", "repo_type"}`` where
     ``adapter_prefix`` is the value to hand ``deploy_adapter`` to serve that exact step and
     ``subfolder`` is the full path of the adapter folder in the repo. Returns ``[]`` when the
-    run has no artifact namespace or no published snapshots (older runs, or none saved yet)."""
-    return _checkpoints_from_files(spec, _repo_files(spec, strict=False))
+    run has no artifact namespace or no published snapshots (older runs, or none saved yet).
+
+    ``strict=True`` raises ``CheckpointListingError`` when the artifact listing itself fails,
+    instead of reporting that failure as "no checkpoints". User-facing listing and deploy routes
+    use it so a transient store error is not shown as a run whose checkpoints are gone."""
+    return _checkpoints_from_files(spec, _repo_files(spec, strict=strict))
 
 
 def adapter_artifact_exists(

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/serve/deployment/profiles.py

@@ -206,7 +206,7 @@ _PROFILES: dict[str, ServingProfile] = {
         quantization=None,
         kv_cache_dtype="fp8",
         max_model_len=262144,
-        max_num_seqs=8,
+        max_num_seqs=16,
         max_num_batched_tokens=None,
         max_loras=16,
         max_cpu_loras=16,
@@ -234,7 +234,7 @@ _PROFILES: dict[str, ServingProfile] = {
         quantization=None,
         kv_cache_dtype="fp8",
         max_model_len=262144,
-        max_num_seqs=8,
+        max_num_seqs=32,
         max_num_batched_tokens=None,
         max_loras=16,
         max_cpu_loras=16,

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/serve/runtime/sampling.py

@@ -115,6 +115,12 @@ def _normalize_token_position(
     for candidate_id, candidate in candidates.items():
         if type(candidate_id) is not int:
             raise RuntimeNotReadyError("vllm returned a non-integer logprob token id")
+        if candidate_id != token_id and _is_masked_alternative(candidate):
+            # guided decoding (structured outputs, tool grammars) masks forbidden tokens to -inf,
+            # and vllm can still return them among the top-k alternatives. they have zero
+            # probability, so they are dropped instead of failing the whole request; the
+            # response may then carry fewer than `top_logprobs` alternatives.
+            continue
         record = _normalize_logprob_candidate(candidate_id, candidate)
         normalized.append((candidate_id, record))
         if candidate_id == token_id:
@@ -130,6 +136,12 @@ def _normalize_token_position(
     }
 
 
+def _is_masked_alternative(candidate: Any) -> bool:
+    """A grammar-masked candidate: logprob is exactly -inf (NaN and +inf stay errors)."""
+    value = getattr(candidate, "logprob", None)
+    return type(value) in {int, float} and math.isinf(value) and value < 0
+
+
 def _normalize_logprob_candidate(token_id: int, candidate: Any) -> dict[str, Any]:
     raw_value = getattr(candidate, "logprob", None)
     if type(raw_value) not in {int, float}:

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/server/routes/runs.py

@@ -588,7 +588,14 @@ def run_checkpoints(run_id: str, key: Annotated[dict, Depends(require_key)]):
     from flash.runner.lifecycle.state import _internal_spec_from_status
 
     spec = _internal_spec_from_status(status)
-    checkpoints = checkpoint_service.list_checkpoints(spec)
+    try:
+        checkpoints = checkpoint_service.list_checkpoints(spec, strict=True)
+    except checkpoint_service.CheckpointListingError:
+        # a failed artifact listing is not "this run has no checkpoints" (which reads as lost).
+        raise HTTPException(
+            status_code=503,
+            detail=f"could not list checkpoints for run {run_id} right now; retry shortly",
+        ) from None
     with contextlib.suppress(Exception):
         from flash.server.domain.registry.checkpoints import register_checkpoints_best_effort
 

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/server/routes/serving_revisions.py

@@ -79,7 +79,14 @@ def _resolve_deploy_step(run_id: str, spec, checkpoint_id: object) -> int | None
     step = parsed[1]
     if step is None:
         return None
-    checkpoints = run_checkpoints.list_checkpoints(spec)
+    try:
+        checkpoints = run_checkpoints.list_checkpoints(spec, strict=True)
+    except run_checkpoints.CheckpointListingError:
+        # do not turn a transient listing failure into "no deployable checkpoint (available: none)"
+        raise HTTPException(
+            status_code=503,
+            detail=f"could not list checkpoints for run {run_id} right now; retry shortly",
+        ) from None
     if any(candidate["step"] == step for candidate in checkpoints):
         return step
     available = ", ".join(str(candidate["step"]) for candidate in checkpoints) or "none"

{freesolo_flash_dev-1.4.15 → freesolo_flash_dev-1.4.16}/flash/serving/app/modal_app.py

@@ -148,23 +148,17 @@ def scaledown_window_for(gpu: str) -> int:
 # gpu model engines scale to zero. inference and adapter registration remote calls start the matching
 # parameter-bound engine on demand.
 MIN_CONTAINERS = 0
-# Limit GPU autoscaling to four containers per model, including its active buffer. `base_model` is
+# Limit GPU autoscaling to four containers per model. `base_model` is
 # a modal.parameter(), so each model has its own pool and cannot borrow another model's capacity.
 # Requests queue when all four containers are busy. This applies independently in each environment.
 MAX_CONTAINERS = 4
-# One spare warm container for each autoscaling pool -- every base-model engine AND the cpu router --
-# so a burst past `TARGET_INPUTS` does not pay a full cold boot (420s on L40S, 900s on H100, 1010s on
-# H200) before it can be served. Modal only provisions the buffer while a Function is ACTIVE, so this
-# does not defeat `MIN_CONTAINERS = 0`: idle engines still scale to zero and bill nothing.
-BUFFER_CONTAINERS = 1
-# Concurrent requests packed onto one base-model GPU before Modal autoscales a new (costly) one.
-# A prior real-GPU sweep on A10G/Qwen2.5-1.5B showed vLLM
-# throughput scaling near-linearly with no saturation through 128 concurrent, while TTFT stayed
-# <60 ms — so the old default of 32 left ~2x of each GPU idle. 64 packs ~2.3x the throughput per
-# GPU for ~+14% per-request latency (now blunted by the fp8 KV cache, which is on for every base —
-# see settings.KV_CACHE_DTYPE), roughly halving GPU count for the same load.
-# Per-engine concurrency is sized by _engine_concurrency from each model's max_num_seqs; these are
-# the router/global ceiling. TARGET_INPUTS auto-derives to 48 (= 64*3//4).
+# A spare GPU per active model doubles the allocation for lightly loaded pools, then
+# remains billable through the 35-minute idle window. Scale GPUs from actual demand;
+# retain a cheap CPU-router buffer so admission does not wait on router startup.
+ENGINE_BUFFER_CONTAINERS = 0
+ROUTER_BUFFER_CONTAINERS = 1
+# Router concurrency and the fallback ceiling for engines without a sequence cap.
+# Qwen engines use their own qualified max_num_seqs, not this router capacity.
 MAX_INPUTS = 64
 TARGET_INPUTS = max(1, MAX_INPUTS * 3 // 4)
 
@@ -341,22 +335,18 @@ from flash.serving.src.engine.model_config import (  # noqa: E402
 def _engine_concurrency(base_model: str) -> tuple[int, int]:
     """return Modal input admission aligned with the engine's sequence capacity.
 
-    Modal counts requests while vLLM schedules sequences. The prior 2x buffer admitted 16 requests
-    onto every current 8-sequence engine, so normal ``n=1`` traffic queued half of them inside the
-    container and delayed scale-out until 12 inputs. OpenAI ``n`` can fan one input out to as many as
-    four sequences, but sizing every container for that rare worst case would leave normal traffic
-    underutilizing the GPU and cold-boot expensive replicas prematurely. Cap request admission at the
-    authored sequence capacity instead: current tiers admit 8 and scale out at 6. This deliberately
-    changes Modal's request knobs, not ``max_num_seqs``: raising the engine cap is not allocation-free,
-    and the 35B tier has a documented startup profiling OOM at higher sequence counts. Models without
-    an authored sequence cap retain the global sizing until their real capacity is explicit.
+    Fill the qualified sequence capacity before requesting another GPU. A 75% target
+    provisioned extra replicas while ordinary n=1 batches still had room. Keep the hard
+    request cap at the sequence limit so this does not introduce a hidden vLLM queue.
+    n>1 requests still share the same bounded scheduler capacity. Larger sequence caps
+    need GPU qualification, especially for the memory-bound 35B MoE. Models without an
+    authored cap use the global ceiling.
     """
     configured = engine_overrides_for(base_model).get("max_num_seqs")
     if configured is None:
-        return MAX_INPUTS, TARGET_INPUTS
+        return MAX_INPUTS, MAX_INPUTS
     max_inputs = max(1, min(MAX_INPUTS, int(configured)))
-    target_inputs = max(1, max_inputs * 3 // 4)
-    return max_inputs, target_inputs
+    return max_inputs, max_inputs
 
 
 def _engine_class_name(gpu: str, max_inputs: int) -> str:
@@ -474,7 +464,7 @@ def _build_engine(
         timeout=REQUEST_TIMEOUT_SECONDS,
         min_containers=MIN_CONTAINERS,
         max_containers=MAX_CONTAINERS,
-        buffer_containers=BUFFER_CONTAINERS,
+        buffer_containers=ENGINE_BUFFER_CONTAINERS,
     )(modal.concurrent(max_inputs=max_inputs, target_inputs=target_inputs)(_Engine))
     # Rebind the module name to the decorated handle, matching the normal ``@app.cls class X`` pattern
     # where the module attribute ends up referring to the decorated class.
@@ -498,7 +488,7 @@ def _distinct_engine_keys() -> list[tuple[str, int]]:
 # One engine class per distinct (GPU tier, concurrency) — see _engine_concurrency for why max_inputs
 # is part of the class identity (Modal fixes concurrency per class).
 ENGINE_BY_KEY: dict[tuple[str, int], Any] = {
-    (gpu, mi): _build_engine(gpu, _engine_class_name(gpu, mi), mi, max(1, mi * 3 // 4))
+    (gpu, mi): _build_engine(gpu, _engine_class_name(gpu, mi), mi, mi)
     for (gpu, mi) in _distinct_engine_keys()
 }
 
@@ -874,11 +864,11 @@ def _base_model_records() -> list:
 @app.function(
     secrets=runtime_secrets,
     region="us-east",  # keep admission, private HTTP dispatch, and serving ingress nearby
-    cpu=8,  # exact long-context token counting and concurrent stream framing
-    memory=16384,  # pinned tokenizers plus bounded in-flight prompt state
+    cpu=2,  # HTTP parsing and stream framing; the engine handles tokenization
+    memory=4096,  # reservation, not a hard cap: near-limit request bursts need additional memory
     nonpreemptible=True,  # a CPU-host interruption cannot resume an open token stream
     min_containers=1,  # one warm CPU front door (hardcoded; no deploy-time knob)
-    buffer_containers=BUFFER_CONTAINERS,  # scales out with the engines; see BUFFER_CONTAINERS
+    buffer_containers=ROUTER_BUFFER_CONTAINERS,
     timeout=ROUTER_TIMEOUT_SECONDS,  # must cover engine cold start (see ROUTER_TIMEOUT_SECONDS)
 )
 @modal.concurrent(max_inputs=MAX_INPUTS, target_inputs=TARGET_INPUTS)