sie-server 0.7.2__tar.gz → 0.8.0__tar.gz

{sie_server-0.7.2 → sie_server-0.8.0}/.gitignore

@@ -1,4 +1,6 @@
 # Byte-compiled / optimized / DLL files
+.codex/artifacts/worktree-task/
+
 __pycache__/
 *.py[codz]
 *$py.class

{sie_server-0.7.2 → sie_server-0.8.0}/Dockerfile.cpu

@@ -101,8 +101,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # shared-deps resolver.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path. Content is identical across bundles.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed, then move every discovered bundle path ahead of the shared venv so
+# bundle versions shadow it. Content is identical across bundles.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # .venv cleanup + stdlib trim — runs here so /app/.venv reaches its final

{sie_server-0.7.2 → sie_server-0.8.0}/Dockerfile.cuda12

@@ -110,12 +110,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # intentionally not imported here.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path via a .pth file executed by site.py at
-# interpreter startup. `sys.path.insert(0, ...)` puts bundle-libs at
-# position 0 so bundle-specific versions SHADOW shared venv versions
-# (required for the transformers5 bundle). Content is identical for every
-# bundle, so writing it here keeps the shared venv layer byte-identical.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed at interpreter startup. Then move every discovered bundle path
+# ahead of the shared venv so bundle-specific versions SHADOW it (required for
+# the transformers5 bundle). Content is identical for every bundle, so writing
+# it here keeps the shared venv layer byte-identical.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # Conservative venv cleanup — only things known to be runtime-safe. Keep
@@ -210,10 +210,24 @@ RUN set -eux; \
 # smaller CUDA base runtime; SGLang-family bundles need the devel toolkit
 # because flashinfer/tvm_ffi perform runtime JIT through nvcc on first decode.
 FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS runtime-default
+# The CTranslate2 bundle uses the CUDA runtime image, not the devel toolchain.
+FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS runtime-ctranslate2
 FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS runtime-transformers5
 FROM nvidia/cuda:12.9.1-cudnn-devel-ubuntu22.04 AS runtime-sglang
 ENV CUDA_HOME=/usr/local/cuda \
     LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH"
+# FFmpeg shared libraries: SGLang decodes video inputs with torchcodec, which
+# dlopens them at first use and has no bundled FFmpeg.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libavcodec58 \
+    libavdevice58 \
+    libavfilter7 \
+    libavformat58 \
+    libavutil56 \
+    libswresample3 \
+    libswscale5
 FROM runtime-sglang AS runtime-sglang-embedding
 FROM runtime-sglang AS runtime-sglang-vision-extract
 # NOTE: the sglang-cu130 bundle (CUDA 13 / sglang 0.5.13 / torch 2.11 cu130) is built by

{sie_server-0.7.2 → sie_server-0.8.0}/Dockerfile.cuda13

@@ -1,14 +1,12 @@
 # syntax=docker/dockerfile:1
-# SIE Server - CUDA 13 Image (SGLang cu130 bundles)
+# SIE Server - CUDA 13 Image (engine-isolated cu130 bundles)
 # Build from repo root:
 #   docker build -f packages/sie_server/Dockerfile.cuda13 --build-arg BUNDLE=sglang-cu130 -t sie-server:cuda13-sglang-cu130 .
 #
-# WHY A SEPARATE PLATFORM: the sglang-cu130 bundle needs sglang 0.5.13, which pins
-# torch==2.11.0 (cu130) + transformers 5.8.x and JIT-compiles cu130 kernels
-# (flashinfer / tvm_ffi) through nvcc on first decode — so it requires a CUDA 13
-# devel runtime. Keeping it here (platform ``cuda13``) instead of inside
-# Dockerfile.cuda12 means a ``cuda13-sglang-cu130`` image is honestly labelled CUDA 13,
-# not a CUDA-13 image hiding under a ``cuda12-*`` tag (see #1415).
+# WHY A SEPARATE PLATFORM: CUDA 13 engines carry independent exact Torch,
+# Transformers, FlashInfer, and native-library closures. Keeping them here
+# (platform ``cuda13``) instead of Dockerfile.cuda12 makes image tags honest
+# while still sharing the expensive CUDA 13 base layers.
 #
 # STRUCTURE: mirrors Dockerfile.cuda12's deps→base→builder→runtime stages so the
 # shared logic (standalone Python, editable installs, bundle-libs .pth shadow,
@@ -18,7 +16,7 @@
 # cu130 bundle), so there is no per-bundle runtime indirection.
 #
 # DUAL-TORCH CARRY: the shared base venv still installs torch 2.9 (cu129) from
-# pyproject, and the sglang-cu130 bundle overlays torch 2.11 (cu130) into bundle-libs
+# pyproject, and CUDA 13 bundles overlay torch 2.11 (cu130) into bundle-libs
 # (~+3 GB). Slimming torch out of sie-server core to drop the carry is a tracked
 # follow-up (#1415).
 
@@ -50,13 +48,16 @@ ENV DEBIAN_FRONTEND=noninteractive \
     UV_PYTHON_INSTALL_DIR=/opt/python \
     PIP_DISABLE_PIP_VERSION_CHECK=1
 
-# Minimal apt footprint: just enough for curl + git; no deadsnakes PPA.
+# Minimal apt footprint: curl/git plus the MPI runtime required by
+# TensorRT-LLM's Python import; no OpenMPI development headers are needed.
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
     apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
-    git
+    git \
+    libopenmpi3 \
+    openmpi-bin
 
 ARG UV_VERSION
 RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh \
@@ -117,12 +118,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Sanity-check shared venv imports.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path via a .pth executed by site.py at startup.
-# sys.path.insert(0, ...) puts bundle-libs at position 0 so bundle-specific
-# versions SHADOW the shared venv (the cu130 torch/transformers/sglang
-# overlay). Content is identical for every bundle, so writing it here keeps the
-# shared venv layer byte-identical.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed at interpreter startup. Then move every discovered bundle path
+# ahead of the shared venv so bundle-specific versions SHADOW it (the cu130
+# torch/transformers/engine overlay). Content is identical for every bundle, so
+# writing it here keeps the shared venv layer byte-identical.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # Conservative venv cleanup — only runtime-safe removals. Keep torch/include
@@ -155,7 +156,7 @@ RUN .venv/bin/python -m sie_server.cli resolve-deps \
     && cat /tmp/bundle-requirements.txt
 
 # cu130 bundle deps go into a separate site-packages tree (``bundle-libs``,
-# prepended to sys.path via the .pth above). The sglang-cu130 bundle needs torch 2.11
+# prepended to sys.path via the .pth above). CUDA 13 engines need torch 2.11
 # from the cu130 PyTorch index, where the only ``2.11.0`` is the ``+cu130``
 # local-version wheel; pip's multi-index resolution does not reliably prefer it
 # over PyPI's default-CUDA build, so cuda13 bundles resolve with ``uv`` and
@@ -164,7 +165,7 @@ RUN .venv/bin/python -m sie_server.cli resolve-deps \
 # cu130-index resolution was validated with sglang 0.5.13, torch 2.11, and a
 # cu130 runtime in a fresh GPU environment.
 #
-# COVERAGE NOTE: the cuda13 image CI smoke (.github/workflows/sglang-cu130-image.yml)
+# COVERAGE NOTE: the CUDA 13 image CI callers
 # builds this image and asserts on CPU that the bundle-libs cu130 overlay
 # SHADOWS the base venv (import torch -> 2.11+cu130, transformers 5.x). It does
 # NOT exercise GPU init: the cu12 base venv + cu130 bundle-libs both expose the
@@ -220,6 +221,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 # libnuma1: required by sgl_kernel; import fails with a misleading SM-arch error
 #   without it.
 # libgomp1: torch OpenMP runtime.
+# libopenmpi3/openmpi-bin: TensorRT-LLM imports MPI at runtime; headers are not
+#   required by the qualified wheel closure.
 # libspatialindex-c6 / libgl1 / libglib2.0-0 / libice6 / libsm6 / libx11-6 /
 #   libxcb1 / libxext6: docling/opencv dlopen chain (kept symmetric with the
 #   cuda12 runtime so the same model bundle deps import cleanly).
@@ -235,9 +238,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     libgomp1 \
     libice6 \
     libnuma1 \
+    libopenmpi3 \
     libsm6 \
     libspatialindex-c6 \
     ninja-build \
+    openmpi-bin \
     libx11-6 \
     libxcb1 \
     libxext6
@@ -278,7 +283,8 @@ LABEL org.opencontainers.image.title="SIE Server" \
       org.opencontainers.image.description="Search Inference Engine - ${BUNDLE} bundle (CUDA 13)" \
       sie.bundle="${BUNDLE}"
 
-ENV PATH="/app/.venv/bin:$PATH" \
+# ``uv --target`` writes engine entrypoints beside the package overlay.
+ENV PATH="/app/bundle-libs/bin:/app/.venv/bin:$PATH" \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     HF_HOME=/app/.cache/huggingface \

{sie_server-0.7.2 → sie_server-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sie-server
-Version: 0.7.2
+Version: 0.8.0
 Summary: Search Inference Engine - GPU inference server for search workloads
 License: Apache-2.0
 License-File: LICENSE

{sie_server-0.7.2 → sie_server-0.8.0}/README.md

@@ -39,6 +39,15 @@ pip install sie-server
 sie-server serve --port 8080 --device cuda:0
 ```
 
+### TensorRT-LLM generation
+
+TensorRT-LLM buffers completion token IDs and emits the final text together only
+after generation finishes and the terminal stream is verified, even with
+`stream=True`. This preserves context-sensitive spacing and punctuation and
+removes matched stop sequences consistently from returned text, completion-token
+usage, and logprobs. Long completions delay the first visible text; existing
+request timeouts still apply. Other adapters are unaffected.
+
 ## Configuration
 
 `sie-server` reads its config from `SIE_*` environment variables (Pydantic

sie_server-0.8.0/bundles/ctranslate2.yaml

@@ -0,0 +1,6 @@
+name: ctranslate2
+priority: 19
+adapters:
+- sie_server.adapters.ctranslate2.generation
+deps:
+  ctranslate2: '==4.8.1'

{sie_server-0.7.2 → sie_server-0.8.0}/bundles/sglang-cu130.yaml

@@ -2,8 +2,8 @@ name: sglang-cu130
 priority: 20
 default: false
 # GPU platform this bundle targets — selects packages/sie_server/Dockerfile.cuda13
-# and tags images ``cuda13-sglang-cu130`` (not ``cuda12-*``). Gemma 4 needs sglang
-# 0.5.13 / torch 2.11 cu130 on a CUDA 13 devel runtime (see below). Read by the
+# and tags images ``cuda13-sglang-cu130`` (not ``cuda12-*``). The pinned engine
+# needs torch cu130 on a CUDA 13 devel runtime (see below). Read by the
 # image build. Bundles without this field default to ``cuda12``.
 platform: cuda13
 # General CUDA 13 / cu130 SGLang worker image, introduced for Google Gemma 4
@@ -26,13 +26,31 @@ platform: cuda13
 # JIT kernel (jit_kernel/csrc/elementwise/activation.cuh) fails to compile
 # under strict nvcc — "select_kernel ... expected a type, got
 # ActivationKind::kSiLU" — confirmed broken on BOTH CUDA 12.9 and 13.0. 0.5.13
-# fixes it. Both 0.5.11 and 0.5.13 require torch 2.11 (the torch 2.9->2.11 bump
-# landed at 0.5.11, not 0.5.12), so there is no torch-2.9 path to Gemma 4 —
-# this bundle's image carries torch 2.11 (cu130) on a CUDA 13 base — built by
+# fixed it, and this bundle pinned 0.5.13 until the move to 0.5.20 below.
+#
+# Now pinned to 0.5.20, which is the first release carrying the ``glm5_next``
+# model (python/sglang/srt/models/glm5_next.py, EntryClass
+# ``Glm5NextForConditionalGeneration``). 0.5.13 has no such module, so the
+# architecture cannot load on it at any width. 0.5.20 moves its own exact pins
+# with it: torch 2.11 -> 2.13 (cu130 wheels exist for both), transformers
+# 5.8.1 -> 5.12.1, flashinfer-python 0.6.12 -> 0.6.18, sgl-deep-gemm
+# 0.1.2 -> 0.2.0, sglang-kernel 0.4.3 -> 0.4.7, apache-tvm-ffi 0.1.9 -> 0.1.11,
+# and it adds humming-kernels and quack-kernels. xgrammar stays at 0.2.1.
+# Those versions are sglang's own ``==`` pins, not choices made here.
+#
+# 0.5.20 also renames launch flags SIE uses. ``--mamba-scheduler-strategy`` is
+# now ``--mamba-radix-cache-strategy``; ``--cuda-graph-max-bs`` is split into
+# ``--cuda-graph-max-bs-decode`` and ``--cuda-graph-max-bs-prefill``; and
+# ``--disable-piecewise-cuda-graph`` is gone, the prefill-phase graph it turned
+# off now controlled by ``--disable-prefill-cuda-graph``. The adapters routed
+# here and the profiles that use them carry the new names. The ``sglang``
+# bundle's engine keeps the old ones.
+#
+# The bundle's image carries torch (cu130) on a CUDA 13 base — built by
 # the dedicated ``cuda13`` platform (packages/sie_server/Dockerfile.cuda13),
 # selected via the ``platform: cuda13`` field above. The shared base venv still
-# ships torch 2.9.1, so the cu130 image carries two torches (~+3 GB); slimming
-# torch out of sie-server core would remove that (tracked follow-up in #1415).
+# ships its own torch, so the cu130 image carries two torches (~+3 GB); slimming
+# torch out of sie-server core would remove that.
 #
 # Distinct thin adapter modules scope routing here because bundle compatibility
 # keys on module paths. Gemma models use `...sglang.gemma`; Qwen profiles that
@@ -43,28 +61,27 @@ adapters:
 - sie_server.adapters.sglang.gemma
 - sie_server.adapters.sglang.cuda13
 deps:
-  # sglang 0.5.13 = the validated Gemma 4 engine (0.5.11 has the activation.cuh
-  # JIT bug above). Verified transitive resolution on the Modal L4 cu130 image:
-  #   torch==2.11.0 (cu130), transformers==5.8.1, kernels==0.14.1,
-  #   sgl-deep-gemm==0.1.2, sglang-kernel==0.4.3, and
-  #   flashinfer-python==0.6.12 (the exact SGLang 0.5.13 closure).
+  # sglang 0.5.20 (see the VERSION NOTE above). Verified transitive resolution
+  # on the Modal cu130 image:
+  #   torch==2.13.0 (cu130), transformers==5.12.1, kernels==0.14.1,
+  #   sgl-deep-gemm==0.2.0, sglang-kernel==0.4.7, and
+  #   flashinfer-python==0.6.18 (the exact SGLang 0.5.20 closure).
   # These resolve from the cu130 wheel indexes at build/serve time
   # (download.pytorch.org/whl/cu130 + https://docs.sglang.ai/whl/cu130/),
   # supplied by the build (Dockerfile.cuda13 / Modal image). torch,
-  # transformers, and kernels are intentionally NOT re-pinned here — sglang
-  # 0.5.13's own exact pins drive them. (Pinning kernels<0.13 makes the resolve
-  # unsatisfiable: 0.5.13 requires kernels>=0.14.1,<0.15. sglang declares a
-  # bare unpinned `kernels`, so without 0.5.13's transitive pin uv would grab
-  # the latest, which breaks sglang's import — let sglang constrain it.)
-  sglang: '==0.5.13'
+  # transformers, and kernels are intentionally NOT re-pinned here — sglang's
+  # own exact pins drive them. (sglang declares a bare unpinned `kernels`, so
+  # without its transitive pin uv would grab the latest, which breaks sglang's
+  # import — let sglang constrain it.)
+  sglang: '==0.5.20'
   # Structured-output backend the Gemma model configs select
   # (tasks.generate ... grammar_backend: xgrammar). Pinned to the version
-  # sglang 0.5.13 resolves, to prevent silent drift (mirrors the sglang
-  # bundle's rationale). NOTE: 0.5.13 uses xgrammar 0.2.x, not the 0.1.32 the
-  # sglang bundle pins for the older engine.
+  # sglang 0.5.20 resolves, to prevent silent drift (mirrors the sglang
+  # bundle's rationale). NOTE: this engine uses xgrammar 0.2.x, not the 0.1.32
+  # the sglang bundle pins for the older engine.
   xgrammar: '==0.2.1'
   # XGrammar 0.2.1 imports ``tvm_ffi`` while its package metadata leaves the
-  # dependency open at >=0.1.9. SGLang 0.5.13 pins apache-tvm-ffi==0.1.9
+  # dependency open at >=0.1.9. SGLang 0.5.20 pins apache-tvm-ffi==0.1.11
   # exactly, so declare the same version explicitly: this keeps the import in
   # the image and lets the bundle resolve as one compatible dependency set.
-  apache-tvm-ffi: '==0.1.9'
+  apache-tvm-ffi: '==0.1.11'

sie_server-0.8.0/bundles/tensorrt-llm.yaml

@@ -0,0 +1,39 @@
+name: tensorrt-llm
+priority: 20
+default: false
+# Reusable direct-HF encoder-decoder runtime. This is a CUDA 13 engine lane,
+# not a checkpoint-specific bundle; compatible T5/BART/mBART-family profiles
+# opt in through the generic adapter module below.
+platform: cuda13
+adapters:
+- sie_server.adapters.tensorrt_llm.generation
+deps:
+  # Exact prerelease qualified from tag commit
+  # 1cef02e901be43081b1ba6d4981e94ed3bd9c1e8. Compatibility overlays verify
+  # the complete rc24 source files they patch and refuse any other build.
+  tensorrt-llm: '==1.3.0rc24'
+  # rc24's direct-HF T5 closure. Torch resolves to the +cu130 wheel through
+  # Dockerfile.cuda13's explicit PyTorch index.
+  torch: '==2.11.0'
+  transformers: '==5.5.4'
+  flashinfer-python: '==0.6.16'
+  # Keep the qualified build stable when the image resolver permits prereleases.
+  cuda-tile: '==1.6.0rc8'
+  triton: '==3.6.0'
+  nvidia-cutlass-dsl: '==4.5.0'
+  nvidia-cuda-tileiras: '==13.1.80'
+  nvidia-matmul-heuristics: '==0.1.0.27'
+  # cuda-python 13.3.1 permits cuda-pathfinder~=1.1, while cuda-core and
+  # cuda-bindings only require >=1.4.2. Pin the import-proven rc24 closure so
+  # a newer transitive release cannot silently change the engine image.
+  cuda-pathfinder: '==1.7.0'
+  # rc24 permits an NCCL range; lock the import-proven cu13 build so PyPI and
+  # the cu130 indexes cannot choose a different ABI-compatible-looking wheel.
+  nvidia-nccl-cu13: '==2.28.9'
+  flash-attn-4: '==4.0.0b11'
+  torch-c-dlpack-ext: '==0.1.3'
+  xgrammar: '==0.1.32'
+  apache-tvm-ffi: '==0.1.6'
+  # The published rc24 closure leaves mpmath open; a newer resolution breaks
+  # native import. 1.3.0 is the exact import-proven value.
+  mpmath: '==1.3.0'

sie_server-0.8.0/models/Qwen__Qwen3-VL-8B-Instruct.yaml

@@ -0,0 +1,132 @@
+sie_id: "Qwen/Qwen3-VL-8B-Instruct"
+hf_id: "Qwen/Qwen3-VL-8B-Instruct"
+hf_revision: 0c351dd01ed87e9c1b53cbc748cba10e6187ff3b
+inputs:
+  # Video is served on the direct /v1/chat/completions route as one inline
+  # base64 ``video_url`` per request; the runtime samples frames per the
+  # ``video`` block of ``--mm-process-config`` below.
+  text: true
+  image: true
+  audio: false
+  video: true
+tasks:
+  encode: null
+  score: null
+  extract: null
+  generate:
+    # Native context is 262,144 tokens; the served window is 32K so one
+    # admission fits a 16K-token video plus the model-card VL output length.
+    context_length: 32768
+    # Model card VL recommendation: out_seq_length=16384.
+    max_output_tokens: 16384
+    # Every profile is non-speculative (no MTP/NEXTN head in the checkpoint),
+    # so the grammar-safe fallback is the same launch shape.
+    grammar_profile: no-spec
+    capabilities:
+      grammar: ["json_schema", "regex"]
+      streaming: true
+      # The chat template renders Hermes-style <tool_call>{json}</tool_call>.
+      tools: true
+max_sequence_length: 32768
+# KV-cache math (text_config, bf16): 36 layers x 8 KV heads x head_dim 128
+#   kv_bytes_per_token = 2 (k+v) x 36 x 8 x 128 x 2 B = 147,456 B = 144 KiB
+# RTX PRO 6000 (96 GB) at mem_fraction_static=0.85 with ~17.5 GB weights:
+#   (96 x 0.85 - 17.5) GB / 144 KiB ~= 430,000 tokens theoretical.
+# kv_budget_tokens admits two full 32K-context requests, leaving headroom for
+# vision-encoder activations on image/video prefill.
+profiles:
+  default:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"
+  # RTX PRO 6000 (96 GB, sm_120) shape; identical launch to ``default``.
+  rtx-pro-6000:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"
+  # Grammar-safe fallback; identical launch to ``default``.
+  no-spec:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"

{sie_server-0.7.2 → sie_server-0.8.0}/models/Qwen__Qwen3.6-27B.yaml

@@ -337,7 +337,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"
@@ -373,7 +373,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"
@@ -410,7 +410,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"

{sie_server-0.7.2 → sie_server-0.8.0}/models/Qwen__Qwen3.8-27B-FP8.yaml

@@ -20,6 +20,7 @@ tasks:
     # exact-hardware profiles whose admission and launch shapes are smoke-tested.
     context_length: 8192
     max_output_tokens: 4096
+    grammar_profile: no-spec
     capabilities:
       grammar: ["json_schema", "regex"]
       streaming: true
@@ -80,6 +81,9 @@ profiles:
         stop_tokens:
           - "<|im_end|>"
 
+  no-spec:
+    extends: default
+
   # H100-80GB: one native-window admission with in-checkpoint MTP. FA3 plus
   # BF16 GDN state is the measured SM90 batch-one winner. Profile-scoped
   # DeepGEMM fast warmup covers every decode M through 1024 and samples the
@@ -118,7 +122,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -129,7 +133,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -163,7 +167,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -174,7 +178,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
 
   # H200-141GB: deliberately retain the measured H100 SM90 recipe and
@@ -210,7 +214,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -221,7 +225,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -255,7 +259,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -266,7 +270,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
 
   # RTX PRO 6000 96GB (SM120): retain FlashInfer and 2K prefill chunks from the
@@ -304,7 +308,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -313,7 +317,7 @@ profiles:
           - "2048"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -347,7 +351,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -356,5 +360,5 @@ profiles:
           - "2048"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"