sie-server 0.7.3__tar.gz → 0.8.0__tar.gz

{sie_server-0.7.3 → sie_server-0.8.0}/.gitignore

@@ -1,4 +1,6 @@
 # Byte-compiled / optimized / DLL files
+.codex/artifacts/worktree-task/
+
 __pycache__/
 *.py[codz]
 *$py.class

{sie_server-0.7.3 → sie_server-0.8.0}/Dockerfile.cuda12

@@ -216,6 +216,18 @@ FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS runtime-transformers5
 FROM nvidia/cuda:12.9.1-cudnn-devel-ubuntu22.04 AS runtime-sglang
 ENV CUDA_HOME=/usr/local/cuda \
     LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH"
+# FFmpeg shared libraries: SGLang decodes video inputs with torchcodec, which
+# dlopens them at first use and has no bundled FFmpeg.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libavcodec58 \
+    libavdevice58 \
+    libavfilter7 \
+    libavformat58 \
+    libavutil56 \
+    libswresample3 \
+    libswscale5
 FROM runtime-sglang AS runtime-sglang-embedding
 FROM runtime-sglang AS runtime-sglang-vision-extract
 # NOTE: the sglang-cu130 bundle (CUDA 13 / sglang 0.5.13 / torch 2.11 cu130) is built by

{sie_server-0.7.3 → sie_server-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sie-server
-Version: 0.7.3
+Version: 0.8.0
 Summary: Search Inference Engine - GPU inference server for search workloads
 License: Apache-2.0
 License-File: LICENSE

{sie_server-0.7.3 → sie_server-0.8.0}/README.md

@@ -39,6 +39,15 @@ pip install sie-server
 sie-server serve --port 8080 --device cuda:0
 ```
 
+### TensorRT-LLM generation
+
+TensorRT-LLM buffers completion token IDs and emits the final text together only
+after generation finishes and the terminal stream is verified, even with
+`stream=True`. This preserves context-sensitive spacing and punctuation and
+removes matched stop sequences consistently from returned text, completion-token
+usage, and logprobs. Long completions delay the first visible text; existing
+request timeouts still apply. Other adapters are unaffected.
+
 ## Configuration
 
 `sie-server` reads its config from `SIE_*` environment variables (Pydantic

{sie_server-0.7.3 → sie_server-0.8.0}/bundles/sglang-cu130.yaml

@@ -2,8 +2,8 @@ name: sglang-cu130
 priority: 20
 default: false
 # GPU platform this bundle targets — selects packages/sie_server/Dockerfile.cuda13
-# and tags images ``cuda13-sglang-cu130`` (not ``cuda12-*``). Gemma 4 needs sglang
-# 0.5.13 / torch 2.11 cu130 on a CUDA 13 devel runtime (see below). Read by the
+# and tags images ``cuda13-sglang-cu130`` (not ``cuda12-*``). The pinned engine
+# needs torch cu130 on a CUDA 13 devel runtime (see below). Read by the
 # image build. Bundles without this field default to ``cuda12``.
 platform: cuda13
 # General CUDA 13 / cu130 SGLang worker image, introduced for Google Gemma 4
@@ -26,13 +26,31 @@ platform: cuda13
 # JIT kernel (jit_kernel/csrc/elementwise/activation.cuh) fails to compile
 # under strict nvcc — "select_kernel ... expected a type, got
 # ActivationKind::kSiLU" — confirmed broken on BOTH CUDA 12.9 and 13.0. 0.5.13
-# fixes it. Both 0.5.11 and 0.5.13 require torch 2.11 (the torch 2.9->2.11 bump
-# landed at 0.5.11, not 0.5.12), so there is no torch-2.9 path to Gemma 4 —
-# this bundle's image carries torch 2.11 (cu130) on a CUDA 13 base — built by
+# fixed it, and this bundle pinned 0.5.13 until the move to 0.5.20 below.
+#
+# Now pinned to 0.5.20, which is the first release carrying the ``glm5_next``
+# model (python/sglang/srt/models/glm5_next.py, EntryClass
+# ``Glm5NextForConditionalGeneration``). 0.5.13 has no such module, so the
+# architecture cannot load on it at any width. 0.5.20 moves its own exact pins
+# with it: torch 2.11 -> 2.13 (cu130 wheels exist for both), transformers
+# 5.8.1 -> 5.12.1, flashinfer-python 0.6.12 -> 0.6.18, sgl-deep-gemm
+# 0.1.2 -> 0.2.0, sglang-kernel 0.4.3 -> 0.4.7, apache-tvm-ffi 0.1.9 -> 0.1.11,
+# and it adds humming-kernels and quack-kernels. xgrammar stays at 0.2.1.
+# Those versions are sglang's own ``==`` pins, not choices made here.
+#
+# 0.5.20 also renames launch flags SIE uses. ``--mamba-scheduler-strategy`` is
+# now ``--mamba-radix-cache-strategy``; ``--cuda-graph-max-bs`` is split into
+# ``--cuda-graph-max-bs-decode`` and ``--cuda-graph-max-bs-prefill``; and
+# ``--disable-piecewise-cuda-graph`` is gone, the prefill-phase graph it turned
+# off now controlled by ``--disable-prefill-cuda-graph``. The adapters routed
+# here and the profiles that use them carry the new names. The ``sglang``
+# bundle's engine keeps the old ones.
+#
+# The bundle's image carries torch (cu130) on a CUDA 13 base — built by
 # the dedicated ``cuda13`` platform (packages/sie_server/Dockerfile.cuda13),
 # selected via the ``platform: cuda13`` field above. The shared base venv still
-# ships torch 2.9.1, so the cu130 image carries two torches (~+3 GB); slimming
-# torch out of sie-server core would remove that (tracked follow-up in #1415).
+# ships its own torch, so the cu130 image carries two torches (~+3 GB); slimming
+# torch out of sie-server core would remove that.
 #
 # Distinct thin adapter modules scope routing here because bundle compatibility
 # keys on module paths. Gemma models use `...sglang.gemma`; Qwen profiles that
@@ -43,28 +61,27 @@ adapters:
 - sie_server.adapters.sglang.gemma
 - sie_server.adapters.sglang.cuda13
 deps:
-  # sglang 0.5.13 = the validated Gemma 4 engine (0.5.11 has the activation.cuh
-  # JIT bug above). Verified transitive resolution on the Modal L4 cu130 image:
-  #   torch==2.11.0 (cu130), transformers==5.8.1, kernels==0.14.1,
-  #   sgl-deep-gemm==0.1.2, sglang-kernel==0.4.3, and
-  #   flashinfer-python==0.6.12 (the exact SGLang 0.5.13 closure).
+  # sglang 0.5.20 (see the VERSION NOTE above). Verified transitive resolution
+  # on the Modal cu130 image:
+  #   torch==2.13.0 (cu130), transformers==5.12.1, kernels==0.14.1,
+  #   sgl-deep-gemm==0.2.0, sglang-kernel==0.4.7, and
+  #   flashinfer-python==0.6.18 (the exact SGLang 0.5.20 closure).
   # These resolve from the cu130 wheel indexes at build/serve time
   # (download.pytorch.org/whl/cu130 + https://docs.sglang.ai/whl/cu130/),
   # supplied by the build (Dockerfile.cuda13 / Modal image). torch,
-  # transformers, and kernels are intentionally NOT re-pinned here — sglang
-  # 0.5.13's own exact pins drive them. (Pinning kernels<0.13 makes the resolve
-  # unsatisfiable: 0.5.13 requires kernels>=0.14.1,<0.15. sglang declares a
-  # bare unpinned `kernels`, so without 0.5.13's transitive pin uv would grab
-  # the latest, which breaks sglang's import — let sglang constrain it.)
-  sglang: '==0.5.13'
+  # transformers, and kernels are intentionally NOT re-pinned here — sglang's
+  # own exact pins drive them. (sglang declares a bare unpinned `kernels`, so
+  # without its transitive pin uv would grab the latest, which breaks sglang's
+  # import — let sglang constrain it.)
+  sglang: '==0.5.20'
   # Structured-output backend the Gemma model configs select
   # (tasks.generate ... grammar_backend: xgrammar). Pinned to the version
-  # sglang 0.5.13 resolves, to prevent silent drift (mirrors the sglang
-  # bundle's rationale). NOTE: 0.5.13 uses xgrammar 0.2.x, not the 0.1.32 the
-  # sglang bundle pins for the older engine.
+  # sglang 0.5.20 resolves, to prevent silent drift (mirrors the sglang
+  # bundle's rationale). NOTE: this engine uses xgrammar 0.2.x, not the 0.1.32
+  # the sglang bundle pins for the older engine.
   xgrammar: '==0.2.1'
   # XGrammar 0.2.1 imports ``tvm_ffi`` while its package metadata leaves the
-  # dependency open at >=0.1.9. SGLang 0.5.13 pins apache-tvm-ffi==0.1.9
+  # dependency open at >=0.1.9. SGLang 0.5.20 pins apache-tvm-ffi==0.1.11
   # exactly, so declare the same version explicitly: this keeps the import in
   # the image and lets the bundle resolve as one compatible dependency set.
-  apache-tvm-ffi: '==0.1.9'
+  apache-tvm-ffi: '==0.1.11'

{sie_server-0.7.3 → sie_server-0.8.0}/bundles/tensorrt-llm.yaml

@@ -17,6 +17,8 @@ deps:
   torch: '==2.11.0'
   transformers: '==5.5.4'
   flashinfer-python: '==0.6.16'
+  # Keep the qualified build stable when the image resolver permits prereleases.
+  cuda-tile: '==1.6.0rc8'
   triton: '==3.6.0'
   nvidia-cutlass-dsl: '==4.5.0'
   nvidia-cuda-tileiras: '==13.1.80'

sie_server-0.8.0/models/Qwen__Qwen3-VL-8B-Instruct.yaml

@@ -0,0 +1,132 @@
+sie_id: "Qwen/Qwen3-VL-8B-Instruct"
+hf_id: "Qwen/Qwen3-VL-8B-Instruct"
+hf_revision: 0c351dd01ed87e9c1b53cbc748cba10e6187ff3b
+inputs:
+  # Video is served on the direct /v1/chat/completions route as one inline
+  # base64 ``video_url`` per request; the runtime samples frames per the
+  # ``video`` block of ``--mm-process-config`` below.
+  text: true
+  image: true
+  audio: false
+  video: true
+tasks:
+  encode: null
+  score: null
+  extract: null
+  generate:
+    # Native context is 262,144 tokens; the served window is 32K so one
+    # admission fits a 16K-token video plus the model-card VL output length.
+    context_length: 32768
+    # Model card VL recommendation: out_seq_length=16384.
+    max_output_tokens: 16384
+    # Every profile is non-speculative (no MTP/NEXTN head in the checkpoint),
+    # so the grammar-safe fallback is the same launch shape.
+    grammar_profile: no-spec
+    capabilities:
+      grammar: ["json_schema", "regex"]
+      streaming: true
+      # The chat template renders Hermes-style <tool_call>{json}</tool_call>.
+      tools: true
+max_sequence_length: 32768
+# KV-cache math (text_config, bf16): 36 layers x 8 KV heads x head_dim 128
+#   kv_bytes_per_token = 2 (k+v) x 36 x 8 x 128 x 2 B = 147,456 B = 144 KiB
+# RTX PRO 6000 (96 GB) at mem_fraction_static=0.85 with ~17.5 GB weights:
+#   (96 x 0.85 - 17.5) GB / 144 KiB ~= 430,000 tokens theoretical.
+# kv_budget_tokens admits two full 32K-context requests, leaving headroom for
+# vision-encoder activations on image/video prefill.
+profiles:
+  default:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"
+  # RTX PRO 6000 (96 GB, sm_120) shape; identical launch to ``default``.
+  rtx-pro-6000:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"
+  # Grammar-safe fallback; identical launch to ``default``.
+  no-spec:
+    max_batch_tokens: 32768
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    kv_budget_tokens: 65536
+    adapter_options:
+      loadtime:
+        mem_fraction_static: 0.85
+        served_model_name: Qwen/Qwen3-VL-8B-Instruct
+        grammar_backend: xgrammar
+        tool_call_parser: qwen25
+        speculative:
+          enabled: false
+        extra_launch_args:
+          - "--mm-process-config"
+          - '{"image":{"min_pixels":65536,"max_pixels":1003520},"video":{"fps":2,"max_frames":64,"total_pixels":8388608}}'
+      runtime:
+        first_chunk_timeout_s: 90
+        inter_chunk_timeout_s: 15
+        overall_timeout_s: 600
+        # Model card VL sampling: temperature 0.7, top_p 0.8, top_k 20,
+        # presence_penalty 1.5. min_new_tokens=1: on an untemplated native
+        # generate prompt the model emits EOS as its first token (empty text).
+        default_sampling:
+          temperature: 0.7
+          top_p: 0.8
+          top_k: 20
+          presence_penalty: 1.5
+          min_new_tokens: 1
+        stop_tokens:
+          - "<|im_end|>"

{sie_server-0.7.3 → sie_server-0.8.0}/models/Qwen__Qwen3.6-27B.yaml

@@ -337,7 +337,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"
@@ -373,7 +373,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"
@@ -410,7 +410,7 @@ profiles:
           - '{"image":{"min_pixels":65536,"max_pixels":1003520}}'
           - "--quantization"
           - "fp8"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--page-size"
           - "64"

{sie_server-0.7.3 → sie_server-0.8.0}/models/Qwen__Qwen3.8-27B-FP8.yaml

@@ -20,6 +20,7 @@ tasks:
     # exact-hardware profiles whose admission and launch shapes are smoke-tested.
     context_length: 8192
     max_output_tokens: 4096
+    grammar_profile: no-spec
     capabilities:
       grammar: ["json_schema", "regex"]
       streaming: true
@@ -80,6 +81,9 @@ profiles:
         stop_tokens:
           - "<|im_end|>"
 
+  no-spec:
+    extends: default
+
   # H100-80GB: one native-window admission with in-checkpoint MTP. FA3 plus
   # BF16 GDN state is the measured SM90 batch-one winner. Profile-scoped
   # DeepGEMM fast warmup covers every decode M through 1024 and samples the
@@ -118,7 +122,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -129,7 +133,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -163,7 +167,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -174,7 +178,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
 
   # H200-141GB: deliberately retain the measured H100 SM90 recipe and
@@ -210,7 +214,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -221,7 +225,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -255,7 +259,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -266,7 +270,7 @@ profiles:
           - "32768"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
 
   # RTX PRO 6000 96GB (SM120): retain FlashInfer and 2K prefill chunks from the
@@ -304,7 +308,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -313,7 +317,7 @@ profiles:
           - "2048"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"
       runtime:
         first_chunk_timeout_s: 900
@@ -347,7 +351,7 @@ profiles:
           - "bfloat16"
           - "--mamba-ssm-dtype"
           - "bfloat16"
-          - "--mamba-scheduler-strategy"
+          - "--mamba-radix-cache-strategy"
           - "extra_buffer"
           - "--disable-overlap-schedule"
           - "--page-size"
@@ -356,5 +360,5 @@ profiles:
           - "2048"
           - "--max-running-requests"
           - "1"
-          - "--cuda-graph-max-bs"
+          - "--cuda-graph-max-bs-decode"
           - "1"

{sie_server-0.7.3 → sie_server-0.8.0}/models/google__madlad400-3b-mt.yaml

@@ -40,3 +40,5 @@ profiles:
         first_chunk_timeout_s: 180
         inter_chunk_timeout_s: 10
         overall_timeout_s: 600
+        default_sampling:
+          temperature: 0.0

sie_server-0.8.0/models/google__translategemma-4b-it.yaml

@@ -0,0 +1,40 @@
+sie_id: "google/translategemma-4b-it"
+hf_id: "google/translategemma-4b-it"
+hf_revision: 10042cb0e6e7fdce748996a71dc3dc432a4e0c89
+inputs:
+  text: true
+  image: false
+  audio: false
+  video: false
+tasks:
+  encode: null
+  score: null
+  extract: null
+  generate:
+    context_length: 2560
+    max_output_tokens: 512
+    capabilities:
+      grammar: []
+      streaming: true
+      tools: false
+    chat_template_kwargs: {}
+max_sequence_length: 2560
+profiles:
+  default:
+    max_batch_tokens: 8192
+    kv_budget_tokens: 8192
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.generation:SGLangGenerationAdapter
+    adapter_options:
+      loadtime:
+        served_model_name: google/translategemma-4b-it
+        mem_fraction_static: 0.85
+        grammar_backend: xgrammar
+        attention_backend: triton
+        trust_remote_code: false
+      runtime:
+        first_chunk_timeout_s: 180
+        inter_chunk_timeout_s: 10
+        overall_timeout_s: 600
+        stop_tokens:
+          - <end_of_turn>

sie_server-0.8.0/models/zai-org__GLM-5.3-Flash.yaml

@@ -0,0 +1,75 @@
+sie_id: "zai-org/GLM-5.3-Flash"
+hf_id: "zai-org/GLM-5.3-Flash"
+hf_revision: eb9eb208eb0d988989d07a6a12d0fdeb5f52574a
+# Glm5NextForConditionalGeneration: 321B parameters in a 305.8 GiB blockwise
+# FP8 checkpoint (e4m3, 128x128 weight blocks, dynamic activations), which
+# SGLang detects from the checkpoint itself. No transformers release carries
+# glm5_next; the sglang-cu130 engine registers the architecture and its
+# processor. The weights need eight 80 GB accelerators.
+inputs:
+  text: true
+  # The architecture reads images, but the pinned engine resolves transformers
+  # 5.12.1, which has no glm5_next processor. SGLang then falls back to a bare
+  # tokenizer and never turns an image into visual tokens, so an answer about
+  # an image is a guess from the text (sgl-project/sglang#39831). Text only
+  # until the engine moves to transformers 5.16.1 or later
+  # (sgl-project/sglang#39012) and images are measured again.
+  image: false
+  audio: false
+  video: false
+tasks:
+  encode: null
+  score: null
+  extract: null
+  generate:
+    # The checkpoint's native window is 1,048,576 tokens. This route admits
+    # what was measured on eight 80 GB accelerators.
+    context_length: 16384
+    max_output_tokens: 4096
+    # The chat template always opens a thinking block and never reads
+    # enable_thinking. Declaring it true marks that reasoning private: the
+    # queued worker drops everything before </think>, and the direct route
+    # has the engine's glm45 parser split it off.
+    chat_template_kwargs:
+      enable_thinking: true
+    # Grammar and tool calling are not claimed yet. On the queued route the
+    # engine applies a grammar from the first generated token, which is inside
+    # the open thinking block, and the worker does not parse this template's
+    # <arg_key>/<arg_value> tool calls.
+    capabilities:
+      grammar: []
+      streaming: true
+      tools: false
+max_sequence_length: 16384
+profiles:
+  default:
+    max_batch_tokens: 16384
+    kv_budget_tokens: 16384
+    compute_precision: bfloat16
+    adapter_path: sie_server.adapters.sglang.cuda13:SGLangCuda13Adapter
+    adapter_options:
+      loadtime:
+        tensor_parallel_size: 8
+        # The first request after a cold start compiled kernels for 91 s before
+        # its first byte, measured on eight H100s.
+        request_read_timeout_s: 300
+        # Cold starts on eight H100s took from 684 s to about 36 minutes.
+        # Reading the 306 GiB checkpoint took from a few minutes to 25 minutes,
+        # and decode CUDA-graph capture about 7.5 more.
+        startup_timeout_s: 2400
+        max_seq_length: 16384
+        mem_fraction_static: 0.85
+        served_model_name: zai-org/GLM-5.3-Flash
+        trust_remote_code: false
+        grammar_backend: xgrammar
+        reasoning_parser: glm45
+        speculative:
+          enabled: false
+      runtime:
+        first_chunk_timeout_s: 300
+        inter_chunk_timeout_s: 60
+        overall_timeout_s: 600
+        # generation_config.json at the pinned revision.
+        default_sampling:
+          temperature: 1.0
+          top_p: 0.95