mjolnir-cli 0.2.0__py3-none-any.whl

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_DSPARK/config.yaml

@@ -0,0 +1,102 @@
+# NVFP4_DSPARK — EXPERIMENT (opt-in e2e leg, official image): NVFP4 with the DSpark drafter
+# instead of the built-in MTP head. Stock vLLM 0.31.0: FlashInfer, fp8 KV — what
+# DSpark gives WITHOUT Mjolnir. Measure it next to the headline pair:
+#   mjolnir bench e2e --optional            (all optional legs)
+#   mjolnir bench e2e --only official --official-config NVFP4_DSPARK
+# Untested on Thor before its first run: watch the startup log for the drafter's
+# attention backend and the acceptance rate (`mjolnir verify spec` reads it).
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+attention-backend: flashinfer # TRITON_ATTN
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+kv-cache-dtype: fp8   # NVFP4 KV isn't available on SM110 for this arch; FP8 matches the checkpoint's own design (NVFP4 weights, FP8 KV)
+# The drafter's layers (indices 64-68, after the target's 64) keep bf16 KV:
+# its non-causal block attention then runs on any backend, and the KV it holds
+# is small (5 layers, 8 KV heads).
+kv-cache-dtype-skip-layers: ["sliding_window", "64", "65", "66", "67", "68"]
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — DSpark drafter, K=7 ──────────────────────────
+# RadixArk's DSpark drafter (5 dense Qwen3 layers + a Markov head) through the
+# vLLM-ready fork (byte-identical weights; its "architectures" is set to
+# Qwen3DSparkModel, which vLLM 0.31.0 registers natively — no custom image).
+#   num_speculative_tokens 7 — the drafter's trained block (block_size 7), not 8
+#   draft_sample_method probabilistic — greedy drafting measured ~23% slower in
+#     RadixArk's eval
+#   enable_adaptive_verification stays OFF (the default): vLLM rejects it for
+#     Qwen3.8, whose GDN layers cannot trim a verification on device
+# The checkpoint's built-in MTP head is not loaded.
+speculative-config:
+  method: dspark
+  model: Doopeworld/Qwen3.8-27B-DSpark-vLLM
+  num_speculative_tokens: 7
+  draft_sample_method: probabilistic
+  # the drafter's attention: FlashInfer, like the target. If it fails to boot on
+  # sm_110, try attention_backend: FLASH_ATTN (its layers keep bf16 KV — the skip
+  # list in the KV Cache section).
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA2/config.yaml

@@ -0,0 +1,84 @@
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 10
+max-model-len: 160K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.88
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# NVFP4_FA2 — the stock-vLLM baseline on Thor: FlashAttention 2 with a bf16 KV
+# cache (the third leg of the Mjolnir / FlashInfer / FA2 comparison). The 48
+# GDN (linear-attention) layers are a separate backend and are untouched.
+attention-config:
+  backend: FLASH_ATTN        # the 16 full-attn layers (hd256) -> vLLM's FlashAttention backend
+  flash_attn_version: 2      # FA2 = what stock vLLM 0.31.0 runs on sm_110 (FA2 has no fp8 KV,
+                             # so the KV cache below stays bf16 — 2x the decode bytes)
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+kv-cache-dtype: auto   # bf16: FlashAttention 2 has no fp8 KV path
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA4hd256/config.yaml

@@ -0,0 +1,104 @@
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 10
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.88
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# NVFP4_FA4hd256 — the ONLY change vs NVFP4 is the attention opt-in below.
+#
+# NVFP4 used `attention-backend: flashinfer` (top-level), which forced ALL 16
+# full-attention (head_dim=256) layers onto FlashInfer — FA4 could never serve them
+# because sm_110 defaults to FA2 and the Blackwell hd256 kernel was downgraded on FP8 KV.
+# Here we replace it with an `attention-config` block (the two are mutually exclusive):
+#   - backend: FLASH_ATTN      -> the 16 full-attn layers resolve to the FA backend
+#   - flash_attn_version: 4    -> the explicit FA4 opt-in. This is REQUIRED: without it
+#                                 sm_110 stays FA2 (passive-by-default gate), and the
+#                                 fa4-hd256 FP8 policy probe is unreachable (its grant is
+#                                 gated on fa_version==4). With it + FP8 KV + head 256,
+#                                 the dedicated 2-CTA hd256 kernel is selected and its
+#                                 one-shot probe validates the FP8 descale path on the box.
+# The 48 GDN (linear-attention) layers are a SEPARATE backend (linear-backend / mamba) and
+# are untouched by this — this is why only the full-attn group changes.
+# attention-backend: flashinfer # replaced by attention-config below (mutually exclusive)
+attention-config:
+  backend: FLASH_ATTN        # target's 16 full-attn layers (hd256) -> FA4 (was FlashInfer)
+  flash_attn_version: 4      # explicit FA4 opt-in (sm_110 defaults to FA2 otherwise)
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# kv-cache-dtype: fp8 == fp8_e4m3 (vLLM aliases fp8->fp8_e4m3; both satisfy
+# is_quantized_kv_cache), so the FP8 KV requirement is already met. No change.
+kv-cache-dtype: fp8   # NVFP4 KV isn't available on SM110 for this arch; FP8 matches the checkpoint's own design (NVFP4 weights, FP8 KV)
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+# No global `block-size` and no `mamba-block-size` — same as NVFP4. The FA4 hd256
+# kernel requires page_size==128 (TMA tile_n=128), so its KV group is auto-pinned to
+# 128 by get_supported_kernel_block_sizes when FA4 hd256 is active. The GDN/mamba group
+# resolves its own block size independently (per-group block sizes in the hybrid KV
+# coordinator). No conflicting global size is set, so the two coexist. Verified by the
+# server smoke test's "kv cache group sizes" log line.
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR/config.yaml

@@ -0,0 +1,86 @@
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+kv-cache-dtype: fp8   # the Mjolnir kernel reads e4m3 KV; NVFP4 KV isn't available on SM110 for this arch; FP8 matches the checkpoint's own design (NVFP4 weights, FP8 KV)
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3/config.yaml

@@ -0,0 +1,93 @@
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DFLASH2/config.yaml

@@ -0,0 +1,123 @@
+# NVFP4_MJOLNIR_E2M3_DFLASH2 — EXPERIMENT (opt-in e2e leg, Mjolnir image): NVFP4_MJOLNIR_E2M3
+# (Mjolnir decode kernel, fp6 KV) with the DFlash2 drafter instead of the built-in MTP head.
+# The target's verify steps (1 + 7 tokens) run on Mjolnir; the drafter runs on
+# FlashInfer with bf16 KV. Unlike DSpark it stays there: its 5 layers are
+# sliding-window (2048 tokens), which neither the decode kernel nor the FA4
+# fallback implements — and a 2048-token window caps its KV at ~42 MB per
+# request on bf16, so fp6 would save ~26 MB a request. Measure it next to the
+# headline pair:
+#   mjolnir bench e2e --optional            (all optional legs)
+#   mjolnir bench e2e --only mjolnir --config NVFP4_MJOLNIR_E2M3_DFLASH2
+# Untested on Thor before its first run: if the server refuses fp6 for the
+# drafter's layers, the skip list below did not reach them — serve the fp8
+# variant (copy this file, kv-cache-dtype: fp8) and report the log line.
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+# The drafter's layers (indices 64-68, after the target's 64) keep bf16 KV:
+# kv_scales.json calibrates the target only.
+kv-cache-dtype-skip-layers: ["sliding_window", "64", "65", "66", "67", "68"]
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — DFlash2 drafter, K=7 ─────────────────────────
+# The DFlash2 block-diffusion drafter (5 Qwen3 layers, 2048-token sliding
+# window, non-causal block attention, a grouped local convolution and a
+# top-16 candidate selector) — architecture DFlash2DraftModel, native in vLLM
+# 0.31.0 (vllm-project/vllm#52816) under method "dflash".
+#   num_speculative_tokens 7 — the checkpoint's block size is 8 (1 anchor + 7)
+#   the drafter shares the TARGET's lm_head: keep it unquantized — do not stack
+#     the fp8 lm_head / draft-vocab env knobs (LMH8, VT*) on this config
+#   DFlash2 lives on the V2 model runner (vLLM 0.31's default); if the log says
+#     "Model Runner V2 does not yet support …", the drafter silently degrades to
+#     DFlash1 drafting — check the acceptance rate before trusting a number
+# z-lab/Qwen3.8-27B-DFlash2 mirrors the same checkpoint.
+speculative-config:
+  method: dflash
+  model: incoai/Qwen3.8-27B-DFlash2
+  num_speculative_tokens: 7
+  # the drafter's own attention backend: its sliding-window layers hold bf16 KV
+  # (the skip list in the KV Cache section) — MJOLNIR has no sliding window
+  attention_backend: FLASHINFER
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja