mjolnir-cli 0.2.0__py3-none-any.whl

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DSPARK/config.yaml

@@ -0,0 +1,124 @@
+# NVFP4_MJOLNIR_E2M3_DSPARK — EXPERIMENT (opt-in e2e leg, Mjolnir image): NVFP4_MJOLNIR_E2M3
+# (Mjolnir decode kernel, fp6 KV) with the DSpark drafter instead of the built-in MTP head —
+# the drafter on Mjolnir and fp6 KV too. Its 5 layers (indices 64-68) hold a K/V
+# per context token, 20 KB/token on bf16 next to the target's 24 KB on fp6; on
+# fp6 they take 7.5 KB, so the cache holds ~1.4x the tokens of a bf16 drafter.
+#   - the target's verify steps (1 + 7 tokens) and the drafter's non-causal
+#     block steps (1 + 7 queries over the whole context) both run on the decode
+#     kernel (MJOLNIR_ATTN_NONCAUSAL); other steps on FlashAttention 4
+#   - fp6 scales for the target AND the drafter, in one file next to this one —
+#     NVFP4_MJOLNIR_E2M3_DSPARK.kv_scales.json, written by:
+#       mjolnir calibrate --config NVFP4_MJOLNIR_E2M3_DSPARK
+#     (`mjolnir serve` refuses to start this config without it)
+# Measure it next to the headline pair:
+#   mjolnir bench e2e --optional            (all optional legs)
+#   mjolnir bench e2e --only mjolnir --config NVFP4_MJOLNIR_E2M3_DSPARK
+# Untested on Thor before its first run: check the startup log for "fp6_e2m3 KV
+# scales ... for 5 layer(s)" (the drafter's group) and the acceptance rate.
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — DSpark drafter, K=7 ──────────────────────────
+# RadixArk's DSpark drafter (5 dense Qwen3 layers + a Markov head) through the
+# vLLM-ready fork (byte-identical weights; its "architectures" is set to
+# Qwen3DSparkModel, which vLLM 0.31.0 registers natively — no custom image).
+#   num_speculative_tokens 7 — the drafter's trained block (block_size 7), not 8
+#   draft_sample_method probabilistic — greedy drafting measured ~23% slower in
+#     RadixArk's eval
+#   enable_adaptive_verification stays OFF (the default): vLLM rejects it for
+#     Qwen3.8, whose GDN layers cannot trim a verification on device
+# The checkpoint's built-in MTP head is not loaded.
+speculative-config:
+  method: dspark
+  model: Doopeworld/Qwen3.8-27B-DSpark-vLLM
+  num_speculative_tokens: 7
+  draft_sample_method: probabilistic
+  # the drafter on Mjolnir + fp6 KV. Explicit: vLLM gives a DSpark drafter its
+  # own backend choice (auto, unless set here) — it does not inherit the target's
+  attention_backend: MJOLNIR
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8/.env

@@ -0,0 +1,2 @@
+# env for NVFP4_MJOLNIR_E2M3_LMH8 (KEY=VALUE; {configs} = the model's config dir in the container)
+MJOLNIR_LM_HEAD=fp8

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8/config.yaml

@@ -0,0 +1,98 @@
+# NVFP4_MJOLNIR_E2M3_LMH8 — EXPERIMENT (opt-in): NVFP4_MJOLNIR_E2M3 plus an fp8 lm_head (MJOLNIR_LM_HEAD=fp8, NVFP4_MJOLNIR_E2M3_LMH8.env).
+# The YAML is identical to NVFP4_MJOLNIR_E2M3; the switches are env vars in NVFP4_MJOLNIR_E2M3_LMH8.env,
+# which `mjolnir serve` / `bench` / `verify` pass to the container. Check it before using:
+#   mjolnir verify spec --legs mjolnir:NVFP4_MJOLNIR_E2M3,mjolnir:NVFP4_MJOLNIR_E2M3_LMH8 --drafts "3 4"
+#   mjolnir bench ab --backends NVFP4_MJOLNIR_E2M3,NVFP4_MJOLNIR_E2M3_LMH8 --repeat 2
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT32K/.env

@@ -0,0 +1,4 @@
+# env for NVFP4_MJOLNIR_E2M3_LMH8_VT32K (KEY=VALUE; {configs} = the model's config dir in the container)
+MJOLNIR_LM_HEAD=fp8
+MJOLNIR_DRAFT_VOCAB={configs}/draft_vocab.json
+MJOLNIR_DRAFT_VOCAB_SIZE=32768

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT32K/config.yaml

@@ -0,0 +1,98 @@
+# NVFP4_MJOLNIR_E2M3_LMH8_VT32K — EXPERIMENT (opt-in): NVFP4_MJOLNIR_E2M3 plus an fp8 lm_head and the drafter's logits over the 32K most frequent target tokens (both env sidecar knobs).
+# The YAML is identical to NVFP4_MJOLNIR_E2M3; the switches are env vars in NVFP4_MJOLNIR_E2M3_LMH8_VT32K.env,
+# which `mjolnir serve` / `bench` / `verify` pass to the container. Check it before using:
+#   mjolnir verify spec --legs mjolnir:NVFP4_MJOLNIR_E2M3,mjolnir:NVFP4_MJOLNIR_E2M3_LMH8_VT32K --drafts "3 4"
+#   mjolnir bench ab --backends NVFP4_MJOLNIR_E2M3,NVFP4_MJOLNIR_E2M3_LMH8_VT32K --repeat 2
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/.env

@@ -0,0 +1,4 @@
+# env for NVFP4_MJOLNIR_E2M3_LMH8_VT64K (KEY=VALUE; {configs} = the model's config dir in the container)
+MJOLNIR_LM_HEAD=fp8
+MJOLNIR_DRAFT_VOCAB={configs}/draft_vocab.json
+MJOLNIR_DRAFT_VOCAB_SIZE=65536

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/config.yaml

@@ -0,0 +1,102 @@
+# NVFP4_MJOLNIR_E2M3_LMH8_VT64K — EXPERIMENT (opt-in): NVFP4_MJOLNIR_E2M3 plus an fp8 lm_head and the drafter's logits over the 64K most frequent target tokens (both env sidecar knobs).
+# The YAML is identical to NVFP4_MJOLNIR_E2M3; the switches are env vars in NVFP4_MJOLNIR_E2M3_LMH8_VT64K.env,
+# which `mjolnir serve` / `bench` / `verify` pass to the container. Check it before using:
+#   mjolnir verify spec --legs mjolnir:NVFP4_MJOLNIR_E2M3,mjolnir:NVFP4_MJOLNIR_E2M3_LMH8_VT64K --drafts "3 4"
+#   mjolnir bench ab --backends NVFP4_MJOLNIR_E2M3,NVFP4_MJOLNIR_E2M3_LMH8_VT64K --repeat 2
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=5 ─────────────────────
+# 5 drafts: with the fp8 lm_head and the 64K draft vocab a draft is cheap
+# enough that the 5th and 6th pay — best at 4 requests, 1% off best at 1
+# (verify spec 20261014-182829: 34.0 vs 33.8 t/s at 4 requests, 39.4 vs 39.8
+# at 1 request)
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 5
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_VT32K/.env

@@ -0,0 +1,3 @@
+# env for NVFP4_MJOLNIR_E2M3_VT32K (KEY=VALUE; {configs} = the model's config dir in the container)
+MJOLNIR_DRAFT_VOCAB={configs}/draft_vocab.json
+MJOLNIR_DRAFT_VOCAB_SIZE=32768

mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_VT32K/config.yaml

@@ -0,0 +1,98 @@
+# NVFP4_MJOLNIR_E2M3_VT32K — EXPERIMENT (opt-in): NVFP4_MJOLNIR_E2M3 plus the drafter's logits over the 32K most frequent target tokens (VocabTrim; NVFP4_MJOLNIR_E2M3_VT32K.env, draft_vocab.json from `mjolnir calibrate --draft-vocab`).
+# The YAML is identical to NVFP4_MJOLNIR_E2M3; the switches are env vars in NVFP4_MJOLNIR_E2M3_VT32K.env,
+# which `mjolnir serve` / `bench` / `verify` pass to the container. Check it before using:
+#   mjolnir verify spec --legs mjolnir:NVFP4_MJOLNIR_E2M3,mjolnir:NVFP4_MJOLNIR_E2M3_VT32K --drafts "3 4"
+#   mjolnir bench ab --backends NVFP4_MJOLNIR_E2M3,NVFP4_MJOLNIR_E2M3_VT32K --repeat 2
+model: QUASAR-QAT/Qwen3.8-27B-QUASAR-NVFP4
+served-model-name: Qwen/Qwen3.8-27B
+
+dtype: bfloat16
+
+performance-mode: throughput
+optimization-level: 3
+compilation-config:
+  cudagraph_num_of_warmups: 5
+  custom_ops:
+    - +rotary_embedding
+  pass_config:
+    eliminate_noops: true
+    fuse_allreduce_rms: false
+
+disable-custom-all-reduce: true
+trust-remote-code: true
+enforce-eager: false
+
+max-num-seqs: 8
+max-model-len: 200K
+max-num-batched-tokens: 16K
+gpu-memory-utilization: 0.80
+
+enable-auto-tool-choice: true
+tool-call-parser: qwen3_coder
+reasoning-parser: qwen3
+
+# ── Attention / linear backends ─────────────────────────────────────────────
+# The 16 full-attention (head_dim=256) layers run the Mjolnir decode kernel
+# (decode steps) on top of FlashAttention 4 (prefill, KV writes, mixed steps):
+# the image resolves FLASH_ATTN to FA4 for MJOLNIR on sm_110 — no
+# flash_attn_version needed here — and FA4's hd256 kernel pins the KV page to
+# 128 tokens. MJOLNIR_ATTN_FALLBACK=FLASHINFER restores the FlashInfer prefill.
+# `mjolnir serve` adds VLLM_KV_CACHE_LAYOUT=HND for MJOLNIR configs: the
+# kernel reads head-major pages 11-17% faster at long context.
+# Kernel knobs: MJOLNIR_ATTN_* env vars (see src/mjolnir/README.md).
+# KV format: fp6 e2m3 (see the KV Cache section).
+attention-backend: MJOLNIR
+enable-flashinfer-autotune: true
+mm-encoder-attn-backend: TORCH_SDPA   # SM110 ViT PTX-crash workaround (vLLM issue #38411)
+linear-backend: auto                  # do NOT hardcode flashinfer-cutlass — that's the RTX5090/SM120 GEMM path, not SM110
+
+# ── KV Cache ─────────────────────────────────────────────────────────────
+# NVFP4_MJOLNIR_E2M3 — the ONLY change vs NVFP4_MJOLNIR: a 6-bit e2m3 KV cache
+# (4 values in 3 bytes, 25% fewer KV bytes than fp8, +33% tokens in the same
+# memory). Mjolnir's decode kernel reads it directly (tcgen05 kind::f8f6f4);
+# prefill / mixed steps expand the pages they read to fp8 for FlashAttention 4.
+# Needs per-(layer, KV head) scales: `mjolnir calibrate` writes kv_scales.json
+# next to this file, and `mjolnir serve` points MJOLNIR_ATTN_KV_SCALES at it.
+kv-cache-dtype: fp6_e2m3
+kv-cache-dtype-skip-layers: sliding_window
+mamba-cache-mode: align
+mamba-cache-dtype: bfloat16
+mamba-ssm-cache-dtype: bfloat16
+
+# ── Multimodal ─────────────────────────────────────────────────────────────
+language-model-only: false
+mm-encoder-tp-mode: data
+mm-processor-cache-type: shm
+# limit-mm-per-prompt:
+#   image: 4
+#   video: 0
+
+# ── Caching & Scheduling ─────────────────────────────────────────────────
+enable-prefix-caching: true
+enable-chunked-prefill: true
+async-scheduling: true
+
+# ── Speculative Decoding — MTP (built-in head), K=3 ─────────────────────
+speculative-config:
+  method: mtp
+  num_speculative_tokens: 3
+
+# ── Generation defaults (Qwen3.8 card's documented Thinking-mode preset) ────
+# NOTE: the verified Thor launch args do not set this server-side (sampling is
+# left to the client/proxy). Included here as the model card's recommendation
+# if you want it pinned at the server instead.
+override-generation-config:
+  temperature: 1.0
+  top_p: 0.95
+  top_k: 20
+  min_p: 0.0
+  presence_penalty: 0.0
+  repetition_penalty: 1.0
+
+# ── Chat Template ──────────────────────────────────────────────────────────
+default-chat-template-kwargs:
+  reasoning_effort: medium
+  preserve_thinking: true
+  enable_thinking: true
+
+chat-template: /configs/Qwen/Qwen3.8_chat_template.jinja