flashinfer-python 0.6.16rc3__tar.gz → 0.6.16rc4__tar.gz

{flashinfer_python-0.6.16rc3/flashinfer_python.egg-info → flashinfer_python-0.6.16rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashinfer-python
-Version: 0.6.16rc3
+Version: 0.6.16rc4
 Summary: FlashInfer: Kernel Library for LLM Serving
 Author: FlashInfer team
 License-Expression: Apache-2.0

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/csrc/trtllm_batched_gemm_runner.cu

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#include <algorithm>
 #include <cstring>
 #include <vector>
 
@@ -36,6 +37,49 @@ using namespace batchedGemm::trtllm::gen;
 
 static BatchedGemmInterface::ModuleCache globalTrtllmGenBatchedGemmModuleCache;
 
+namespace {
+
+// The trtllm-gen cubin manifest is a downloaded artifact, so which architectures
+// it actually covers is not knowable at compile time. Encode only the
+// cubin-arch -> SM-version compatibility rules here and let `config.mSm` decide
+// what is available. Unknown cubin families are rejected so that a newly shipped
+// one fails loudly instead of being mis-dispatched onto hardware that cannot run
+// it (see #4107).
+bool isArchCompatible(int smVersion, tg::CudaArch cubinArch) {
+  switch (cubinArch) {
+    case tg::CudaArch::Sm100a:
+      return smVersion == 100;
+    case tg::CudaArch::Sm100f:
+      return smVersion == 100 || smVersion == 103;
+    case tg::CudaArch::Sm103a:
+      return smVersion == 103;
+#ifdef TLLM_RUBIN_FEATURES
+    // CudaArch::Sm107a only exists in the Rubin cubin pin's generated headers,
+    // which is also the only build that defines TLLM_RUBIN_FEATURES. sm107 is
+    // unreachable in the non-Rubin module: get_trtllm_moe_sm100_module() picks
+    // the Rubin variant by device compute capability before this runs.
+    case tg::CudaArch::Sm107a:
+      return smVersion == 107;
+#endif
+    default:
+      return false;
+  }
+}
+
+void checkPassingConfigIndex(std::vector<int64_t> const& passingConfigIndices,
+                             int32_t configIndex) {
+  auto const it = std::find(passingConfigIndices.begin(), passingConfigIndices.end(), configIndex);
+  if (it == passingConfigIndices.end()) {
+    std::ostringstream msg;
+    msg << "Config index " << configIndex
+        << " is not in this runner's compatible config set (device architecture or GEMM options "
+           "mismatch)";
+    FLASHINFER_ERROR(msg.str());
+  }
+}
+
+}  // namespace
+
 std::vector<int64_t> prioritizePredefinedConfigs(
     int m, int n, int k, std::vector<int64_t> const& sortedIndices,
     batchedGemm::batchedGemm::BatchedGemmConfig const* configs) {
@@ -117,9 +161,22 @@ TrtllmGenBatchedGemmRunner::TrtllmGenBatchedGemmRunner(
         if (options.mFusedBiasShuffleMode != mOptions.fusedBiasShuffleMode) continue;
         if (options.mBiasDtype != mOptions.biasDtype) continue;
       }
-      if (mOptions.usePerTokenScaling) {
-        if (options.mTransposeMmaOutput && !options.mUsePerTokenSfB) continue;
-        if (!options.mTransposeMmaOutput && !options.mUsePerTokenSfA) continue;
+      bool const usesPerTokenScaling =
+          options.mTransposeMmaOutput ? options.mUsePerTokenSfB : options.mUsePerTokenSfA;
+      if (mOptions.usePerTokenScaling && !usesPerTokenScaling) continue;
+      // The MoE pipeline allocates and consumes output scaling factors in the
+      // block format's default dtype. Reject cubins that override that
+      // contract, such as bmm_E2m1xFp32_* kernels that emit linear FP32
+      // scaling factors into a buffer sized for E4M3 factors.
+      //
+      // The published cubin packages predate BatchedGemmOptions::mDtypeSfC, so
+      // filter by kernel name instead: the offending kernels encode the linear
+      // FP32 output scale-factor dtype as the "E2m1xFp32" name token (present
+      // only in the sm_107a package). TODO: restore the typed mDtypeSfC check
+      // once packages carrying the field are published and pinned.
+      if (config.mFunctionName != nullptr &&
+          std::strstr(config.mFunctionName, "E2m1xFp32") != nullptr) {
+        continue;
       }
       if (mOptions.usePerChannelScaling) {
         if (options.mTransposeMmaOutput && !options.mUsePerTokenSfA) continue;
@@ -133,23 +190,9 @@ TrtllmGenBatchedGemmRunner::TrtllmGenBatchedGemmRunner(
       if ((int64_t)options.mEltwiseActType != (int64_t)mOptions.eltwiseActType) {
         continue;
       }
-#ifdef TLLM_RUBIN_FEATURES
-      // CudaArch::Sm107a only exists in the Rubin cubin pin's generated
-      // headers (TRTLLM_GEN_BMM_RUBIN). The default pin has no such enumerator,
-      // so this comparison must not be compiled into the non-Rubin module.
-      // sm_version == 107 is unreachable there anyway: the Rubin module is
-      // selected by device compute capability before this runs.
-      if (sm_version == 107) {
-        if (config.mSm != tg::CudaArch::Sm107a) continue;
-      }
-#endif
-      if (sm_version == 103) {
-        if (config.mSm != tg::CudaArch::Sm103a && config.mSm != tg::CudaArch::Sm100f) continue;
-        if (options.mPatchF2fp && config.mSm != tg::CudaArch::Sm103a) continue;
-      }
-      if (sm_version == 100) {
-        if (config.mSm != tg::CudaArch::Sm100a && config.mSm != tg::CudaArch::Sm100f) continue;
-      }
+      if (!isArchCompatible(sm_version, config.mSm)) continue;
+      // Sm100f cubins miss the f2fp patch, so sm103 must fall back to Sm103a for it.
+      if (sm_version == 103 && options.mPatchF2fp && config.mSm != tg::CudaArch::Sm103a) continue;
       if (mOptions.transposeMmaOutput && options.mEpilogueTileM == mOptions.epilogueTileM) {
         // Skip cubins with clusterZ > 1 due to correctness issues described in
         // https://github.com/flashinfer-ai/flashinfer/issues/3197
@@ -159,6 +202,23 @@ TrtllmGenBatchedGemmRunner::TrtllmGenBatchedGemmRunner(
     }
   }
 
+  if (mPassingConfigIndices.empty()) {
+    // Distinguish "this GPU has no cubins at all" from "no cubin matches these GEMM
+    // options". The former is the common failure on unsupported hardware, and the
+    // option dump below would send users looking in entirely the wrong place.
+    bool anyArchCompatible = false;
+    for (size_t i = 0; i < bmm.getNumBatchedGemmConfigs(); ++i) {
+      if (isArchCompatible(sm_version, configs[i].mSm)) {
+        anyArchCompatible = true;
+        break;
+      }
+    }
+    std::ostringstream arch_msg;
+    arch_msg << "The trtllm-gen batched GEMM cubin manifest contains no kernels runnable on sm"
+             << sm_version << "; this backend currently ships cubins for sm100, sm103 and sm107.";
+    FLASHINFER_CHECK(anyArchCompatible, arch_msg.str());
+  }
+
   std::ostringstream error_msg;
   error_msg << "No kernel found for the given options: "
             << "mDtypeA: " << tg::dtypeToString(mOptions.dtypeA)
@@ -181,6 +241,7 @@ TrtllmGenBatchedGemmRunner::TrtllmGenBatchedGemmRunner(
 size_t TrtllmGenBatchedGemmRunner::getWorkspaceSizeInBytes(
     int32_t m, int32_t n, int32_t k, std::vector<int32_t> const& batchedTokens, int32_t numTokens,
     int32_t numBatches, int32_t maxNumCtasInBatchDim, int32_t configIndex) const {
+  checkPassingConfigIndex(mPassingConfigIndices, configIndex);
   BatchedGemmData gemmData{};
   gemmData.mProblemDimensions.mNumBatches = numBatches;
   gemmData.mProblemDimensions.mNumTokens = numTokens;
@@ -224,7 +285,12 @@ void TrtllmGenBatchedGemmRunner::run(
   BatchedGemmData gemmData{};
 
   auto const configs = bmm.getBatchedGemmConfigs();
+  auto const numConfigs = bmm.getNumBatchedGemmConfigs();
 
+  FLASHINFER_CHECK(
+      configIndex >= 0 && static_cast<size_t>(configIndex) < static_cast<size_t>(numConfigs),
+      "Config index", configIndex, "is out of range; the manifest has", numConfigs, "configs");
+  checkPassingConfigIndex(mPassingConfigIndices, configIndex);
   auto const& config = configs[configIndex];
   // printf("running config %d: %s\n", configIndex, config.mFunctionName);
 

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/csrc/trtllm_fused_moe_kernel_launcher.cu

@@ -1104,8 +1104,11 @@ class Fp8PerTensorLauncher : public FusedMoeLauncher {
           static_cast<batchedGemm::gemm::MatrixLayout>(weight_layout),
           // FP8 per-tensor doesn't use Mn-bias (LoRA) cubins.
           /*gemm1BiasType*/ batchedGemm::gemm::BiasType::None,
-          true,  // usePerTokenScalingGemm1. always true for per-tensor fp8 due to llama4 routing
-          false, false, false);
+          // Keep tactic enumeration aligned with prepare_moe_common(). The
+          // one-way cubin filter still admits the exported FP8 FC1 kernels
+          // that optionally consume BF16 routing scales.
+          /*usePerTokenScalingGemm1*/ false,
+          /*usePerTokenScalingGemm2*/ false, false, false);
 
       auto cfgs = moe_runner->getValidConfigIndices(top_k, hidden_size, intermediate_size,
                                                     num_local_experts, num_tokens);
@@ -2134,9 +2137,11 @@ class FP4BlockScaleLauncher : public FusedMoeLauncher {
           /*weight_layout*/ batchedGemm::gemm::MatrixLayout::MajorK,
           // FP4 MoE getValidConfigs doesn't exercise the Mn-bias (LoRA) cubins.
           /*gemm1BiasType*/ batchedGemm::gemm::BiasType::None,
-          // NOTE(siyuan): currently FP4 MoE always apply per-token scaling to both FC1 and FC2.
           /*usePerTokenScalingGemm1*/ use_per_token_scaling,
-          /*usePerTokenScalingGemm2*/ use_per_token_scaling, false, false);
+          // Match prepare_moe_common(): only NVFP4 uses the explicit
+          // per-token scale operand for FC2.
+          /*usePerTokenScalingGemm2*/
+          use_per_token_scaling && dtype_act == btg::Dtype::E2m1, false, false);
 
       auto cfgs = moe_runner->getValidConfigIndices(top_k, hidden_size, intermediate_size,
                                                     num_local_experts, num_tokens);

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/csrc/trtllm_gemm_runner.cu

@@ -16,6 +16,8 @@
 
 #include <cuda.h>
 
+#include <algorithm>
+#include <sstream>
 #include <string>
 
 #include "flashinfer/exception.h"
@@ -33,6 +35,38 @@ static thread_local gemm::gemm::GemmInterface::ModuleCache globalTrtllmGenGemmMo
 
 namespace flashinfer {
 
+namespace {
+
+// The trtllm-gen cubin manifest is a downloaded artifact, so which architectures
+// it actually covers is not knowable at compile time. Encode only the
+// cubin-arch -> SM-version compatibility rules here and let `config.mSm` decide
+// what is available. Unknown cubin families are rejected so that a newly shipped
+// one fails loudly instead of being mis-dispatched onto hardware that cannot run
+// it (see #4107).
+bool isArchCompatible(int smVersion, gemm::trtllm::gen::CudaArch cubinArch) {
+  using CudaArch = gemm::trtllm::gen::CudaArch;
+  switch (cubinArch) {
+    case CudaArch::Sm100a:
+      return smVersion == 100;
+    case CudaArch::Sm100f:
+      return smVersion == 100 || smVersion == 103;
+    case CudaArch::Sm103a:
+      return smVersion == 103;
+#ifdef TLLM_RUBIN_FEATURES
+    // CudaArch::Sm107a only exists in the Rubin cubin pin's generated headers,
+    // which is also the only build that defines TLLM_RUBIN_FEATURES. sm107 is
+    // unreachable in the non-Rubin module: get_trtllm_gemm_module() picks the
+    // Rubin variant by device compute capability before this runs.
+    case CudaArch::Sm107a:
+      return smVersion == 107;
+#endif
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+
 struct TrtllmGenGemmRunnerOptions {
   gemm::trtllm::gen::Dtype eltType;
   gemm::trtllm::gen::Dtype outputType;
@@ -110,6 +144,7 @@ class TrtllmGenGemmRunner {
     auto const configs = gemm.getGemmConfigs();
 
     mPassingConfigIndices.clear();
+    int const sv = getSMVersion();
 
     for (size_t i = 0; i < gemm.getNumGemmConfigs(); ++i) {
       auto const options = configs[i].mOptions;
@@ -118,10 +153,30 @@ class TrtllmGenGemmRunner {
           options.mTransposeMmaOutput == mOptions.transposeMmaOutput &&
           options.mSfLayoutB == mOptions.sfLayoutB &&
           options.mLayoutA == mOptions.layoutA) {  // FIXME(siyuanf): expose matrix layout to user
+        if (!isArchCompatible(sv, configs[i].mSm)) continue;
         mPassingConfigIndices.push_back(i);
       }
     }
 
+    if (mPassingConfigIndices.empty()) {
+      // Distinguish "this GPU has no cubins at all" from "no cubin matches these GEMM
+      // options". The former is the common failure on unsupported hardware, and the
+      // option dump below would send users looking in entirely the wrong place.
+      bool anyArchCompatible = false;
+      for (size_t i = 0; i < gemm.getNumGemmConfigs(); ++i) {
+        if (isArchCompatible(sv, configs[i].mSm)) {
+          anyArchCompatible = true;
+          break;
+        }
+      }
+      if (!anyArchCompatible) {
+        std::ostringstream arch_msg;
+        arch_msg << "The trtllm-gen GEMM cubin manifest contains no kernels runnable on sm" << sv
+                 << "; this backend currently ships cubins for sm100, sm103 and sm107.";
+        FLASHINFER_ERROR(arch_msg.str());
+      }
+    }
+
     FLASHINFER_CHECK(mPassingConfigIndices.size() > 0,
                      "No valid tactic found for the given options",
                      "mDtypeA: ", gemm::trtllm::gen::dtypeToString(mOptions.eltType),
@@ -130,11 +185,20 @@ class TrtllmGenGemmRunner {
                      "mSfLayoutB: ", gemm::trtllm::gen::sfLayoutToString(mOptions.sfLayoutB));
   }
 
+  void checkPassingConfigIndex(int64_t tactic) const {
+    auto it = std::find(mPassingConfigIndices.begin(), mPassingConfigIndices.end(), tactic);
+    TVM_FFI_ICHECK(it != mPassingConfigIndices.end())
+        << "Tactic " << tactic
+        << " is not in this runner's compatible config set (device architecture or GEMM options "
+           "mismatch)";
+  }
+
   int64_t getWorkspaceSizeInBytes(int64_t m, int64_t n, int64_t k, int64_t tactic) {
     auto gemm = gemm::gemm::GemmInterface();
     auto const configs = gemm.getGemmConfigs();
     FLASHINFER_CHECK(tactic >= 0 && tactic < gemm.getNumGemmConfigs(),
                      "Invalid tactic in getWorkspaceSizeInBytes");
+    checkPassingConfigIndex(tactic);
     auto const config = configs[tactic];
 
     gemm::gemm::GemmData gemmData;
@@ -156,6 +220,7 @@ class TrtllmGenGemmRunner {
     auto gemm = gemm::gemm::GemmInterface();
     auto const configs = gemm.getGemmConfigs();
     TVM_FFI_ICHECK(tactic >= 0 && tactic < gemm.getNumGemmConfigs()) << "Invalid tactic id in run";
+    checkPassingConfigIndex(tactic);
     auto const& config = configs[tactic];
     TVM_FFI_ICHECK(config.mOptions.mSfLayoutB == mOptions.sfLayoutB) << "Invalid sf layout in run";
 

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/csrc/xqa/mha.cu

@@ -1579,8 +1579,6 @@ CUBIN_EXPORT __global__
 #endif
         uint32_t* __restrict__ semaphores = nullptr, void* __restrict__ scratch = nullptr) {
 
-  float const qScaleValue = qScalePtr != nullptr ? qScalePtr[0] : qScale;
-  float const kvCacheScaleValue = kvScalePtr != nullptr ? kvScalePtr[0] : kvCacheScale;
   assert(allowMultiBlockMode || gridDim.x == 1);
   bool const isMultiBlock = allowMultiBlockMode && (gridDim.x != 1);
   uint32_t const nbSubSeqPerSeq = allowMultiBlockMode ? gridDim.x : 1;
@@ -1594,37 +1592,7 @@ CUBIN_EXPORT __global__
   SharedMem& smem = *reinterpret_cast<SharedMem*>(&smemByteBuf[0]);
 
   uint32_t const idxReq = blockIdx.z;
-#if SPEC_DEC
-  // Variable query sequence length support.
-  bool const variableQSeqLen = qCuSeqLens != nullptr;
-  uint32_t const actualQSeqLen =
-      variableQSeqLen ? uint32_t(qCuSeqLens[idxReq + 1] - qCuSeqLens[idxReq]) : qSeqLen;
-  // Same as idxReq * qSeqLen if all sequences all the same.
-  // Take different beams as different requests/sequences currently.
-  uint32_t const reqSeqOffset = variableQSeqLen ? uint32_t(qCuSeqLens[idxReq]) : (qSeqLen * idxReq);
-  // A ragged request may have 0 draft tokens (all rejected). It owns no rows
-  // and min(row, actualQSeqLen - 1) would underflow, so exit before any
-  // barrier/semaphore interaction (uniform across the request's CTAs).
-  if (variableQSeqLen && actualQSeqLen == 0) {
-    return;
-  }
-
-  uint32_t const nbVHeads = nbKHeads;
-  uint32_t const nbQHeads = nbKHeads * headGrpSize;
-  uint32_t const nbQHeadTokens = nbQHeads * actualQSeqLen;
-  uint32_t const nbQKVHeads = nbQHeads + nbKHeads + nbVHeads;
-
-  uint32_t const nbTokenBlocksPerGrp = gridDim.y / nbKHeads;
-  uint32_t const idxHeadGrp = blockIdx.y / nbTokenBlocksPerGrp;  // inside one request
-  uint32_t const idxHeadTokenInGrp = (blockIdx.y % nbTokenBlocksPerGrp) * warpTile.y;
-  uint32_t const totalNbHeadTokensInGrp = actualQSeqLen * headGrpSize;
-  uint32_t const nbValidHeadTokens =
-      idxHeadTokenInGrp > totalNbHeadTokensInGrp
-          ? 0u
-          : mha::min(totalNbHeadTokensInGrp - idxHeadTokenInGrp, rowsPerBlock);
-  // Shift the mask ptr by batch_idx.
-  mask += reqSeqOffset * divUp(qSeqLen, 32u);
-#else
+#if !SPEC_DEC
   uint32_t const nbQHeads = nbKHeads * headGrpSize;
 
   uint32_t const idxHeadGrp = blockIdx.y;  // inside one request
@@ -1687,6 +1655,41 @@ CUBIN_EXPORT __global__
   acqBulk();
 #endif
 
+  // The scale tensors and qCuSeqLens may be written by the previous grid;
+  // do not read them before acqBulk().
+  float const qScaleValue = qScalePtr != nullptr ? qScalePtr[0] : qScale;
+  float const kvCacheScaleValue = kvScalePtr != nullptr ? kvScalePtr[0] : kvCacheScale;
+
+#if SPEC_DEC
+  bool const variableQSeqLen = qCuSeqLens != nullptr;
+  uint32_t const actualQSeqLen =
+      variableQSeqLen ? uint32_t(qCuSeqLens[idxReq + 1] - qCuSeqLens[idxReq]) : qSeqLen;
+  // Same as idxReq * qSeqLen if all sequences all the same.
+  // Take different beams as different requests/sequences currently.
+  uint32_t const reqSeqOffset = variableQSeqLen ? uint32_t(qCuSeqLens[idxReq]) : (qSeqLen * idxReq);
+  // A request whose drafts were all rejected has 0 tokens and owns no rows.
+  // The exit is uniform across the request's CTAs.
+  if (variableQSeqLen && actualQSeqLen == 0) {
+    return;
+  }
+
+  uint32_t const nbVHeads = nbKHeads;
+  uint32_t const nbQHeads = nbKHeads * headGrpSize;
+  uint32_t const nbQHeadTokens = nbQHeads * actualQSeqLen;
+  uint32_t const nbQKVHeads = nbQHeads + nbKHeads + nbVHeads;
+
+  uint32_t const nbTokenBlocksPerGrp = gridDim.y / nbKHeads;
+  uint32_t const idxHeadGrp = blockIdx.y / nbTokenBlocksPerGrp;  // inside one request
+  uint32_t const idxHeadTokenInGrp = (blockIdx.y % nbTokenBlocksPerGrp) * warpTile.y;
+  uint32_t const totalNbHeadTokensInGrp = actualQSeqLen * headGrpSize;
+  uint32_t const nbValidHeadTokens =
+      idxHeadTokenInGrp > totalNbHeadTokensInGrp
+          ? 0u
+          : mha::min(totalNbHeadTokensInGrp - idxHeadTokenInGrp, rowsPerBlock);
+  // Shift the mask ptr by batch_idx.
+  mask += reqSeqOffset * divUp(qSeqLen, 32u);
+#endif
+
   constexpr bool qkSwizzle = true;
   // load whole Q heads into shared memory
 #if SPEC_DEC

flashinfer_python-0.6.16rc4/flashinfer/_build_meta.py

@@ -0,0 +1,3 @@
+"""Build metadata for flashinfer package."""
+__version__ = "0.6.16rc4"
+__git_version__ = "eca3683525bbba18b192c86fd0e519094df02254"

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/flashinfer/cute_dsl/utils.py

@@ -95,11 +95,20 @@ def is_cute_dsl_arch_supported(
             return False
         family = _family_fallback_arch(major, minor)
         if family is not None:
-            # Pin the DSL's default compile target before the first kernel
-            # compile; without this the DSL derives the target from the
-            # device and raises KeyError. An explicit user setting wins.
-            os.environ.setdefault("CUTE_DSL_ARCH", family)
-            return True
+            # The device's native arch is absent, but the DSL has the
+            # family-conditional target (e.g. ``sm_100f`` for an sm_107
+            # device); family-portable kernels compile and run correctly
+            # when the DSL targets it. Ground truth is the target the DSL
+            # captured at first ``cutlass`` import (from ``CUTE_DSL_ARCH``),
+            # not ``os.environ`` now: an env var set after that import does
+            # not retarget the DSL, so checking the environment here would
+            # report supported while ``cute.compile`` still fails.
+            target = _dsl_captured_arch()
+            if target is None:
+                # Internal API unavailable: fall back to the env-var proxy.
+                target = os.environ.get("CUTE_DSL_ARCH", "")
+            if target.replace("_", "").lower() == family.replace("_", "").lower():
+                return True
         return False
     except Exception:
         # Arch module layout changed or import failed: fall back to
@@ -120,6 +129,18 @@ def _family_fallback_arch(major: int, minor: int) -> Optional[str]:
         return None
 
 
+def _dsl_captured_arch() -> Optional[str]:
+    r"""Return the compile target the DSL captured when ``cutlass`` was first
+    imported (e.g. ``"sm_100f"`` when ``CUTE_DSL_ARCH`` was exported before
+    the process started), or ``None`` if the internal API is unavailable."""
+    try:
+        from cutlass.cutlass_dsl import CuTeDSL
+
+        return str(CuTeDSL._get_dsl().envar.arch)
+    except Exception:
+        return None
+
+
 def require_cute_dsl_arch(device, native_only: bool = False) -> None:
     r"""Raise :class:`NotImplementedError` when the installed CuTe DSL cannot
     target ``device``'s architecture (see :func:`is_cute_dsl_arch_supported`)."""
@@ -127,8 +148,17 @@ def require_cute_dsl_arch(device, native_only: bool = False) -> None:
 
     major, minor = torch.cuda.get_device_capability(device)
     if not is_cute_dsl_arch_supported(major, minor, native_only=native_only):
+        hint = ""
+        if not native_only:
+            family = _family_fallback_arch(major, minor)
+            if family is not None:
+                hint = (
+                    f"; family-portable CuTe-DSL kernels can run on this device "
+                    f"when CUTE_DSL_ARCH={family} is exported in the environment "
+                    f"before the process starts (see the release notes)"
+                )
         raise NotImplementedError(
-            f"the installed CuTe DSL does not support sm_{major}{minor} on this device"
+            f"the installed CuTe DSL does not support sm_{major}{minor} on this device{hint}"
         )
 
 

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/flashinfer/decode.py

@@ -3577,6 +3577,9 @@ def xqa_batch_decode_with_kv_cache(
     window_left : int = -1
         The left (inclusive) window size for the attention window, when set to ``-1``, the window
         size will be set to the full length of the sequence. Defaults to ``-1``.
+        On SM90 with fp8 KV cache, speculative decode with a non-negative window
+        runs on the generic kernel instead of the Hopper fp8 kernel (see
+        :func:`flashinfer.xqa.xqa`).
 
     out :  Optional[torch.Tensor] = None
         output tensor, if not provided, will be allocated with ``query.dtype``.
@@ -3610,7 +3613,9 @@ def xqa_batch_decode_with_kv_cache(
         ragged Q: requests may have different draft lengths. When given,
         query/out stay packed as [total_q_tokens, num_heads, head_dim],
         q_len_per_req must be the maximum draft length, and mask rows are
-        packed by the same cumulative offsets.
+        packed by the same cumulative offsets. On SM90 with fp8 KV cache,
+        speculative decode runs on the generic kernel instead of the Hopper
+        fp8 kernel (see :func:`flashinfer.xqa.xqa`).
 
     Returns
     -------

{flashinfer_python-0.6.16rc3 → flashinfer_python-0.6.16rc4}/flashinfer/fused_moe/api.py

@@ -223,6 +223,17 @@ class ExecutionConfig:
 # Backend configs — each declares hardware preconditions
 # ---------------------------------------------------------------------------
 
+# Architectures the TRT-LLM routed-MoE cubin manifest ships kernels for.  The
+# manifest is a downloaded artifact, so this cannot be derived at import time and
+# has to be kept in sync with the cubin-arch compatibility rules in
+# csrc/trtllm_batched_gemm_runner.cu.  SM110/120/121 need upstream cubins first;
+# claiming them here makes the batched-GEMM runner abort at dispatch (#4107).
+_TRTLLM_ROUTED_ARCHS = (100, 103, 107)
+
+# The FP8 kernels are validated on the SM100 family only — the outer JIT module
+# compiles for major 12 as well, but those cubins fail at runtime on SM120/121.
+_TRTLLM_ROUTED_FP8_ARCHS = (100, 103)
+
 
 @dataclass(frozen=True)
 class TrtllmFp4Config:
@@ -230,11 +241,7 @@ class TrtllmFp4Config:
 
     @classmethod
     def supported(cls, arch: int) -> bool:
-        # SM100+ only: the routed runner delegates to the trtllm-gen sm100
-        # module, which core.is_trtllm_moe_supported() gates on major >= 10.
-        # Returning True on SM90 would mark the backend available on H100 and
-        # then fail at dispatch.
-        return arch >= 100
+        return arch in _TRTLLM_ROUTED_ARCHS
 
     @staticmethod
     def prepare_weights(
@@ -274,10 +281,7 @@ class TrtllmFp8BlockConfig:
 
     @classmethod
     def supported(cls, arch: int) -> bool:
-        # The available TRTLLM block-FP8 BMM cubins are validated only on the
-        # SM100 family. The outer JIT can compile for major 12, but its FP8
-        # kernels currently fail at runtime on SM120/121.
-        return arch in (100, 103)
+        return arch in _TRTLLM_ROUTED_FP8_ARCHS
 
     @staticmethod
     def prepare_weights(
@@ -327,7 +331,7 @@ class TrtllmFp8PerTensorConfig:
 
     @classmethod
     def supported(cls, arch: int) -> bool:
-        return arch in (100, 103)
+        return arch in _TRTLLM_ROUTED_FP8_ARCHS
 
     @staticmethod
     def prepare_weights(
@@ -375,7 +379,7 @@ class TrtllmBf16Config:
 
     @classmethod
     def supported(cls, arch: int) -> bool:
-        return arch >= 100
+        return arch in _TRTLLM_ROUTED_ARCHS
 
     @staticmethod
     def prepare_weights(
@@ -415,7 +419,11 @@ class TrtllmMxInt4Config:
 
     @classmethod
     def supported(cls, arch: int) -> bool:
-        return arch >= 100
+        # Same trtllm-gen routed batched-GEMM path as the FP4/BF16 backends, so it
+        # inherits the same manifest coverage.  Whether sm107a MxE2m1 cubins exist
+        # is not verifiable from this repo; this only narrows the previous
+        # ``arch >= 100``, leaving SM100/103/107 behaviour unchanged.
+        return arch in _TRTLLM_ROUTED_ARCHS
 
     def __repr__(self) -> str:
         return "TrtllmMxInt4Config()"