flashinfer-python 0.7.0__py3-none-any.whl → 0.7.0.post1__py3-none-any.whl

flashinfer/_build_meta.py

@@ -1,3 +1,3 @@
 """Build metadata for flashinfer package."""
-__version__ = "0.7.0"
-__git_commit__ = "4d75a33f19aaf48b44d5b1c5dbca33bc1eca5c58"
+__version__ = "0.7.0.post1"
+__git_commit__ = "946200de1ae94fc93fdd0926f0a13afd1fa7f0f1"

flashinfer/aot.py

@@ -128,6 +128,7 @@ from .jit.gemm import (
     gen_gemm_sm100_module_cutlass_nvfp4_svdquant,
     gen_gemm_sm100_module_cutlass_fp8,
     gen_gemm_sm100_module_cutlass_mxfp8,
+    gen_gemm_sm103_module_cutlass_fp4,
     gen_gemm_sm120_module,
     gen_gemm_sm120_module_cutlass_fp4,
     gen_mm_bf16_cublaslt_module,
@@ -257,6 +258,7 @@ def gen_attention(
     use_logits_soft_cap_: List[bool],
     has_sm90: bool,
     has_sm100: bool,
+    has_sm103: bool,
     add_gemma: bool,
     add_oai_oss: bool,
 ) -> Iterator[JitSpec]:
@@ -407,7 +409,7 @@ def gen_attention(
 
     # fmha_cutlass_sm100a
     # NOTE: currently there's only one uri.
-    if has_sm100:
+    if has_sm100 or has_sm103:
         yield gen_fmha_cutlass_sm100a_module(
             dtype_q=torch.bfloat16,
             dtype_kv=torch.bfloat16,
@@ -440,7 +442,7 @@ def gen_attention(
             )
 
     # MLA SM100
-    if has_sm100:
+    if has_sm100 or has_sm103:
         yield gen_mla_module()
 
 
@@ -453,11 +455,12 @@ def gen_xqa(
     use_sliding_window_: List[bool],
     has_sm90: bool,
     has_sm100: bool,
+    has_sm103: bool,
     has_sm120: bool,
     has_sm121: bool,
 ) -> Iterator[JitSpec]:
     """Generate XQA modules for various configurations."""
-    if not has_sm90 and not has_sm100 and not has_sm120 and not has_sm121:
+    if not any((has_sm90, has_sm100, has_sm103, has_sm120, has_sm121)):
         return  # XQA requires SM90+
 
     for (
@@ -578,6 +581,7 @@ def gen_all_modules(
             use_logits_soft_cap_,
             has_sm90,
             has_sm100,
+            has_sm103,
             add_gemma,
             add_oai_oss,
         )
@@ -732,13 +736,14 @@ def gen_all_modules(
             # the fixed E=256/N=512/K=2048 shape (BS8).
             jit_specs.append(gen_monomoe_module())
         if has_sm100:
+            # SM103 registers its own FP4 quantization, CUTLASS fused MoE and FP4 GEMM
+            # below, and takes the SM100f TGV GEMMs instead of these sm_100a ones.
             jit_specs.append(gen_fp4_quantization_sm100_module())
             jit_specs.append(gen_cutlass_fused_moe_sm100_module())
-            jit_specs.append(gen_gemm_sm100_module())
             jit_specs.append(gen_gemm_sm100_module_cutlass_fp4())
-            jit_specs.append(gen_gemm_sm100_module_cutlass_nvfp4_svdquant())
-            jit_specs.append(gen_gemm_sm100_module_cutlass_fp8())
-            jit_specs.append(gen_gemm_sm100_module_cutlass_mxfp8())
+        # Both TGV variants share a module name, so a build that also targets SM103
+        # must keep only the SM100f one, which runs on both.
+        if has_sm100 and not has_sm103:
             # Add TGV GEMM modules for both bf16 and fp16
             jit_specs.append(
                 gen_tgv_gemm_sm10x_module(torch.bfloat16, use_sm_100f=False)
@@ -746,12 +751,18 @@ def gen_all_modules(
             jit_specs.append(
                 gen_tgv_gemm_sm10x_module(torch.float16, use_sm_100f=False)
             )
-            jit_specs.append(gen_mxfp8_quantization_sm100_module())
+        if has_sm100 or has_sm103:
+            # SM103 loads these SM100-named modules too; they build for the targeted SM10x arch.
             jit_specs.append(gen_trtllm_gen_gemm_module())
             jit_specs.append(gen_trtllm_low_latency_gemm_module())
+            jit_specs.append(gen_gemm_sm100_module())
+            jit_specs.append(gen_gemm_sm100_module_cutlass_nvfp4_svdquant())
+            jit_specs.append(gen_gemm_sm100_module_cutlass_fp8())
+            jit_specs.append(gen_gemm_sm100_module_cutlass_mxfp8())
+            jit_specs.append(gen_mxfp8_quantization_sm100_module())
             jit_specs.append(gen_trtllm_gen_fused_moe_sm100_module())
             jit_specs.append(gen_trtllm_gen_routing_module())
-        if has_sm100f:
+        if has_sm100f or has_sm103:
             # Add TGV GEMM modules compiled with SM100f flags for both bf16 and fp16
             jit_specs.append(
                 gen_tgv_gemm_sm10x_module(torch.bfloat16, use_sm_100f=True)
@@ -764,6 +775,7 @@ def gen_all_modules(
             jit_specs.append(gen_fp4_quantization_sm103_module())
             jit_specs.append(gen_cutlass_fused_moe_sm103_module())
             jit_specs.append(gen_cake_fused_moe_warp_decode_module())
+            jit_specs.append(gen_gemm_sm103_module_cutlass_fp4())
         if has_sm107:
             jit_specs.append(gen_fp4_quantization_sm107_module())
             jit_specs.append(gen_trtllm_gen_gemm_module(enable_rubin=True))
@@ -810,12 +822,12 @@ def gen_all_modules(
             or has_sm121
         ):
             jit_specs.append(gen_trtllm_comm_module())
-        if has_sm100:
+        if has_sm100 or has_sm103:
             jit_specs.append(gen_trtllm_mnnvl_comm_module())
             jit_specs.append(gen_moe_alltoall_module())
             # dcp_alltoall: kernel itself supports SM90+, but ptxas 12.6.0 has
             # a known state-space inference bug on cp.async.bulk that aborts
-            # compilation. has_sm100 implies CUDA >= 12.8, which avoids the bug.
+            # compilation. has_sm100/has_sm103 imply CUDA >= 12.8, which avoids the bug.
             # SM90/SM12x users still get this via JIT.
             jit_specs.append(gen_dcp_alltoall_module())
         jit_specs.append(gen_vllm_comm_module())
@@ -837,7 +849,7 @@ def gen_all_modules(
             gen_topk_module(),
         ]
         # Fused RMSNorm+SiLU: pre-compile all LUT configs (SM100+ only)
-        if has_sm100:
+        if has_sm100 or has_sm103:
             for C in _SUPPORTED_C:
                 for tokens in _SUPPORTED_TOKENS:
                     for dtype in ["bf16", "fp8", "nvfp4"]:
@@ -939,7 +951,7 @@ def gen_all_modules(
                         *dtype_combo, dim, dstate, ntokens, cs_dtype, na_dtype
                     )
                 )
-        if has_sm90 or has_sm100:
+        if has_sm90 or has_sm100 or has_sm103:
             jit_specs.append(gen_trtllm_utils_module())
         # FP4 KV cache quantization/dequantization
         jit_specs.append(gen_fp4_kv_dequantization_module())
@@ -966,6 +978,7 @@ def gen_all_modules(
                 use_sliding_window_,
                 has_sm90,
                 has_sm100,
+                has_sm103,
                 has_sm120,
                 has_sm121,
             )

flashinfer/jit/gemm/core.py

@@ -716,6 +716,16 @@ def gen_gemm_sm120_module() -> JitSpec:
     )
 
 
+def _trtllm_gen_gemm_nvcc_flags(enable_rubin: bool) -> list[str]:
+    if enable_rubin:
+        return sm107a_nvcc_flags
+    # The runner has no device code of its own (kernels ship as cubins), so target
+    # the SM10x arch being built; an SM103 build must not embed an sm_100a image.
+    return current_compilation_context.get_nvcc_flags_list(
+        supported_major_versions=[10], map_sm107_to_100f=True
+    )
+
+
 def gen_trtllm_gen_gemm_module(enable_rubin: bool = False) -> JitSpec:
     # Fetch "flashinferMetaInfo.h" from the online kernel cache. This file
     # contains the `tllmGenGemmList` as the list of available kernels online.
@@ -770,7 +780,7 @@ def gen_trtllm_gen_gemm_module(enable_rubin: bool = False) -> JitSpec:
             "-DTLLM_ENABLE_CUDA",
             f'-DTLLM_GEN_GEMM_CUBIN_PATH=\\"{gemm_path}\\"',
         ]
-        + (sm107a_nvcc_flags if enable_rubin else sm100a_nvcc_flags),
+        + _trtllm_gen_gemm_nvcc_flags(enable_rubin),
         extra_include_paths=[
             gen_root,
             jit_env.FLASHINFER_GEN_SRC_DIR,
@@ -953,7 +963,7 @@ def gen_trtllm_low_latency_gemm_module(enable_rubin: bool = False) -> JitSpec:
             "-DTLLM_ENABLE_CUDA",
             f'-DTLLM_GEN_GEMM_CUBIN_PATH=\\"{gemm_path}\\"',
         ]
-        + (sm107a_nvcc_flags if enable_rubin else sm100a_nvcc_flags),
+        + _trtllm_gen_gemm_nvcc_flags(enable_rubin),
         extra_include_paths=[
             gen_root,
             jit_env.FLASHINFER_GEN_SRC_DIR,

{flashinfer_python-0.7.0.dist-info → flashinfer_python-0.7.0.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashinfer-python
-Version: 0.7.0
+Version: 0.7.0.post1
 Summary: FlashInfer: Kernel Library for LLM Serving
 Author: FlashInfer team
 License-Expression: Apache-2.0

{flashinfer_python-0.7.0.dist-info → flashinfer_python-0.7.0.post1.dist-info}/RECORD

@@ -2,9 +2,9 @@ build_backend.py,sha256=4CRGt9gnaCXVdhsgn0bTnqCcSl6dt-l9xkFlTzyuYxc,47474
 build_utils.py,sha256=oFHo3_xPsS-drjyMNJBPfsP8crP8LosqJJqQrWxBs7E,3576
 flashinfer/__init__.py,sha256=O1VwXJUYhwvXBXjB-tIGjRINTxL9LRwHNCPIEqOtK4w,16516
 flashinfer/__main__.py,sha256=NRaJY2e95yJXZ9gl5Q-w1Xa8dvrC-Dmbkszz7IgoVJY,38407
-flashinfer/_build_meta.py,sha256=2AokBmOL_ioBE_Dh0xIPN_dHgNyQ0Ht0C7KMd8HzI-g,127
+flashinfer/_build_meta.py,sha256=PsNph7w4FQAxKc_p6Zpp2fp2sLc_xAcX_fDNezjfklI,133
 flashinfer/activation.py,sha256=3zvIaz7oq1Dx5_ProkbSawdPxylBrRM5phJ57rV-uho,8164
-flashinfer/aot.py,sha256=-eZH2Sv1pkcxUHz9gNRa985rLWOf7n7Q8ftZdhULtpE,50146
+flashinfer/aot.py,sha256=gvMi06zmBVU6edX0rjroafwnCw2vpL438VexvUoKIak,50913
 flashinfer/api_logging.py,sha256=5NX5OcGneTEKWw5VDNPUOqDPaE4nIC0KwD5NtlT32w8,109502
 flashinfer/artifacts.py,sha256=UeeUn7bZjVVI52FD5CLb4wkvARKDXEc24oslDMqoT6s,13978
 flashinfer/autotune_cache.py,sha256=ydOH4eXSK_xcTmAdqO4jWGscPLcMxFdbLpIodKUR3WI,31425
@@ -6788,7 +6788,7 @@ flashinfer/jit/attention/fmha_v2/generate_kernels.py,sha256=2_erMYw_s89A6v5MvHQI
 flashinfer/jit/attention/fmha_v2/generator_utils.py,sha256=mSIILmzSkcWDh-0J5PuVbedscgIJWV39pnYb9g_LSDo,235441
 flashinfer/jit/attention/fmha_v2/utils.py,sha256=7YCKF2_P14H9TH57kTK6r1_xi-8V279C5Eopcoqur14,35766
 flashinfer/jit/gemm/__init__.py,sha256=82s4SwTJREnO6fkjFYHCoWNP6tuL0ESNKPNnHO5ROxc,1958
-flashinfer/jit/gemm/core.py,sha256=Q5-1MB9YH1mUvgZQSUDV33pSmBveO0wxRd5B5x_4AYI,34120
+flashinfer/jit/gemm/core.py,sha256=6OZoW_9xogXHWi3O_WZVAkTzw0JJWoJBRkijAKFO6l8,34501
 flashinfer/jit/gemm/deepgemm.py,sha256=2NWIiV1MJF7DeStouleTg5Lzx-WwN7MRw1SRuosyXVc,1023
 flashinfer/jit/gemm/fp8_blockscale.py,sha256=ThULUDOn-VJ3FT5QgoiIpqAx3j2KxNz51Yc74ij0qdw,2207
 flashinfer/jit/gemm/cutlass/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -7292,9 +7292,9 @@ flashinfer/triton/kernels/sm_constraint_gemm.py,sha256=_sMXOFtbFve3VGYUj2De5HnVq
 flashinfer/triton/kernels/ssd_chunk_state.py,sha256=SLS-MU-5wDnEx_EqQe8qPZFjxYU8vHtF_BWWomIS3BE,7351
 flashinfer/tuning_configs/v0_1_trtllm_fused_moe_NVIDIA_B200.py,sha256=UxR6bSyyacbdP03-gHampKfA4bnYbVeu2u6nrDv8YQc,8173
 flashinfer/tuning_configs/v0_1_trtllm_fused_moe_NVIDIA_GB200.py,sha256=JFBj1XfiLhghAGpl8aY4KSYFXxWZguuiPX-xf-vnBFg,4879
-flashinfer_python-0.7.0.dist-info/licenses/LICENSE,sha256=y2fCJPUD4KBjkIlQsS-JpygMblJ9z_rJcqoRTkvzxd4,12332
-flashinfer_python-0.7.0.dist-info/METADATA,sha256=IgYcgkhgo9VHaJN_DIbbWGeG8FhaXUn7Kw-am8u-Eok,11859
-flashinfer_python-0.7.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-flashinfer_python-0.7.0.dist-info/entry_points.txt,sha256=XEl-3MAg0fpIpkyEXNS01Mej9-xTQZJ-HZK8a15Eenw,55
-flashinfer_python-0.7.0.dist-info/top_level.txt,sha256=HX9X5yOVTUf4kO3Z_gb2iQGomGD8RJpDsJ-74laGZ-E,37
-flashinfer_python-0.7.0.dist-info/RECORD,,
+flashinfer_python-0.7.0.post1.dist-info/licenses/LICENSE,sha256=y2fCJPUD4KBjkIlQsS-JpygMblJ9z_rJcqoRTkvzxd4,12332
+flashinfer_python-0.7.0.post1.dist-info/METADATA,sha256=khk4kuKTfG8rt0Bkr1IhDYD0ooag8aXHtq0P-hUyzIo,11865
+flashinfer_python-0.7.0.post1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+flashinfer_python-0.7.0.post1.dist-info/entry_points.txt,sha256=XEl-3MAg0fpIpkyEXNS01Mej9-xTQZJ-HZK8a15Eenw,55
+flashinfer_python-0.7.0.post1.dist-info/top_level.txt,sha256=HX9X5yOVTUf4kO3Z_gb2iQGomGD8RJpDsJ-74laGZ-E,37
+flashinfer_python-0.7.0.post1.dist-info/RECORD,,