flashinfer-python 0.2.1.post2__tar.gz → 0.2.2__tar.gz

{flashinfer_python-0.2.1.post2/flashinfer_python.egg-info → flashinfer_python-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: flashinfer-python
-Version: 0.2.1.post2
+Version: 0.2.2
 Summary: FlashInfer: Kernel Library for LLM Serving
 Author: FlashInfer team
 License: Apache License 2.0
@@ -165,6 +165,7 @@ We are thrilled to share that FlashInfer is being adopted by many cutting-edge p
 - [TGI](https://github.com/huggingface/text-generation-inference)
 - [lorax](https://github.com/predibase/lorax)
 - [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
+- [LightLLM](https://github.com/ModelTC/lightllm)
 
 ## Acknowledgement
 

{flashinfer_python-0.2.1.post2 → flashinfer_python-0.2.2}/README.md

@@ -150,6 +150,7 @@ We are thrilled to share that FlashInfer is being adopted by many cutting-edge p
 - [TGI](https://github.com/huggingface/text-generation-inference)
 - [lorax](https://github.com/predibase/lorax)
 - [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)
+- [LightLLM](https://github.com/ModelTC/lightllm)
 
 ## Acknowledgement
 

flashinfer_python-0.2.2/csrc/aot_default_additional_params.h

@@ -0,0 +1,45 @@
+#define SINGLE_DECODE_ADDITIONAL_FUNC_PARAMS , std::optional<at::Tensor> maybe_alibi_slopes, double logits_soft_cap, double sm_scale, double rope_rcp_scale, double rope_rcp_theta
+
+#define SINGLE_DECODE_ADDITIONAL_PARAMS_SETTER params.maybe_alibi_slopes = maybe_alibi_slopes ? static_cast<float*>(maybe_alibi_slopes->data_ptr()): nullptr; \
+params.logits_soft_cap = logits_soft_cap; \
+params.sm_scale = sm_scale; \
+params.rope_rcp_scale = rope_rcp_scale; \
+params.rope_rcp_theta = rope_rcp_theta;
+
+#define SINGLE_PREFILL_ADDITIONAL_FUNC_PARAMS , std::optional<at::Tensor> maybe_custom_mask, std::optional<at::Tensor> maybe_alibi_slopes, double logits_soft_cap, double sm_scale, double rope_rcp_scale, double rope_rcp_theta
+
+#define SINGLE_PREFILL_ADDITIONAL_PARAMS_SETTER params.maybe_custom_mask = maybe_custom_mask ? static_cast<uint8_t*>(maybe_custom_mask->data_ptr()): nullptr; \
+params.maybe_alibi_slopes = maybe_alibi_slopes ? static_cast<float*>(maybe_alibi_slopes->data_ptr()): nullptr; \
+params.logits_soft_cap = logits_soft_cap; \
+params.sm_scale = sm_scale; \
+params.rope_rcp_scale = rope_rcp_scale; \
+params.rope_rcp_theta = rope_rcp_theta;
+
+#define SINGLE_PREFILL_SM90_ADDITIONAL_FUNC_PARAMS , double logits_soft_cap, double sm_scale
+
+#define SINGLE_PREFILL_SM90_ADDITIONAL_PARAMS_SETTER params.additional_params.logits_soft_cap = logits_soft_cap; \
+params.additional_params.sm_scale = sm_scale;
+
+#define BATCH_DECODE_ADDITIONAL_FUNC_PARAMS , std::optional<at::Tensor> maybe_alibi_slopes, double logits_soft_cap, double sm_scale, double rope_rcp_scale, double rope_rcp_theta
+
+#define BATCH_DECODE_ADDITIONAL_PARAMS_SETTER params.maybe_alibi_slopes = maybe_alibi_slopes ? static_cast<float*>(maybe_alibi_slopes->data_ptr()): nullptr; \
+params.logits_soft_cap = logits_soft_cap; \
+params.sm_scale = sm_scale; \
+params.rope_rcp_scale = rope_rcp_scale; \
+params.rope_rcp_theta = rope_rcp_theta;
+
+#define BATCH_PREFILL_ADDITIONAL_FUNC_PARAMS , std::optional<at::Tensor> maybe_custom_mask, std::optional<at::Tensor> maybe_mask_indptr, std::optional<at::Tensor> maybe_alibi_slopes, double logits_soft_cap, double sm_scale, double rope_rcp_scale, double rope_rcp_theta
+
+#define BATCH_PREFILL_ADDITIONAL_PARAMS_SETTER params.maybe_custom_mask = maybe_custom_mask ? static_cast<uint8_t*>(maybe_custom_mask->data_ptr()): nullptr; \
+params.maybe_mask_indptr = maybe_mask_indptr ? static_cast<int32_t*>(maybe_mask_indptr->data_ptr()): nullptr; \
+params.maybe_alibi_slopes = maybe_alibi_slopes ? static_cast<float*>(maybe_alibi_slopes->data_ptr()): nullptr; \
+params.logits_soft_cap = logits_soft_cap; \
+params.sm_scale = sm_scale; \
+params.rope_rcp_scale = rope_rcp_scale; \
+params.rope_rcp_theta = rope_rcp_theta;
+
+#define BATCH_PREFILL_SM90_ADDITIONAL_FUNC_PARAMS , double logits_soft_cap, double sm_scale
+
+#define BATCH_PREFILL_SM90_ADDITIONAL_PARAMS_SETTER params.additional_params.logits_soft_cap = logits_soft_cap; \
+params.additional_params.sm_scale = sm_scale;
+

{flashinfer_python-0.2.1.post2 → flashinfer_python-0.2.2}/csrc/batch_mla_run.cu

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include <flashinfer/attention/mla_fa2.cuh>
+#include <flashinfer/attention/mla.cuh>
 #include <flashinfer/attention/scheduler.cuh>
 #include <flashinfer/fastdiv.cuh>
 #include <optional>

flashinfer_python-0.2.2/csrc/batch_mla_sm90_plan.cu

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2025 by FlashInfer team.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <flashinfer/attention/scheduler.cuh>
+#include <optional>
+
+#include "batch_mla_sm90_config.inc"
+#include "pytorch_conversion_utils.h"
+#include "pytorch_extension_utils.h"
+
+using namespace flashinfer;
+
+at::Tensor BatchMLAPagedAttentionSM90Plan(at::Tensor float_workspace_buffer,
+                                          at::Tensor int_workspace_buffer,
+                                          at::Tensor page_locked_int_workspace_buffer,
+                                          at::Tensor qo_indptr, at::Tensor kv_indptr,
+                                          at::Tensor kv_len, int64_t num_heads, int64_t head_dim_o,
+                                          bool causal, int64_t cuda_stream) {
+  size_t float_workspace_size_in_bytes =
+      float_workspace_buffer.size(0) * float_workspace_buffer.element_size();
+  size_t int_workspace_size_in_bytes =
+      int_workspace_buffer.size(0) * int_workspace_buffer.element_size();
+
+  MLAPlanInfo plan_info;
+
+  int batch_size = kv_len.size(0);
+
+  cudaStream_t stream = reinterpret_cast<cudaStream_t>(cuda_stream);
+  cudaError_t status =
+      MLAPlan(float_workspace_buffer.data_ptr(), float_workspace_size_in_bytes,
+              int_workspace_buffer.data_ptr(), page_locked_int_workspace_buffer.data_ptr(),
+              int_workspace_size_in_bytes, plan_info, static_cast<IdType*>(qo_indptr.data_ptr()),
+              static_cast<IdType*>(kv_indptr.data_ptr()), static_cast<IdType*>(kv_len.data_ptr()),
+              batch_size, num_heads, head_dim_o, causal, stream);
+
+  TORCH_CHECK(status == cudaSuccess, "Failed to plan MLA, error: ", cudaGetErrorString(status));
+
+  return vec_to_tensor(plan_info.ToVector());
+}

flashinfer_python-0.2.2/csrc/batch_mla_sm90_pybind.cu

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2025 by FlashInfer team.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "batch_mla_sm90_config.inc"
+#include "pytorch_extension_utils.h"
+
+at::Tensor BatchMLAPagedAttentionSM90Plan(at::Tensor float_workspace_buffer,
+                                          at::Tensor int_workspace_buffer,
+                                          at::Tensor page_locked_int_workspace_buffer,
+                                          at::Tensor qo_indptr, at::Tensor kv_indptr,
+                                          at::Tensor kv_len, int64_t num_heads, int64_t head_dim_o,
+                                          bool causal, int64_t cuda_stream);
+
+void BatchMLAPagedAttentionSM90Run(at::Tensor float_workspace_buffer,
+                                   at::Tensor int_workspace_buffer, at::Tensor plan_info_vec,
+                                   at::Tensor q_nope, at::Tensor q_pe, at::Tensor ckv_cache,
+                                   at::Tensor kpe_cache, at::Tensor kv_indices, at::Tensor o,
+                                   std::optional<at::Tensor> maybe_lse, int64_t mask_mode_code,
+                                   int64_t num_heads, int64_t page_size, double sm_scale,
+                                   int64_t cuda_stream);
+
+TORCH_LIBRARY_FRAGMENT(TORCH_EXTENSION_NAME, m) {
+  m.def("plan", &BatchMLAPagedAttentionSM90Plan);
+  m.def("run", &BatchMLAPagedAttentionSM90Run);
+}

flashinfer_python-0.2.2/csrc/batch_mla_sm90_run.cu

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2025 by FlashInfer team.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <flashinfer/attention/mla_hopper.cuh>
+#include <flashinfer/attention/scheduler.cuh>
+#include <flashinfer/fastdiv.cuh>
+#include <optional>
+
+#include "batch_mla_sm90_config.inc"
+#include "pytorch_conversion_utils.h"
+#include "pytorch_extension_utils.h"
+
+using namespace flashinfer;
+
+void BatchMLAPagedAttentionSM90Run(at::Tensor float_workspace_buffer,
+                                   at::Tensor int_workspace_buffer, at::Tensor plan_info_vec,
+                                   at::Tensor q_nope, at::Tensor q_pe, at::Tensor ckv_cache,
+                                   at::Tensor kpe_cache, at::Tensor kv_indices, at::Tensor o,
+                                   std::optional<at::Tensor> maybe_lse, int64_t mask_mode_code,
+                                   int64_t num_heads, int64_t page_size, double sm_scale,
+                                   int64_t cuda_stream) {
+  // q_nope: [n, num_heads, head_dim_ckv]
+  // q_pe: [n, num_heads, head_dim_kpe]
+  // ckv_cache: [num_pages, page_size, head_dim_ckv]
+  // kpe_cache: [num_pages, page_size, head_dim_kpe]
+  MLAPlanInfo plan_info;
+  plan_info.FromVector(tensor_to_vec(plan_info_vec));
+
+  auto device = q_nope.device();
+
+  void* float_buffer_ptr = float_workspace_buffer.data_ptr();
+  void* int_buffer_ptr = int_workspace_buffer.data_ptr();
+
+  const MaskMode mask_mode = static_cast<MaskMode>(mask_mode_code);
+
+  auto q_scalar_type = q_nope.scalar_type();
+  auto kv_scalar_type = ckv_cache.scalar_type();
+
+  unsigned int q_nope_stride_n = q_nope.stride(0);
+  unsigned int q_nope_stride_h = q_nope.stride(1);
+  unsigned int q_pe_stride_n = q_pe.stride(0);
+  unsigned int q_pe_stride_h = q_pe.stride(1);
+  unsigned int ckv_stride_page = ckv_cache.stride(0);
+  unsigned int ckv_stride_n = ckv_cache.stride(1);
+  unsigned int kpe_stride_page = kpe_cache.stride(0);
+  unsigned int kpe_stride_n = kpe_cache.stride(1);
+  unsigned int o_stride_n = o.stride(0);
+  unsigned int o_stride_h = o.stride(1);
+
+  cudaStream_t stream = reinterpret_cast<cudaStream_t>(cuda_stream);
+
+  DISPATCH_context(
+      DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_CKV, HEAD_DIM_KPE, Params, [&] {
+        Params params;
+
+        params.q_nope = static_cast<DTypeQ*>(q_nope.data_ptr());
+        params.q_pe = static_cast<DTypeQ*>(q_pe.data_ptr());
+        params.ckv = static_cast<DTypeKV*>(ckv_cache.data_ptr());
+        params.kpe = static_cast<DTypeKV*>(kpe_cache.data_ptr());
+
+        params.q_indptr = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.q_indptr_offset);
+        params.kv_indptr = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_indptr_offset);
+        params.partial_indptr =
+            GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.partial_indptr_offset);
+        params.kv_indices = static_cast<IdType*>(kv_indices.data_ptr());
+        params.q_len = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.q_len_offset);
+        params.kv_len = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_len_offset);
+        params.q_start = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.q_start_offset);
+        params.kv_start = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_start_offset);
+        params.kv_end = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_end_offset);
+        params.work_indptr =
+            GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.work_indptr_offset);
+        params.merge_packed_offset_start = GetPtrFromBaseOffset<IdType>(
+            int_buffer_ptr, plan_info.merge_packed_offset_start_offset);
+        params.merge_packed_offset_end =
+            GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.merge_packed_offset_end_offset);
+        params.merge_indptr =
+            GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.merge_indptr_offset);
+        params.final_o = static_cast<DTypeO*>(o.data_ptr());
+        params.final_lse =
+            maybe_lse.has_value() ? static_cast<float*>(maybe_lse->data_ptr()) : nullptr;
+        params.partial_o =
+            GetPtrFromBaseOffset<float>(float_buffer_ptr, plan_info.partial_o_offset);
+        params.partial_lse =
+            GetPtrFromBaseOffset<float>(float_buffer_ptr, plan_info.partial_lse_offset);
+
+        params.num_heads = uint_fastdiv(num_heads);
+        params.block_size = uint_fastdiv(page_size);
+
+        params.q_nope_stride_n = q_nope_stride_n;
+        params.q_nope_stride_h = q_nope_stride_h;
+        params.q_pe_stride_n = q_pe_stride_n;
+        params.q_pe_stride_h = q_pe_stride_h;
+        params.ckv_stride_page = ckv_stride_page;
+        params.ckv_stride_n = ckv_stride_n;
+        params.kpe_stride_page = kpe_stride_page;
+        params.kpe_stride_n = kpe_stride_n;
+        params.o_stride_n = o_stride_n;
+        params.o_stride_h = o_stride_h;
+
+        params.sm_scale = sm_scale;
+
+        cudaError_t status =
+            mla::BatchMLAPageAttentionHopper<MASK_MODE, HEAD_DIM_CKV, HEAD_DIM_KPE>(
+                params, plan_info.num_blks_x, plan_info.num_blks_y, stream);
+
+        TORCH_CHECK(status == cudaSuccess,
+                    "Failed to run MLA, error: ", cudaGetErrorString(status));
+      });
+}

{flashinfer_python-0.2.1.post2 → flashinfer_python-0.2.2}/custom_backend.py

@@ -38,4 +38,5 @@ def build_editable(wheel_directory, config_settings=None, metadata_directory=Non
     ln("3rdparty/cutlass", "cutlass")
     ln("csrc", "csrc")
     ln("include", "include")
+    ln("tvm_binding", "tvm_binding")
     return orig.build_editable(wheel_directory, config_settings, metadata_directory)

flashinfer_python-0.2.2/flashinfer/_build_meta.py

@@ -0,0 +1 @@
+__version__ = '0.2.2'

{flashinfer_python-0.2.1.post2 → flashinfer_python-0.2.2}/flashinfer/activation.py

@@ -104,7 +104,7 @@ def silu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor:
         Input tensor, shape (..., 2 * hidden_size).
 
     out: Optional[torch.Tensor]
-        The the output tensor, if specified, the kernel will update this tensor inplace.
+        The output tensor, if specified, the kernel will update this tensor inplace.
 
     Returns
     -------
@@ -139,7 +139,7 @@ def gelu_tanh_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Te
         Input tensor, shape (..., 2 * hidden_size).
 
     out: Optional[torch.Tensor]
-        The the output tensor, if specified, the kernel will update this tensor inplace.
+        The output tensor, if specified, the kernel will update this tensor inplace.
 
     Returns
     -------
@@ -171,7 +171,7 @@ def gelu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor:
         Input tensor, shape (..., 2 * hidden_size).
 
     out: Optional[torch.Tensor]
-        The the output tensor, if specified, the kernel will update this tensor inplace.
+        The output tensor, if specified, the kernel will update this tensor inplace.
 
     Returns
     -------

{flashinfer_python-0.2.1.post2 → flashinfer_python-0.2.2}/flashinfer/jit/__init__.py

@@ -20,13 +20,20 @@ from .activation import get_act_and_mul_cu_str as get_act_and_mul_cu_str
 from .attention import gen_batch_decode_mla_module as gen_batch_decode_mla_module
 from .attention import gen_batch_decode_module as gen_batch_decode_module
 from .attention import gen_batch_mla_module as gen_batch_mla_module
+from .attention import gen_batch_mla_tvm_binding as gen_batch_mla_tvm_binding
 from .attention import gen_batch_prefill_module as gen_batch_prefill_module
 from .attention import (
     gen_customize_batch_decode_module as gen_customize_batch_decode_module,
 )
+from .attention import (
+    gen_customize_batch_decode_tvm_binding as gen_customize_batch_decode_tvm_binding,
+)
 from .attention import (
     gen_customize_batch_prefill_module as gen_customize_batch_prefill_module,
 )
+from .attention import (
+    gen_customize_batch_prefill_tvm_binding as gen_customize_batch_prefill_tvm_binding,
+)
 from .attention import (
     gen_customize_single_decode_module as gen_customize_single_decode_module,
 )

flashinfer_python-0.2.2/flashinfer/jit/aot_config.py

@@ -0,0 +1 @@
+prebuilt_ops_uri = set(['single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'single_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e4m3_dtype_kv_e4m3_dtype_o_e4m3_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_e5m2_dtype_kv_e5m2_dtype_o_e5m2_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True', 'batch_decode_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e4m3_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_e5m2_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e4m3_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_e5m2_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'single_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_128_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_256_head_dim_vo_256_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_f16_dtype_kv_f16_dtype_o_f16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_True_use_logits_cap_False_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_True_f16qk_False_sm90', 'batch_prefill_with_kv_cache_dtype_q_bf16_dtype_kv_bf16_dtype_o_bf16_dtype_idx_i32_head_dim_qk_192_head_dim_vo_128_posenc_0_use_swa_False_use_logits_cap_False_f16qk_False_sm90'])

flashinfer_python-0.2.2/flashinfer/jit/attention/__init__.py

@@ -0,0 +1,48 @@
+"""
+Copyright (c) 2025 by FlashInfer team.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from . import pytorch, tvm
+from .pytorch import gen_batch_decode_mla_module as gen_batch_decode_mla_module
+from .pytorch import gen_batch_decode_module as gen_batch_decode_module
+from .pytorch import gen_batch_mla_module as gen_batch_mla_module
+from .pytorch import gen_batch_prefill_module as gen_batch_prefill_module
+from .pytorch import (
+    gen_customize_batch_decode_module as gen_customize_batch_decode_module,
+)
+from .pytorch import (
+    gen_customize_batch_prefill_module as gen_customize_batch_prefill_module,
+)
+from .pytorch import (
+    gen_customize_single_decode_module as gen_customize_single_decode_module,
+)
+from .pytorch import (
+    gen_customize_single_prefill_module as gen_customize_single_prefill_module,
+)
+from .pytorch import gen_single_decode_module as gen_single_decode_module
+from .pytorch import gen_single_prefill_module as gen_single_prefill_module
+from .pytorch import get_batch_decode_mla_uri as get_batch_decode_mla_uri
+from .pytorch import get_batch_decode_uri as get_batch_decode_uri
+from .pytorch import get_batch_mla_uri as get_batch_mla_uri
+from .pytorch import get_batch_prefill_uri as get_batch_prefill_uri
+from .pytorch import get_single_decode_uri as get_single_decode_uri
+from .pytorch import get_single_prefill_uri as get_single_prefill_uri
+from .tvm import gen_batch_mla_tvm_binding as gen_batch_mla_tvm_binding
+from .tvm import (
+    gen_customize_batch_decode_tvm_binding as gen_customize_batch_decode_tvm_binding,
+)
+from .tvm import (
+    gen_customize_batch_prefill_tvm_binding as gen_customize_batch_prefill_tvm_binding,
+)