sglang 0.4.3.post1__py3-none-any.whl → 0.4.3.post3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. sglang/api.py +1 -1
  2. sglang/bench_offline_throughput.py +19 -0
  3. sglang/bench_one_batch.py +2 -2
  4. sglang/bench_serving.py +123 -79
  5. sglang/global_config.py +8 -3
  6. sglang/lang/backend/runtime_endpoint.py +1 -1
  7. sglang/lang/ir.py +1 -1
  8. sglang/srt/_custom_ops.py +83 -91
  9. sglang/srt/configs/load_config.py +4 -1
  10. sglang/srt/configs/model_config.py +48 -2
  11. sglang/srt/configs/qwen2_5_vl_config.py +5 -2
  12. sglang/srt/constrained/base_grammar_backend.py +117 -15
  13. sglang/srt/constrained/llguidance_backend.py +151 -0
  14. sglang/srt/constrained/outlines_backend.py +24 -33
  15. sglang/srt/constrained/xgrammar_backend.py +69 -38
  16. sglang/srt/distributed/device_communicators/custom_all_reduce.py +225 -80
  17. sglang/srt/distributed/parallel_state.py +48 -3
  18. sglang/srt/entrypoints/engine.py +67 -9
  19. sglang/srt/entrypoints/http_server.py +190 -41
  20. sglang/srt/entrypoints/verl_engine.py +147 -0
  21. sglang/srt/function_call_parser.py +0 -1
  22. sglang/srt/layers/activation.py +11 -0
  23. sglang/srt/layers/attention/{__init__.py → base_attn_backend.py} +14 -6
  24. sglang/srt/layers/attention/double_sparsity_backend.py +1 -1
  25. sglang/srt/layers/attention/flashinfer_backend.py +208 -295
  26. sglang/srt/layers/attention/flashinfer_mla_backend.py +582 -0
  27. sglang/srt/layers/attention/torch_native_backend.py +1 -1
  28. sglang/srt/layers/attention/triton_backend.py +9 -6
  29. sglang/srt/layers/attention/triton_ops/decode_attention.py +3 -0
  30. sglang/srt/layers/attention/triton_ops/extend_attention.py +20 -4
  31. sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py +439 -0
  32. sglang/srt/layers/attention/utils.py +39 -0
  33. sglang/srt/layers/attention/vision.py +60 -63
  34. sglang/srt/layers/dp_attention.py +142 -1
  35. sglang/srt/layers/layernorm.py +1 -1
  36. sglang/srt/layers/linear.py +3 -1
  37. sglang/srt/layers/logits_processor.py +281 -45
  38. sglang/srt/layers/moe/ep_moe/kernels.py +126 -8
  39. sglang/srt/layers/moe/ep_moe/layer.py +140 -28
  40. sglang/srt/layers/moe/fused_moe_native.py +2 -0
  41. sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  42. sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +146 -0
  43. sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json +50 -50
  44. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json +18 -18
  45. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Instinct_MI325X.json +18 -18
  46. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Radeon_Graphics.json +18 -18
  47. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json +18 -18
  48. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Instinct_MI325X.json +18 -18
  49. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Radeon_Graphics.json +18 -18
  50. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json +18 -18
  51. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Instinct_MI325X.json +18 -18
  52. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Radeon_Graphics.json +18 -18
  53. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json +16 -16
  54. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json +16 -16
  55. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json +16 -16
  56. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json +18 -18
  57. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Instinct_MI325X.json +18 -18
  58. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Radeon_Graphics.json +18 -18
  59. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json +15 -15
  60. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json +15 -15
  61. sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json +15 -15
  62. sglang/srt/layers/moe/fused_moe_triton/fused_moe.py +88 -20
  63. sglang/srt/layers/moe/fused_moe_triton/layer.py +34 -13
  64. sglang/srt/layers/moe/topk.py +13 -4
  65. sglang/srt/layers/quantization/__init__.py +111 -7
  66. sglang/srt/layers/quantization/blockwise_int8.py +409 -0
  67. sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  68. sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  69. sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  70. sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  71. sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  72. sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  73. sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  74. sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  75. sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  76. sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  77. sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  78. sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  79. sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  80. sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  81. sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  82. sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  83. sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  84. sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  85. sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  86. sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  87. sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  88. sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  89. sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  90. sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  91. sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  92. sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  93. sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  94. sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  95. sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  96. sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  97. sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json +164 -0
  98. sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  99. sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  100. sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json +146 -0
  101. sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json +26 -0
  102. sglang/srt/layers/quantization/fp8.py +69 -28
  103. sglang/srt/layers/quantization/fp8_utils.py +17 -1
  104. sglang/srt/layers/quantization/gptq.py +416 -0
  105. sglang/srt/layers/quantization/int8_kernel.py +327 -0
  106. sglang/srt/layers/quantization/int8_utils.py +73 -0
  107. sglang/srt/layers/quantization/modelopt_quant.py +18 -1
  108. sglang/srt/layers/radix_attention.py +1 -0
  109. sglang/srt/layers/rotary_embedding.py +0 -1
  110. sglang/srt/layers/sampler.py +76 -31
  111. sglang/srt/layers/vocab_parallel_embedding.py +14 -13
  112. sglang/srt/lora/lora.py +17 -1
  113. sglang/srt/lora/lora_config.py +5 -0
  114. sglang/srt/lora/lora_manager.py +1 -3
  115. sglang/srt/managers/cache_controller.py +193 -62
  116. sglang/srt/managers/configure_logging.py +2 -1
  117. sglang/srt/managers/data_parallel_controller.py +6 -2
  118. sglang/srt/managers/detokenizer_manager.py +124 -102
  119. sglang/srt/managers/image_processor.py +2 -1
  120. sglang/srt/managers/io_struct.py +143 -6
  121. sglang/srt/managers/schedule_batch.py +238 -197
  122. sglang/srt/managers/schedule_policy.py +29 -29
  123. sglang/srt/managers/scheduler.py +681 -259
  124. sglang/srt/managers/session_controller.py +6 -2
  125. sglang/srt/managers/tokenizer_manager.py +224 -68
  126. sglang/srt/managers/tp_worker.py +15 -4
  127. sglang/srt/managers/tp_worker_overlap_thread.py +3 -4
  128. sglang/srt/mem_cache/chunk_cache.py +18 -11
  129. sglang/srt/mem_cache/hiradix_cache.py +394 -0
  130. sglang/srt/mem_cache/memory_pool.py +44 -18
  131. sglang/srt/mem_cache/radix_cache.py +58 -47
  132. sglang/srt/metrics/collector.py +94 -36
  133. sglang/srt/model_executor/cuda_graph_runner.py +55 -24
  134. sglang/srt/model_executor/forward_batch_info.py +49 -16
  135. sglang/srt/model_executor/model_runner.py +209 -28
  136. sglang/srt/model_loader/loader.py +3 -3
  137. sglang/srt/model_loader/weight_utils.py +36 -14
  138. sglang/srt/models/baichuan.py +31 -6
  139. sglang/srt/models/chatglm.py +39 -7
  140. sglang/srt/models/commandr.py +29 -5
  141. sglang/srt/models/dbrx.py +31 -5
  142. sglang/srt/models/deepseek.py +43 -6
  143. sglang/srt/models/deepseek_nextn.py +32 -19
  144. sglang/srt/models/deepseek_v2.py +265 -29
  145. sglang/srt/models/exaone.py +19 -9
  146. sglang/srt/models/gemma.py +22 -8
  147. sglang/srt/models/gemma2.py +25 -12
  148. sglang/srt/models/gemma2_reward.py +5 -1
  149. sglang/srt/models/gpt2.py +28 -13
  150. sglang/srt/models/gpt_bigcode.py +27 -5
  151. sglang/srt/models/granite.py +21 -9
  152. sglang/srt/models/grok.py +21 -4
  153. sglang/srt/models/internlm2.py +36 -6
  154. sglang/srt/models/internlm2_reward.py +5 -1
  155. sglang/srt/models/llama.py +26 -9
  156. sglang/srt/models/llama_classification.py +5 -1
  157. sglang/srt/models/llama_eagle.py +17 -4
  158. sglang/srt/models/llama_embedding.py +5 -1
  159. sglang/srt/models/llama_reward.py +7 -2
  160. sglang/srt/models/llava.py +19 -3
  161. sglang/srt/models/llavavid.py +10 -1
  162. sglang/srt/models/minicpm.py +26 -2
  163. sglang/srt/models/minicpm3.py +39 -3
  164. sglang/srt/models/minicpmv.py +45 -14
  165. sglang/srt/models/mixtral.py +20 -9
  166. sglang/srt/models/mixtral_quant.py +50 -8
  167. sglang/srt/models/mllama.py +57 -11
  168. sglang/srt/models/olmo.py +34 -6
  169. sglang/srt/models/olmo2.py +34 -13
  170. sglang/srt/models/olmoe.py +26 -4
  171. sglang/srt/models/phi3_small.py +29 -10
  172. sglang/srt/models/qwen.py +26 -3
  173. sglang/srt/models/qwen2.py +26 -4
  174. sglang/srt/models/qwen2_5_vl.py +46 -8
  175. sglang/srt/models/qwen2_eagle.py +17 -5
  176. sglang/srt/models/qwen2_moe.py +44 -6
  177. sglang/srt/models/qwen2_rm.py +78 -0
  178. sglang/srt/models/qwen2_vl.py +39 -8
  179. sglang/srt/models/stablelm.py +32 -5
  180. sglang/srt/models/torch_native_llama.py +5 -2
  181. sglang/srt/models/xverse.py +21 -9
  182. sglang/srt/models/xverse_moe.py +45 -7
  183. sglang/srt/models/yivl.py +2 -1
  184. sglang/srt/openai_api/adapter.py +109 -24
  185. sglang/srt/openai_api/protocol.py +17 -1
  186. sglang/srt/reasoning_parser.py +154 -0
  187. sglang/srt/sampling/penaltylib/__init__.py +4 -6
  188. sglang/srt/sampling/penaltylib/frequency_penalty.py +66 -0
  189. sglang/srt/sampling/penaltylib/{penalizers/min_new_tokens.py → min_new_tokens.py} +15 -23
  190. sglang/srt/sampling/penaltylib/orchestrator.py +39 -188
  191. sglang/srt/sampling/penaltylib/presence_penalty.py +66 -0
  192. sglang/srt/sampling/sampling_batch_info.py +79 -157
  193. sglang/srt/sampling/sampling_params.py +16 -13
  194. sglang/srt/server_args.py +136 -52
  195. sglang/srt/speculative/build_eagle_tree.py +2 -8
  196. sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +0 -1
  197. sglang/srt/speculative/eagle_utils.py +92 -58
  198. sglang/srt/speculative/eagle_worker.py +186 -94
  199. sglang/srt/speculative/spec_info.py +1 -13
  200. sglang/srt/utils.py +43 -17
  201. sglang/srt/warmup.py +47 -0
  202. sglang/test/few_shot_gsm8k.py +4 -1
  203. sglang/test/runners.py +389 -126
  204. sglang/test/send_one.py +88 -0
  205. sglang/test/test_block_fp8_ep.py +361 -0
  206. sglang/test/test_programs.py +1 -1
  207. sglang/test/test_utils.py +138 -84
  208. sglang/utils.py +50 -60
  209. sglang/version.py +1 -1
  210. {sglang-0.4.3.post1.dist-info → sglang-0.4.3.post3.dist-info}/METADATA +21 -15
  211. {sglang-0.4.3.post1.dist-info → sglang-0.4.3.post3.dist-info}/RECORD +214 -166
  212. {sglang-0.4.3.post1.dist-info → sglang-0.4.3.post3.dist-info}/WHEEL +1 -1
  213. sglang/bench_latency.py +0 -1
  214. sglang/srt/sampling/penaltylib/penalizers/frequency_penalty.py +0 -75
  215. sglang/srt/sampling/penaltylib/penalizers/presence_penalty.py +0 -74
  216. sglang/srt/sampling/penaltylib/penalizers/repetition_penalty.py +0 -85
  217. sglang/test/srt/sampling/penaltylib/utils.py +0 -344
  218. {sglang-0.4.3.post1.dist-info → sglang-0.4.3.post3.dist-info}/LICENSE +0 -0
  219. {sglang-0.4.3.post1.dist-info → sglang-0.4.3.post3.dist-info}/top_level.txt +0 -0
sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "BLOCK_SIZE_K": 256,
6
6
  "GROUP_SIZE_M": 1,
7
7
  "num_warps": 4,
8
- "num_stages": 0,
8
+ "num_stages": 2,
9
9
  "waves_per_eu": 4,
10
10
  "matrix_instr_nonkdim": 16,
11
11
  "kpack": 2
@@ -16,7 +16,7 @@
16
16
  "BLOCK_SIZE_K": 256,
17
17
  "GROUP_SIZE_M": 1,
18
18
  "num_warps": 4,
19
- "num_stages": 0,
19
+ "num_stages": 2,
20
20
  "waves_per_eu": 1,
21
21
  "matrix_instr_nonkdim": 16,
22
22
  "kpack": 1
@@ -27,7 +27,7 @@
27
27
  "BLOCK_SIZE_K": 256,
28
28
  "GROUP_SIZE_M": 1,
29
29
  "num_warps": 8,
30
- "num_stages": 0,
30
+ "num_stages": 2,
31
31
  "waves_per_eu": 2,
32
32
  "matrix_instr_nonkdim": 16,
33
33
  "kpack": 2
@@ -38,7 +38,7 @@
38
38
  "BLOCK_SIZE_K": 256,
39
39
  "GROUP_SIZE_M": 1,
40
40
  "num_warps": 8,
41
- "num_stages": 0,
41
+ "num_stages": 2,
42
42
  "waves_per_eu": 1,
43
43
  "matrix_instr_nonkdim": 16,
44
44
  "kpack": 2
@@ -49,7 +49,7 @@
49
49
  "BLOCK_SIZE_K": 256,
50
50
  "GROUP_SIZE_M": 1,
51
51
  "num_warps": 4,
52
- "num_stages": 0,
52
+ "num_stages": 2,
53
53
  "waves_per_eu": 2,
54
54
  "matrix_instr_nonkdim": 16,
55
55
  "kpack": 2
@@ -60,7 +60,7 @@
60
60
  "BLOCK_SIZE_K": 256,
61
61
  "GROUP_SIZE_M": 1,
62
62
  "num_warps": 4,
63
- "num_stages": 0,
63
+ "num_stages": 2,
64
64
  "waves_per_eu": 1,
65
65
  "matrix_instr_nonkdim": 16,
66
66
  "kpack": 1
@@ -79,7 +79,7 @@
79
79
  "BLOCK_SIZE_K": 256,
80
80
  "GROUP_SIZE_M": 1,
81
81
  "num_warps": 4,
82
- "num_stages": 0,
82
+ "num_stages": 2,
83
83
  "waves_per_eu": 2,
84
84
  "matrix_instr_nonkdim": 16,
85
85
  "kpack": 2
@@ -90,7 +90,7 @@
90
90
  "BLOCK_SIZE_K": 128,
91
91
  "GROUP_SIZE_M": 1,
92
92
  "num_warps": 8,
93
- "num_stages": 0,
93
+ "num_stages": 2,
94
94
  "waves_per_eu": 4,
95
95
  "matrix_instr_nonkdim": 16,
96
96
  "kpack": 2
@@ -101,7 +101,7 @@
101
101
  "BLOCK_SIZE_K": 128,
102
102
  "GROUP_SIZE_M": 1,
103
103
  "num_warps": 8,
104
- "num_stages": 0,
104
+ "num_stages": 2,
105
105
  "waves_per_eu": 2,
106
106
  "matrix_instr_nonkdim": 16,
107
107
  "kpack": 2
@@ -112,7 +112,7 @@
112
112
  "BLOCK_SIZE_K": 128,
113
113
  "GROUP_SIZE_M": 1,
114
114
  "num_warps": 8,
115
- "num_stages": 0,
115
+ "num_stages": 2,
116
116
  "waves_per_eu": 2,
117
117
  "matrix_instr_nonkdim": 16,
118
118
  "kpack": 2
@@ -123,7 +123,7 @@
123
123
  "BLOCK_SIZE_K": 64,
124
124
  "GROUP_SIZE_M": 1,
125
125
  "num_warps": 8,
126
- "num_stages": 0,
126
+ "num_stages": 2,
127
127
  "waves_per_eu": 2,
128
128
  "matrix_instr_nonkdim": 16,
129
129
  "kpack": 1
@@ -134,7 +134,7 @@
134
134
  "BLOCK_SIZE_K": 64,
135
135
  "GROUP_SIZE_M": 1,
136
136
  "num_warps": 8,
137
- "num_stages": 0,
137
+ "num_stages": 2,
138
138
  "waves_per_eu": 1,
139
139
  "matrix_instr_nonkdim": 16,
140
140
  "kpack": 1
@@ -145,7 +145,7 @@
145
145
  "BLOCK_SIZE_K": 64,
146
146
  "GROUP_SIZE_M": 1,
147
147
  "num_warps": 8,
148
- "num_stages": 0,
148
+ "num_stages": 2,
149
149
  "waves_per_eu": 0,
150
150
  "matrix_instr_nonkdim": 16,
151
151
  "kpack": 1
@@ -156,7 +156,7 @@
156
156
  "BLOCK_SIZE_K": 64,
157
157
  "GROUP_SIZE_M": 1,
158
158
  "num_warps": 8,
159
- "num_stages": 0,
159
+ "num_stages": 2,
160
160
  "waves_per_eu": 1,
161
161
  "matrix_instr_nonkdim": 16,
162
162
  "kpack": 1
@@ -167,7 +167,7 @@
167
167
  "BLOCK_SIZE_K": 64,
168
168
  "GROUP_SIZE_M": 1,
169
169
  "num_warps": 4,
170
- "num_stages": 0,
170
+ "num_stages": 2,
171
171
  "waves_per_eu": 2,
172
172
  "matrix_instr_nonkdim": 16,
173
173
  "kpack": 2
sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "BLOCK_SIZE_K": 256,
6
6
  "GROUP_SIZE_M": 1,
7
7
  "num_warps": 4,
8
- "num_stages": 0,
8
+ "num_stages": 2,
9
9
  "waves_per_eu": 4,
10
10
  "matrix_instr_nonkdim": 16,
11
11
  "kpack": 2
@@ -16,7 +16,7 @@
16
16
  "BLOCK_SIZE_K": 256,
17
17
  "GROUP_SIZE_M": 1,
18
18
  "num_warps": 4,
19
- "num_stages": 0,
19
+ "num_stages": 2,
20
20
  "waves_per_eu": 1,
21
21
  "matrix_instr_nonkdim": 16,
22
22
  "kpack": 1
@@ -27,7 +27,7 @@
27
27
  "BLOCK_SIZE_K": 256,
28
28
  "GROUP_SIZE_M": 1,
29
29
  "num_warps": 8,
30
- "num_stages": 0,
30
+ "num_stages": 2,
31
31
  "waves_per_eu": 2,
32
32
  "matrix_instr_nonkdim": 16,
33
33
  "kpack": 2
@@ -38,7 +38,7 @@
38
38
  "BLOCK_SIZE_K": 256,
39
39
  "GROUP_SIZE_M": 1,
40
40
  "num_warps": 8,
41
- "num_stages": 0,
41
+ "num_stages": 2,
42
42
  "waves_per_eu": 1,
43
43
  "matrix_instr_nonkdim": 16,
44
44
  "kpack": 2
@@ -49,7 +49,7 @@
49
49
  "BLOCK_SIZE_K": 256,
50
50
  "GROUP_SIZE_M": 1,
51
51
  "num_warps": 4,
52
- "num_stages": 0,
52
+ "num_stages": 2,
53
53
  "waves_per_eu": 2,
54
54
  "matrix_instr_nonkdim": 16,
55
55
  "kpack": 2
@@ -60,7 +60,7 @@
60
60
  "BLOCK_SIZE_K": 256,
61
61
  "GROUP_SIZE_M": 1,
62
62
  "num_warps": 4,
63
- "num_stages": 0,
63
+ "num_stages": 2,
64
64
  "waves_per_eu": 1,
65
65
  "matrix_instr_nonkdim": 16,
66
66
  "kpack": 1
@@ -79,7 +79,7 @@
79
79
  "BLOCK_SIZE_K": 256,
80
80
  "GROUP_SIZE_M": 1,
81
81
  "num_warps": 4,
82
- "num_stages": 0,
82
+ "num_stages": 2,
83
83
  "waves_per_eu": 2,
84
84
  "matrix_instr_nonkdim": 16,
85
85
  "kpack": 2
@@ -90,7 +90,7 @@
90
90
  "BLOCK_SIZE_K": 128,
91
91
  "GROUP_SIZE_M": 1,
92
92
  "num_warps": 8,
93
- "num_stages": 0,
93
+ "num_stages": 2,
94
94
  "waves_per_eu": 4,
95
95
  "matrix_instr_nonkdim": 16,
96
96
  "kpack": 2
@@ -101,7 +101,7 @@
101
101
  "BLOCK_SIZE_K": 128,
102
102
  "GROUP_SIZE_M": 1,
103
103
  "num_warps": 8,
104
- "num_stages": 0,
104
+ "num_stages": 2,
105
105
  "waves_per_eu": 2,
106
106
  "matrix_instr_nonkdim": 16,
107
107
  "kpack": 2
@@ -112,7 +112,7 @@
112
112
  "BLOCK_SIZE_K": 128,
113
113
  "GROUP_SIZE_M": 1,
114
114
  "num_warps": 8,
115
- "num_stages": 0,
115
+ "num_stages": 2,
116
116
  "waves_per_eu": 2,
117
117
  "matrix_instr_nonkdim": 16,
118
118
  "kpack": 2
@@ -123,7 +123,7 @@
123
123
  "BLOCK_SIZE_K": 64,
124
124
  "GROUP_SIZE_M": 1,
125
125
  "num_warps": 8,
126
- "num_stages": 0,
126
+ "num_stages": 2,
127
127
  "waves_per_eu": 2,
128
128
  "matrix_instr_nonkdim": 16,
129
129
  "kpack": 1
@@ -134,7 +134,7 @@
134
134
  "BLOCK_SIZE_K": 64,
135
135
  "GROUP_SIZE_M": 1,
136
136
  "num_warps": 8,
137
- "num_stages": 0,
137
+ "num_stages": 2,
138
138
  "waves_per_eu": 1,
139
139
  "matrix_instr_nonkdim": 16,
140
140
  "kpack": 1
@@ -145,7 +145,7 @@
145
145
  "BLOCK_SIZE_K": 64,
146
146
  "GROUP_SIZE_M": 1,
147
147
  "num_warps": 8,
148
- "num_stages": 0,
148
+ "num_stages": 2,
149
149
  "waves_per_eu": 0,
150
150
  "matrix_instr_nonkdim": 16,
151
151
  "kpack": 1
@@ -156,7 +156,7 @@
156
156
  "BLOCK_SIZE_K": 64,
157
157
  "GROUP_SIZE_M": 1,
158
158
  "num_warps": 8,
159
- "num_stages": 0,
159
+ "num_stages": 2,
160
160
  "waves_per_eu": 1,
161
161
  "matrix_instr_nonkdim": 16,
162
162
  "kpack": 1
@@ -167,7 +167,7 @@
167
167
  "BLOCK_SIZE_K": 64,
168
168
  "GROUP_SIZE_M": 1,
169
169
  "num_warps": 4,
170
- "num_stages": 0,
170
+ "num_stages": 2,
171
171
  "waves_per_eu": 2,
172
172
  "matrix_instr_nonkdim": 16,
173
173
  "kpack": 2
sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "BLOCK_SIZE_K": 256,
6
6
  "GROUP_SIZE_M": 1,
7
7
  "num_warps": 4,
8
- "num_stages": 0,
8
+ "num_stages": 2,
9
9
  "waves_per_eu": 4,
10
10
  "matrix_instr_nonkdim": 16,
11
11
  "kpack": 2
@@ -16,7 +16,7 @@
16
16
  "BLOCK_SIZE_K": 256,
17
17
  "GROUP_SIZE_M": 1,
18
18
  "num_warps": 4,
19
- "num_stages": 0,
19
+ "num_stages": 2,
20
20
  "waves_per_eu": 1,
21
21
  "matrix_instr_nonkdim": 16,
22
22
  "kpack": 1
@@ -27,7 +27,7 @@
27
27
  "BLOCK_SIZE_K": 256,
28
28
  "GROUP_SIZE_M": 1,
29
29
  "num_warps": 8,
30
- "num_stages": 0,
30
+ "num_stages": 2,
31
31
  "waves_per_eu": 2,
32
32
  "matrix_instr_nonkdim": 16,
33
33
  "kpack": 2
@@ -38,7 +38,7 @@
38
38
  "BLOCK_SIZE_K": 256,
39
39
  "GROUP_SIZE_M": 1,
40
40
  "num_warps": 8,
41
- "num_stages": 0,
41
+ "num_stages": 2,
42
42
  "waves_per_eu": 1,
43
43
  "matrix_instr_nonkdim": 16,
44
44
  "kpack": 2
@@ -49,7 +49,7 @@
49
49
  "BLOCK_SIZE_K": 256,
50
50
  "GROUP_SIZE_M": 1,
51
51
  "num_warps": 4,
52
- "num_stages": 0,
52
+ "num_stages": 2,
53
53
  "waves_per_eu": 2,
54
54
  "matrix_instr_nonkdim": 16,
55
55
  "kpack": 2
@@ -60,7 +60,7 @@
60
60
  "BLOCK_SIZE_K": 256,
61
61
  "GROUP_SIZE_M": 1,
62
62
  "num_warps": 4,
63
- "num_stages": 0,
63
+ "num_stages": 2,
64
64
  "waves_per_eu": 1,
65
65
  "matrix_instr_nonkdim": 16,
66
66
  "kpack": 1
@@ -79,7 +79,7 @@
79
79
  "BLOCK_SIZE_K": 256,
80
80
  "GROUP_SIZE_M": 1,
81
81
  "num_warps": 4,
82
- "num_stages": 0,
82
+ "num_stages": 2,
83
83
  "waves_per_eu": 2,
84
84
  "matrix_instr_nonkdim": 16,
85
85
  "kpack": 2
@@ -90,7 +90,7 @@
90
90
  "BLOCK_SIZE_K": 128,
91
91
  "GROUP_SIZE_M": 1,
92
92
  "num_warps": 8,
93
- "num_stages": 0,
93
+ "num_stages": 2,
94
94
  "waves_per_eu": 4,
95
95
  "matrix_instr_nonkdim": 16,
96
96
  "kpack": 2
@@ -101,7 +101,7 @@
101
101
  "BLOCK_SIZE_K": 128,
102
102
  "GROUP_SIZE_M": 1,
103
103
  "num_warps": 8,
104
- "num_stages": 0,
104
+ "num_stages": 2,
105
105
  "waves_per_eu": 2,
106
106
  "matrix_instr_nonkdim": 16,
107
107
  "kpack": 2
@@ -112,7 +112,7 @@
112
112
  "BLOCK_SIZE_K": 128,
113
113
  "GROUP_SIZE_M": 1,
114
114
  "num_warps": 8,
115
- "num_stages": 0,
115
+ "num_stages": 2,
116
116
  "waves_per_eu": 2,
117
117
  "matrix_instr_nonkdim": 16,
118
118
  "kpack": 2
@@ -123,7 +123,7 @@
123
123
  "BLOCK_SIZE_K": 64,
124
124
  "GROUP_SIZE_M": 1,
125
125
  "num_warps": 8,
126
- "num_stages": 0,
126
+ "num_stages": 2,
127
127
  "waves_per_eu": 2,
128
128
  "matrix_instr_nonkdim": 16,
129
129
  "kpack": 1
@@ -134,7 +134,7 @@
134
134
  "BLOCK_SIZE_K": 64,
135
135
  "GROUP_SIZE_M": 1,
136
136
  "num_warps": 8,
137
- "num_stages": 0,
137
+ "num_stages": 2,
138
138
  "waves_per_eu": 1,
139
139
  "matrix_instr_nonkdim": 16,
140
140
  "kpack": 1
@@ -145,7 +145,7 @@
145
145
  "BLOCK_SIZE_K": 64,
146
146
  "GROUP_SIZE_M": 1,
147
147
  "num_warps": 8,
148
- "num_stages": 0,
148
+ "num_stages": 2,
149
149
  "waves_per_eu": 0,
150
150
  "matrix_instr_nonkdim": 16,
151
151
  "kpack": 1
@@ -156,7 +156,7 @@
156
156
  "BLOCK_SIZE_K": 64,
157
157
  "GROUP_SIZE_M": 1,
158
158
  "num_warps": 8,
159
- "num_stages": 0,
159
+ "num_stages": 2,
160
160
  "waves_per_eu": 1,
161
161
  "matrix_instr_nonkdim": 16,
162
162
  "kpack": 1
@@ -167,7 +167,7 @@
167
167
  "BLOCK_SIZE_K": 64,
168
168
  "GROUP_SIZE_M": 1,
169
169
  "num_warps": 4,
170
- "num_stages": 0,
170
+ "num_stages": 2,
171
171
  "waves_per_eu": 2,
172
172
  "matrix_instr_nonkdim": 16,
173
173
  "kpack": 2