sglang 0.4.6__py3-none-any.whl → 0.4.6.post2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sglang/bench_one_batch.py +2 -0
- sglang/check_env.py +3 -3
- sglang/srt/configs/__init__.py +4 -0
- sglang/srt/configs/kimi_vl.py +38 -0
- sglang/srt/configs/kimi_vl_moonvit.py +32 -0
- sglang/srt/configs/model_config.py +15 -0
- sglang/srt/conversation.py +122 -1
- sglang/srt/disaggregation/decode.py +8 -2
- sglang/srt/disaggregation/fake/__init__.py +1 -0
- sglang/srt/disaggregation/fake/conn.py +88 -0
- sglang/srt/disaggregation/prefill.py +12 -3
- sglang/srt/disaggregation/utils.py +16 -2
- sglang/srt/entrypoints/engine.py +52 -21
- sglang/srt/entrypoints/http_server.py +27 -2
- sglang/srt/function_call_parser.py +97 -0
- sglang/srt/hf_transformers_utils.py +2 -0
- sglang/srt/layers/attention/cutlass_mla_backend.py +278 -0
- sglang/srt/layers/attention/flashinfer_backend.py +107 -82
- sglang/srt/layers/attention/flashinfer_mla_backend.py +27 -16
- sglang/srt/layers/attention/flashmla_backend.py +3 -0
- sglang/srt/layers/attention/utils.py +1 -1
- sglang/srt/layers/dp_attention.py +5 -2
- sglang/srt/layers/moe/ep_moe/token_dispatcher.py +1 -3
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=192,device_name=NVIDIA_H20.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=192,device_name=NVIDIA_H200.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=384,device_name=NVIDIA_H20.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=384,device_name=NVIDIA_H200.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_H20.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=768,device_name=NVIDIA_H200.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/configs/E=128,N=96,device_name=NVIDIA_H20.json +146 -0
- sglang/srt/layers/moe/fused_moe_triton/fused_moe.py +10 -8
- sglang/srt/layers/moe/fused_moe_triton/layer.py +15 -17
- sglang/srt/layers/quantization/__init__.py +2 -2
- sglang/srt/layers/quantization/deep_gemm.py +1 -1
- sglang/srt/layers/quantization/fp8.py +20 -22
- sglang/srt/layers/quantization/fp8_utils.py +2 -2
- sglang/srt/layers/utils.py +35 -0
- sglang/srt/lora/layers.py +35 -9
- sglang/srt/lora/lora_manager.py +84 -35
- sglang/srt/managers/data_parallel_controller.py +52 -34
- sglang/srt/managers/multimodal_processors/kimi_vl.py +73 -0
- sglang/srt/managers/schedule_batch.py +34 -15
- sglang/srt/managers/scheduler.py +273 -67
- sglang/srt/managers/scheduler_output_processor_mixin.py +26 -10
- sglang/srt/managers/tp_worker.py +52 -17
- sglang/srt/managers/tp_worker_overlap_thread.py +18 -7
- sglang/srt/mem_cache/memory_pool.py +70 -36
- sglang/srt/model_executor/cuda_graph_runner.py +82 -19
- sglang/srt/model_executor/forward_batch_info.py +31 -1
- sglang/srt/model_executor/model_runner.py +123 -58
- sglang/srt/models/deepseek_nextn.py +1 -257
- sglang/srt/models/deepseek_v2.py +78 -18
- sglang/srt/models/kimi_vl.py +308 -0
- sglang/srt/models/kimi_vl_moonvit.py +639 -0
- sglang/srt/models/llama.py +92 -30
- sglang/srt/models/llama4.py +2 -1
- sglang/srt/models/llama_eagle.py +4 -1
- sglang/srt/models/llama_eagle3.py +4 -1
- sglang/srt/models/qwen2_moe.py +8 -3
- sglang/srt/models/qwen2_vl.py +0 -12
- sglang/srt/models/qwen3_moe.py +8 -3
- sglang/srt/openai_api/adapter.py +49 -8
- sglang/srt/openai_api/protocol.py +13 -1
- sglang/srt/reasoning_parser.py +25 -1
- sglang/srt/server_args.py +83 -24
- sglang/srt/speculative/eagle_worker.py +3 -2
- sglang/srt/utils.py +91 -9
- sglang/test/runners.py +4 -0
- sglang/test/send_one.py +84 -28
- sglang/test/test_utils.py +67 -0
- sglang/version.py +1 -1
- {sglang-0.4.6.dist-info → sglang-0.4.6.post2.dist-info}/METADATA +5 -4
- {sglang-0.4.6.dist-info → sglang-0.4.6.post2.dist-info}/RECORD +85 -60
- {sglang-0.4.6.dist-info → sglang-0.4.6.post2.dist-info}/WHEEL +1 -1
- {sglang-0.4.6.dist-info → sglang-0.4.6.post2.dist-info}/licenses/LICENSE +0 -0
- {sglang-0.4.6.dist-info → sglang-0.4.6.post2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
{
|
|
2
|
+
"1": {
|
|
3
|
+
"BLOCK_SIZE_M": 64,
|
|
4
|
+
"BLOCK_SIZE_N": 64,
|
|
5
|
+
"BLOCK_SIZE_K": 128,
|
|
6
|
+
"GROUP_SIZE_M": 1,
|
|
7
|
+
"num_warps": 4,
|
|
8
|
+
"num_stages": 3
|
|
9
|
+
},
|
|
10
|
+
"2": {
|
|
11
|
+
"BLOCK_SIZE_M": 64,
|
|
12
|
+
"BLOCK_SIZE_N": 128,
|
|
13
|
+
"BLOCK_SIZE_K": 128,
|
|
14
|
+
"GROUP_SIZE_M": 32,
|
|
15
|
+
"num_warps": 4,
|
|
16
|
+
"num_stages": 4
|
|
17
|
+
},
|
|
18
|
+
"4": {
|
|
19
|
+
"BLOCK_SIZE_M": 16,
|
|
20
|
+
"BLOCK_SIZE_N": 128,
|
|
21
|
+
"BLOCK_SIZE_K": 128,
|
|
22
|
+
"GROUP_SIZE_M": 32,
|
|
23
|
+
"num_warps": 4,
|
|
24
|
+
"num_stages": 3
|
|
25
|
+
},
|
|
26
|
+
"8": {
|
|
27
|
+
"BLOCK_SIZE_M": 16,
|
|
28
|
+
"BLOCK_SIZE_N": 128,
|
|
29
|
+
"BLOCK_SIZE_K": 128,
|
|
30
|
+
"GROUP_SIZE_M": 16,
|
|
31
|
+
"num_warps": 4,
|
|
32
|
+
"num_stages": 5
|
|
33
|
+
},
|
|
34
|
+
"16": {
|
|
35
|
+
"BLOCK_SIZE_M": 16,
|
|
36
|
+
"BLOCK_SIZE_N": 128,
|
|
37
|
+
"BLOCK_SIZE_K": 128,
|
|
38
|
+
"GROUP_SIZE_M": 32,
|
|
39
|
+
"num_warps": 4,
|
|
40
|
+
"num_stages": 3
|
|
41
|
+
},
|
|
42
|
+
"24": {
|
|
43
|
+
"BLOCK_SIZE_M": 16,
|
|
44
|
+
"BLOCK_SIZE_N": 128,
|
|
45
|
+
"BLOCK_SIZE_K": 128,
|
|
46
|
+
"GROUP_SIZE_M": 16,
|
|
47
|
+
"num_warps": 4,
|
|
48
|
+
"num_stages": 5
|
|
49
|
+
},
|
|
50
|
+
"32": {
|
|
51
|
+
"BLOCK_SIZE_M": 16,
|
|
52
|
+
"BLOCK_SIZE_N": 128,
|
|
53
|
+
"BLOCK_SIZE_K": 128,
|
|
54
|
+
"GROUP_SIZE_M": 1,
|
|
55
|
+
"num_warps": 4,
|
|
56
|
+
"num_stages": 3
|
|
57
|
+
},
|
|
58
|
+
"48": {
|
|
59
|
+
"BLOCK_SIZE_M": 16,
|
|
60
|
+
"BLOCK_SIZE_N": 128,
|
|
61
|
+
"BLOCK_SIZE_K": 128,
|
|
62
|
+
"GROUP_SIZE_M": 32,
|
|
63
|
+
"num_warps": 4,
|
|
64
|
+
"num_stages": 3
|
|
65
|
+
},
|
|
66
|
+
"64": {
|
|
67
|
+
"BLOCK_SIZE_M": 16,
|
|
68
|
+
"BLOCK_SIZE_N": 128,
|
|
69
|
+
"BLOCK_SIZE_K": 128,
|
|
70
|
+
"GROUP_SIZE_M": 16,
|
|
71
|
+
"num_warps": 4,
|
|
72
|
+
"num_stages": 3
|
|
73
|
+
},
|
|
74
|
+
"96": {
|
|
75
|
+
"BLOCK_SIZE_M": 16,
|
|
76
|
+
"BLOCK_SIZE_N": 128,
|
|
77
|
+
"BLOCK_SIZE_K": 128,
|
|
78
|
+
"GROUP_SIZE_M": 16,
|
|
79
|
+
"num_warps": 4,
|
|
80
|
+
"num_stages": 3
|
|
81
|
+
},
|
|
82
|
+
"128": {
|
|
83
|
+
"BLOCK_SIZE_M": 16,
|
|
84
|
+
"BLOCK_SIZE_N": 128,
|
|
85
|
+
"BLOCK_SIZE_K": 128,
|
|
86
|
+
"GROUP_SIZE_M": 1,
|
|
87
|
+
"num_warps": 4,
|
|
88
|
+
"num_stages": 3
|
|
89
|
+
},
|
|
90
|
+
"256": {
|
|
91
|
+
"BLOCK_SIZE_M": 64,
|
|
92
|
+
"BLOCK_SIZE_N": 128,
|
|
93
|
+
"BLOCK_SIZE_K": 128,
|
|
94
|
+
"GROUP_SIZE_M": 16,
|
|
95
|
+
"num_warps": 4,
|
|
96
|
+
"num_stages": 4
|
|
97
|
+
},
|
|
98
|
+
"512": {
|
|
99
|
+
"BLOCK_SIZE_M": 64,
|
|
100
|
+
"BLOCK_SIZE_N": 128,
|
|
101
|
+
"BLOCK_SIZE_K": 128,
|
|
102
|
+
"GROUP_SIZE_M": 16,
|
|
103
|
+
"num_warps": 4,
|
|
104
|
+
"num_stages": 4
|
|
105
|
+
},
|
|
106
|
+
"1024": {
|
|
107
|
+
"BLOCK_SIZE_M": 64,
|
|
108
|
+
"BLOCK_SIZE_N": 128,
|
|
109
|
+
"BLOCK_SIZE_K": 128,
|
|
110
|
+
"GROUP_SIZE_M": 16,
|
|
111
|
+
"num_warps": 4,
|
|
112
|
+
"num_stages": 4
|
|
113
|
+
},
|
|
114
|
+
"1536": {
|
|
115
|
+
"BLOCK_SIZE_M": 64,
|
|
116
|
+
"BLOCK_SIZE_N": 128,
|
|
117
|
+
"BLOCK_SIZE_K": 128,
|
|
118
|
+
"GROUP_SIZE_M": 16,
|
|
119
|
+
"num_warps": 4,
|
|
120
|
+
"num_stages": 4
|
|
121
|
+
},
|
|
122
|
+
"2048": {
|
|
123
|
+
"BLOCK_SIZE_M": 64,
|
|
124
|
+
"BLOCK_SIZE_N": 128,
|
|
125
|
+
"BLOCK_SIZE_K": 128,
|
|
126
|
+
"GROUP_SIZE_M": 16,
|
|
127
|
+
"num_warps": 4,
|
|
128
|
+
"num_stages": 4
|
|
129
|
+
},
|
|
130
|
+
"3072": {
|
|
131
|
+
"BLOCK_SIZE_M": 64,
|
|
132
|
+
"BLOCK_SIZE_N": 128,
|
|
133
|
+
"BLOCK_SIZE_K": 128,
|
|
134
|
+
"GROUP_SIZE_M": 16,
|
|
135
|
+
"num_warps": 4,
|
|
136
|
+
"num_stages": 4
|
|
137
|
+
},
|
|
138
|
+
"4096": {
|
|
139
|
+
"BLOCK_SIZE_M": 64,
|
|
140
|
+
"BLOCK_SIZE_N": 128,
|
|
141
|
+
"BLOCK_SIZE_K": 128,
|
|
142
|
+
"GROUP_SIZE_M": 16,
|
|
143
|
+
"num_warps": 4,
|
|
144
|
+
"num_stages": 4
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
{
|
|
2
|
+
"1": {
|
|
3
|
+
"BLOCK_SIZE_M": 16,
|
|
4
|
+
"BLOCK_SIZE_N": 64,
|
|
5
|
+
"BLOCK_SIZE_K": 64,
|
|
6
|
+
"GROUP_SIZE_M": 1,
|
|
7
|
+
"num_warps": 4,
|
|
8
|
+
"num_stages": 5
|
|
9
|
+
},
|
|
10
|
+
"2": {
|
|
11
|
+
"BLOCK_SIZE_M": 16,
|
|
12
|
+
"BLOCK_SIZE_N": 64,
|
|
13
|
+
"BLOCK_SIZE_K": 64,
|
|
14
|
+
"GROUP_SIZE_M": 1,
|
|
15
|
+
"num_warps": 4,
|
|
16
|
+
"num_stages": 4
|
|
17
|
+
},
|
|
18
|
+
"4": {
|
|
19
|
+
"BLOCK_SIZE_M": 16,
|
|
20
|
+
"BLOCK_SIZE_N": 64,
|
|
21
|
+
"BLOCK_SIZE_K": 128,
|
|
22
|
+
"GROUP_SIZE_M": 16,
|
|
23
|
+
"num_warps": 4,
|
|
24
|
+
"num_stages": 2
|
|
25
|
+
},
|
|
26
|
+
"8": {
|
|
27
|
+
"BLOCK_SIZE_M": 16,
|
|
28
|
+
"BLOCK_SIZE_N": 64,
|
|
29
|
+
"BLOCK_SIZE_K": 128,
|
|
30
|
+
"GROUP_SIZE_M": 1,
|
|
31
|
+
"num_warps": 4,
|
|
32
|
+
"num_stages": 3
|
|
33
|
+
},
|
|
34
|
+
"16": {
|
|
35
|
+
"BLOCK_SIZE_M": 16,
|
|
36
|
+
"BLOCK_SIZE_N": 64,
|
|
37
|
+
"BLOCK_SIZE_K": 256,
|
|
38
|
+
"GROUP_SIZE_M": 1,
|
|
39
|
+
"num_warps": 4,
|
|
40
|
+
"num_stages": 2
|
|
41
|
+
},
|
|
42
|
+
"24": {
|
|
43
|
+
"BLOCK_SIZE_M": 16,
|
|
44
|
+
"BLOCK_SIZE_N": 64,
|
|
45
|
+
"BLOCK_SIZE_K": 128,
|
|
46
|
+
"GROUP_SIZE_M": 1,
|
|
47
|
+
"num_warps": 4,
|
|
48
|
+
"num_stages": 4
|
|
49
|
+
},
|
|
50
|
+
"32": {
|
|
51
|
+
"BLOCK_SIZE_M": 16,
|
|
52
|
+
"BLOCK_SIZE_N": 64,
|
|
53
|
+
"BLOCK_SIZE_K": 128,
|
|
54
|
+
"GROUP_SIZE_M": 16,
|
|
55
|
+
"num_warps": 4,
|
|
56
|
+
"num_stages": 2
|
|
57
|
+
},
|
|
58
|
+
"48": {
|
|
59
|
+
"BLOCK_SIZE_M": 16,
|
|
60
|
+
"BLOCK_SIZE_N": 128,
|
|
61
|
+
"BLOCK_SIZE_K": 128,
|
|
62
|
+
"GROUP_SIZE_M": 16,
|
|
63
|
+
"num_warps": 4,
|
|
64
|
+
"num_stages": 3
|
|
65
|
+
},
|
|
66
|
+
"64": {
|
|
67
|
+
"BLOCK_SIZE_M": 16,
|
|
68
|
+
"BLOCK_SIZE_N": 256,
|
|
69
|
+
"BLOCK_SIZE_K": 128,
|
|
70
|
+
"GROUP_SIZE_M": 1,
|
|
71
|
+
"num_warps": 8,
|
|
72
|
+
"num_stages": 2
|
|
73
|
+
},
|
|
74
|
+
"96": {
|
|
75
|
+
"BLOCK_SIZE_M": 16,
|
|
76
|
+
"BLOCK_SIZE_N": 128,
|
|
77
|
+
"BLOCK_SIZE_K": 128,
|
|
78
|
+
"GROUP_SIZE_M": 1,
|
|
79
|
+
"num_warps": 4,
|
|
80
|
+
"num_stages": 2
|
|
81
|
+
},
|
|
82
|
+
"128": {
|
|
83
|
+
"BLOCK_SIZE_M": 16,
|
|
84
|
+
"BLOCK_SIZE_N": 128,
|
|
85
|
+
"BLOCK_SIZE_K": 128,
|
|
86
|
+
"GROUP_SIZE_M": 1,
|
|
87
|
+
"num_warps": 4,
|
|
88
|
+
"num_stages": 2
|
|
89
|
+
},
|
|
90
|
+
"256": {
|
|
91
|
+
"BLOCK_SIZE_M": 32,
|
|
92
|
+
"BLOCK_SIZE_N": 256,
|
|
93
|
+
"BLOCK_SIZE_K": 128,
|
|
94
|
+
"GROUP_SIZE_M": 16,
|
|
95
|
+
"num_warps": 4,
|
|
96
|
+
"num_stages": 2
|
|
97
|
+
},
|
|
98
|
+
"512": {
|
|
99
|
+
"BLOCK_SIZE_M": 64,
|
|
100
|
+
"BLOCK_SIZE_N": 128,
|
|
101
|
+
"BLOCK_SIZE_K": 64,
|
|
102
|
+
"GROUP_SIZE_M": 1,
|
|
103
|
+
"num_warps": 4,
|
|
104
|
+
"num_stages": 3
|
|
105
|
+
},
|
|
106
|
+
"1024": {
|
|
107
|
+
"BLOCK_SIZE_M": 128,
|
|
108
|
+
"BLOCK_SIZE_N": 256,
|
|
109
|
+
"BLOCK_SIZE_K": 64,
|
|
110
|
+
"GROUP_SIZE_M": 16,
|
|
111
|
+
"num_warps": 8,
|
|
112
|
+
"num_stages": 4
|
|
113
|
+
},
|
|
114
|
+
"1536": {
|
|
115
|
+
"BLOCK_SIZE_M": 128,
|
|
116
|
+
"BLOCK_SIZE_N": 256,
|
|
117
|
+
"BLOCK_SIZE_K": 64,
|
|
118
|
+
"GROUP_SIZE_M": 1,
|
|
119
|
+
"num_warps": 8,
|
|
120
|
+
"num_stages": 4
|
|
121
|
+
},
|
|
122
|
+
"2048": {
|
|
123
|
+
"BLOCK_SIZE_M": 128,
|
|
124
|
+
"BLOCK_SIZE_N": 256,
|
|
125
|
+
"BLOCK_SIZE_K": 64,
|
|
126
|
+
"GROUP_SIZE_M": 1,
|
|
127
|
+
"num_warps": 8,
|
|
128
|
+
"num_stages": 4
|
|
129
|
+
},
|
|
130
|
+
"3072": {
|
|
131
|
+
"BLOCK_SIZE_M": 128,
|
|
132
|
+
"BLOCK_SIZE_N": 256,
|
|
133
|
+
"BLOCK_SIZE_K": 64,
|
|
134
|
+
"GROUP_SIZE_M": 1,
|
|
135
|
+
"num_warps": 8,
|
|
136
|
+
"num_stages": 4
|
|
137
|
+
},
|
|
138
|
+
"4096": {
|
|
139
|
+
"BLOCK_SIZE_M": 128,
|
|
140
|
+
"BLOCK_SIZE_N": 256,
|
|
141
|
+
"BLOCK_SIZE_K": 64,
|
|
142
|
+
"GROUP_SIZE_M": 16,
|
|
143
|
+
"num_warps": 8,
|
|
144
|
+
"num_stages": 4
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
{
|
|
2
|
+
"1": {
|
|
3
|
+
"BLOCK_SIZE_M": 16,
|
|
4
|
+
"BLOCK_SIZE_N": 64,
|
|
5
|
+
"BLOCK_SIZE_K": 64,
|
|
6
|
+
"GROUP_SIZE_M": 16,
|
|
7
|
+
"num_warps": 4,
|
|
8
|
+
"num_stages": 4
|
|
9
|
+
},
|
|
10
|
+
"2": {
|
|
11
|
+
"BLOCK_SIZE_M": 16,
|
|
12
|
+
"BLOCK_SIZE_N": 64,
|
|
13
|
+
"BLOCK_SIZE_K": 64,
|
|
14
|
+
"GROUP_SIZE_M": 1,
|
|
15
|
+
"num_warps": 4,
|
|
16
|
+
"num_stages": 4
|
|
17
|
+
},
|
|
18
|
+
"4": {
|
|
19
|
+
"BLOCK_SIZE_M": 16,
|
|
20
|
+
"BLOCK_SIZE_N": 128,
|
|
21
|
+
"BLOCK_SIZE_K": 64,
|
|
22
|
+
"GROUP_SIZE_M": 1,
|
|
23
|
+
"num_warps": 4,
|
|
24
|
+
"num_stages": 4
|
|
25
|
+
},
|
|
26
|
+
"8": {
|
|
27
|
+
"BLOCK_SIZE_M": 16,
|
|
28
|
+
"BLOCK_SIZE_N": 64,
|
|
29
|
+
"BLOCK_SIZE_K": 64,
|
|
30
|
+
"GROUP_SIZE_M": 64,
|
|
31
|
+
"num_warps": 4,
|
|
32
|
+
"num_stages": 4
|
|
33
|
+
},
|
|
34
|
+
"16": {
|
|
35
|
+
"BLOCK_SIZE_M": 16,
|
|
36
|
+
"BLOCK_SIZE_N": 64,
|
|
37
|
+
"BLOCK_SIZE_K": 64,
|
|
38
|
+
"GROUP_SIZE_M": 1,
|
|
39
|
+
"num_warps": 4,
|
|
40
|
+
"num_stages": 3
|
|
41
|
+
},
|
|
42
|
+
"24": {
|
|
43
|
+
"BLOCK_SIZE_M": 16,
|
|
44
|
+
"BLOCK_SIZE_N": 64,
|
|
45
|
+
"BLOCK_SIZE_K": 64,
|
|
46
|
+
"GROUP_SIZE_M": 1,
|
|
47
|
+
"num_warps": 4,
|
|
48
|
+
"num_stages": 3
|
|
49
|
+
},
|
|
50
|
+
"32": {
|
|
51
|
+
"BLOCK_SIZE_M": 16,
|
|
52
|
+
"BLOCK_SIZE_N": 128,
|
|
53
|
+
"BLOCK_SIZE_K": 64,
|
|
54
|
+
"GROUP_SIZE_M": 1,
|
|
55
|
+
"num_warps": 4,
|
|
56
|
+
"num_stages": 3
|
|
57
|
+
},
|
|
58
|
+
"48": {
|
|
59
|
+
"BLOCK_SIZE_M": 16,
|
|
60
|
+
"BLOCK_SIZE_N": 64,
|
|
61
|
+
"BLOCK_SIZE_K": 64,
|
|
62
|
+
"GROUP_SIZE_M": 1,
|
|
63
|
+
"num_warps": 4,
|
|
64
|
+
"num_stages": 3
|
|
65
|
+
},
|
|
66
|
+
"64": {
|
|
67
|
+
"BLOCK_SIZE_M": 16,
|
|
68
|
+
"BLOCK_SIZE_N": 64,
|
|
69
|
+
"BLOCK_SIZE_K": 64,
|
|
70
|
+
"GROUP_SIZE_M": 1,
|
|
71
|
+
"num_warps": 4,
|
|
72
|
+
"num_stages": 3
|
|
73
|
+
},
|
|
74
|
+
"96": {
|
|
75
|
+
"BLOCK_SIZE_M": 16,
|
|
76
|
+
"BLOCK_SIZE_N": 64,
|
|
77
|
+
"BLOCK_SIZE_K": 64,
|
|
78
|
+
"GROUP_SIZE_M": 1,
|
|
79
|
+
"num_warps": 4,
|
|
80
|
+
"num_stages": 3
|
|
81
|
+
},
|
|
82
|
+
"128": {
|
|
83
|
+
"BLOCK_SIZE_M": 16,
|
|
84
|
+
"BLOCK_SIZE_N": 64,
|
|
85
|
+
"BLOCK_SIZE_K": 64,
|
|
86
|
+
"GROUP_SIZE_M": 1,
|
|
87
|
+
"num_warps": 4,
|
|
88
|
+
"num_stages": 3
|
|
89
|
+
},
|
|
90
|
+
"256": {
|
|
91
|
+
"BLOCK_SIZE_M": 16,
|
|
92
|
+
"BLOCK_SIZE_N": 128,
|
|
93
|
+
"BLOCK_SIZE_K": 64,
|
|
94
|
+
"GROUP_SIZE_M": 1,
|
|
95
|
+
"num_warps": 4,
|
|
96
|
+
"num_stages": 3
|
|
97
|
+
},
|
|
98
|
+
"512": {
|
|
99
|
+
"BLOCK_SIZE_M": 64,
|
|
100
|
+
"BLOCK_SIZE_N": 64,
|
|
101
|
+
"BLOCK_SIZE_K": 64,
|
|
102
|
+
"GROUP_SIZE_M": 1,
|
|
103
|
+
"num_warps": 4,
|
|
104
|
+
"num_stages": 2
|
|
105
|
+
},
|
|
106
|
+
"1024": {
|
|
107
|
+
"BLOCK_SIZE_M": 64,
|
|
108
|
+
"BLOCK_SIZE_N": 64,
|
|
109
|
+
"BLOCK_SIZE_K": 64,
|
|
110
|
+
"GROUP_SIZE_M": 1,
|
|
111
|
+
"num_warps": 4,
|
|
112
|
+
"num_stages": 2
|
|
113
|
+
},
|
|
114
|
+
"1536": {
|
|
115
|
+
"BLOCK_SIZE_M": 64,
|
|
116
|
+
"BLOCK_SIZE_N": 64,
|
|
117
|
+
"BLOCK_SIZE_K": 64,
|
|
118
|
+
"GROUP_SIZE_M": 1,
|
|
119
|
+
"num_warps": 4,
|
|
120
|
+
"num_stages": 2
|
|
121
|
+
},
|
|
122
|
+
"2048": {
|
|
123
|
+
"BLOCK_SIZE_M": 64,
|
|
124
|
+
"BLOCK_SIZE_N": 64,
|
|
125
|
+
"BLOCK_SIZE_K": 64,
|
|
126
|
+
"GROUP_SIZE_M": 1,
|
|
127
|
+
"num_warps": 4,
|
|
128
|
+
"num_stages": 2
|
|
129
|
+
},
|
|
130
|
+
"3072": {
|
|
131
|
+
"BLOCK_SIZE_M": 64,
|
|
132
|
+
"BLOCK_SIZE_N": 64,
|
|
133
|
+
"BLOCK_SIZE_K": 64,
|
|
134
|
+
"GROUP_SIZE_M": 1,
|
|
135
|
+
"num_warps": 4,
|
|
136
|
+
"num_stages": 2
|
|
137
|
+
},
|
|
138
|
+
"4096": {
|
|
139
|
+
"BLOCK_SIZE_M": 64,
|
|
140
|
+
"BLOCK_SIZE_N": 64,
|
|
141
|
+
"BLOCK_SIZE_K": 64,
|
|
142
|
+
"GROUP_SIZE_M": 1,
|
|
143
|
+
"num_warps": 4,
|
|
144
|
+
"num_stages": 2
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -45,7 +45,7 @@ if _is_cuda or _is_hip:
|
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
logger = logging.getLogger(__name__)
|
|
48
|
-
padding_size = 128 if bool(int(os.getenv("
|
|
48
|
+
padding_size = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0
|
|
49
49
|
enable_moe_align_block_size_triton = bool(
|
|
50
50
|
int(os.getenv("ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
|
|
51
51
|
)
|
|
@@ -940,10 +940,12 @@ def get_moe_configs(
|
|
|
940
940
|
)
|
|
941
941
|
if os.path.exists(config_file_path):
|
|
942
942
|
with open(config_file_path) as f:
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
943
|
+
# Please note that although we find the config files, performance might still be suboptimal.
|
|
944
|
+
# This is because the tuning environment might differ from your current environment.
|
|
945
|
+
# For example, updating the Triton version might cause all old configs to become suboptimal.
|
|
946
|
+
# To achieve the best performance, consider re-tuning the Triton fused MOE kernel in your environment.
|
|
947
|
+
# For the tuning method, refer to: https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton
|
|
948
|
+
logger.info("Using MoE kernel config from %s.", config_file_path)
|
|
947
949
|
# If a configuration has been found, return it
|
|
948
950
|
return {int(key): val for key, val in json.load(f).items()}
|
|
949
951
|
|
|
@@ -951,8 +953,8 @@ def get_moe_configs(
|
|
|
951
953
|
# configuration
|
|
952
954
|
logger.warning(
|
|
953
955
|
(
|
|
954
|
-
"Using default MoE config. Performance might be sub-optimal! "
|
|
955
|
-
"Config file not found at %s, you can
|
|
956
|
+
"Using default MoE kernel config. Performance might be sub-optimal! "
|
|
957
|
+
"Config file not found at %s, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton"
|
|
956
958
|
),
|
|
957
959
|
config_file_path,
|
|
958
960
|
)
|
|
@@ -1327,7 +1329,7 @@ def fused_experts_impl(
|
|
|
1327
1329
|
if (
|
|
1328
1330
|
not (use_fp8_w8a8 or use_int8_w8a8)
|
|
1329
1331
|
or block_shape is not None
|
|
1330
|
-
or (_is_hip and get_bool_env_var("
|
|
1332
|
+
or (_is_hip and get_bool_env_var("SGLANG_AITER_MOE"))
|
|
1331
1333
|
):
|
|
1332
1334
|
padded_size = 0
|
|
1333
1335
|
|
|
@@ -18,7 +18,7 @@ from sglang.srt.layers.quantization.base_config import (
|
|
|
18
18
|
QuantizationConfig,
|
|
19
19
|
QuantizeMethodBase,
|
|
20
20
|
)
|
|
21
|
-
from sglang.srt.utils import get_bool_env_var, is_hip,
|
|
21
|
+
from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs
|
|
22
22
|
|
|
23
23
|
if torch.cuda.is_available():
|
|
24
24
|
from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_experts
|
|
@@ -30,7 +30,9 @@ import logging
|
|
|
30
30
|
_is_hip = is_hip()
|
|
31
31
|
|
|
32
32
|
if _is_hip:
|
|
33
|
-
from aiter import
|
|
33
|
+
from aiter import ActivationType
|
|
34
|
+
from aiter.fused_moe_bf16_asm import ck_moe_2stages
|
|
35
|
+
from aiter.ops.shuffle import shuffle_weight
|
|
34
36
|
|
|
35
37
|
logger = logging.getLogger(__name__)
|
|
36
38
|
|
|
@@ -102,14 +104,14 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
|
|
102
104
|
set_weight_attrs(w2_weight, extra_weight_attrs)
|
|
103
105
|
|
|
104
106
|
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
|
105
|
-
if _is_hip and get_bool_env_var("
|
|
107
|
+
if _is_hip and get_bool_env_var("SGLANG_AITER_MOE"):
|
|
106
108
|
layer.w13_weight = torch.nn.Parameter(
|
|
107
|
-
|
|
109
|
+
shuffle_weight(layer.w13_weight.data, (16, 16)),
|
|
108
110
|
requires_grad=False,
|
|
109
111
|
)
|
|
110
112
|
torch.cuda.empty_cache()
|
|
111
113
|
layer.w2_weight = torch.nn.Parameter(
|
|
112
|
-
|
|
114
|
+
shuffle_weight(layer.w2_weight.data, (16, 16)),
|
|
113
115
|
requires_grad=False,
|
|
114
116
|
)
|
|
115
117
|
torch.cuda.empty_cache()
|
|
@@ -182,21 +184,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
|
|
182
184
|
routed_scaling_factor=routed_scaling_factor,
|
|
183
185
|
)
|
|
184
186
|
|
|
185
|
-
if _is_hip and get_bool_env_var("
|
|
187
|
+
if _is_hip and get_bool_env_var("SGLANG_AITER_MOE"):
|
|
186
188
|
assert not no_combine, "unsupported"
|
|
187
|
-
return
|
|
189
|
+
return ck_moe_2stages(
|
|
188
190
|
x,
|
|
189
191
|
layer.w13_weight,
|
|
190
192
|
layer.w2_weight,
|
|
191
193
|
topk_weights,
|
|
192
194
|
topk_ids,
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
None,
|
|
197
|
-
32,
|
|
198
|
-
None,
|
|
199
|
-
activation,
|
|
195
|
+
activation=(
|
|
196
|
+
ActivationType.Silu if activation == "silu" else ActivationType.Gelu
|
|
197
|
+
),
|
|
200
198
|
)
|
|
201
199
|
else:
|
|
202
200
|
return fused_experts(
|
|
@@ -527,7 +525,7 @@ class FusedMoE(torch.nn.Module):
|
|
|
527
525
|
# Case input scale: input_scale loading is only supported for fp8
|
|
528
526
|
if "input_scale" in weight_name:
|
|
529
527
|
# INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust input_scale for e4m3fnuz (AMD)
|
|
530
|
-
if _is_hip and get_bool_env_var("
|
|
528
|
+
if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"):
|
|
531
529
|
loaded_weight = loaded_weight * 2.0
|
|
532
530
|
|
|
533
531
|
# this is needed for compressed-tensors only
|
|
@@ -569,7 +567,7 @@ class FusedMoE(torch.nn.Module):
|
|
|
569
567
|
quant_method = getattr(param, "quant_method", None)
|
|
570
568
|
if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value:
|
|
571
569
|
# INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust INT4 column-wise scaling number to e4m3fnuz (AMD)
|
|
572
|
-
if _is_hip and get_bool_env_var("
|
|
570
|
+
if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"):
|
|
573
571
|
loaded_weight = loaded_weight * 0.5
|
|
574
572
|
|
|
575
573
|
self._load_per_channel_weight_scale(
|
|
@@ -592,7 +590,7 @@ class FusedMoE(torch.nn.Module):
|
|
|
592
590
|
)
|
|
593
591
|
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
|
|
594
592
|
# INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust FP8 per-tensor scaling number for e4m3fnuz (AMD)
|
|
595
|
-
if _is_hip and get_bool_env_var("
|
|
593
|
+
if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"):
|
|
596
594
|
loaded_weight = loaded_weight * 2.0
|
|
597
595
|
|
|
598
596
|
self._load_per_tensor_weight_scale(
|
|
@@ -109,7 +109,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
|
|
|
109
109
|
if quantization in VLLM_QUANTIZATION_METHODS and not VLLM_AVAILABLE:
|
|
110
110
|
raise ValueError(
|
|
111
111
|
f"{quantization} quantization requires some operators from vllm. "
|
|
112
|
-
"Pleaes install vllm by `pip install vllm==0.
|
|
112
|
+
"Pleaes install vllm by `pip install vllm==0.8.4`"
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
return QUANTIZATION_METHODS[quantization]
|
|
@@ -310,7 +310,7 @@ def monkey_patch_moe_apply(class_obj: "FusedMoEMethodBase"):
|
|
|
310
310
|
if correction_bias is not None:
|
|
311
311
|
if not has_correction_bias:
|
|
312
312
|
raise ValueError(
|
|
313
|
-
"Please increase the version of your vllm. Try `pip install vllm==0.
|
|
313
|
+
"Please increase the version of your vllm. Try `pip install vllm==0.8.4`"
|
|
314
314
|
)
|
|
315
315
|
kwargs["e_score_correction_bias"] = correction_bias
|
|
316
316
|
return original_apply(**kwargs)
|
|
@@ -293,7 +293,7 @@ def _maybe_compile_deep_gemm_one_type_all(
|
|
|
293
293
|
logger.info(
|
|
294
294
|
f"Try DeepGEMM JIT Compiling for "
|
|
295
295
|
f"<{kernel_helper.name}> N={n}, K={k}, num_groups={num_groups} with all Ms."
|
|
296
|
-
f"{' It only takes a litte time(
|
|
296
|
+
f"{' It only takes a litte time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. ' if not _IN_PRECOMPILE_STAGE else ''}"
|
|
297
297
|
)
|
|
298
298
|
|
|
299
299
|
# NOTE(alcanderian): get_num_sms should be change when 2-batch-overlap is introduced
|