sglang 0.5.2rc2__py3-none-any.whl → 0.5.3rc0__py3-none-any.whl

sglang/test/test_fp4_moe.py

@@ -3,12 +3,15 @@ from typing import Callable
 
 import pytest
 import torch
+from flashinfer import fp4_quantize
 from flashinfer.fused_moe import cutlass_fused_moe as flashinfer_cutlass_fused_moe
-from sgl_kernel import scaled_fp4_quant
+from sgl_kernel import scaled_fp4_grouped_quant, scaled_fp4_quant
+from torch.nn import functional as F
 
 from sglang.srt.layers.activation import SiluAndMul
 from sglang.srt.layers.moe.cutlass_moe import cutlass_moe_fp4
 from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType
+from sglang.srt.layers.moe.flashinfer_cutedsl_moe import flashinfer_cutedsl_moe_masked
 from sglang.srt.layers.moe.topk import TopKConfig, select_experts
 
 if torch.cuda.get_device_capability() < (10, 0):
@@ -78,6 +81,37 @@ def break_fp4_bytes(a, dtype):
     return values.reshape(m, n * 2).to(dtype=dtype)
 
 
+def compute_routing(router_logits: torch.Tensor, top_k: int):
+    routing_weights = torch.softmax(router_logits, dim=1, dtype=torch.float)
+    routing_weights, selected_experts = torch.topk(routing_weights, top_k, dim=-1)
+    routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
+    routing_weights = routing_weights.float()
+    return routing_weights, selected_experts
+
+
+def prepare_inputs(
+    hidden_states: torch.Tensor,
+    router_logits: torch.Tensor,
+    num_experts: int,
+    topk: int,
+):
+    routing_weights, topk_idx = compute_routing(router_logits, topk)
+
+    masked_m = []
+    for i in range(num_experts):
+        mask = topk_idx.view(-1) == i
+        masked_m.append(mask.sum())
+
+    masked_m = torch.tensor(masked_m, dtype=torch.int32)
+    hidden_states_3d = torch.empty(
+        (num_experts, max(masked_m), hidden_states.shape[1]), dtype=hidden_states.dtype
+    )
+    for i in range(num_experts):
+        hidden_states_3d[i, : masked_m[i], :] = hidden_states[topk_idx.view(-1) == i]
+
+    return hidden_states_3d, masked_m, topk_idx, routing_weights
+
+
 MNK_FACTORS = [
     (2, 1024, 1024),
     (2, 1024, 1536),
@@ -114,6 +148,99 @@ def torch_moe(a, w1, w2, score, topk, expert_map):
     ).sum(dim=1)
 
 
+def torch_moe_nvfp4(a, w1, w2, topk, topk_weight, topk_ids):
+    B, D = a.shape
+    a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
+    out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
+
+    topk_weight = topk_weight.view(-1)
+    topk_ids = topk_ids.view(-1)
+
+    for i in range(w1.shape[0]):
+        mask = topk_ids == i
+        if mask.sum():
+            m = w1[i].shape[0]
+            assert m % 2 == 0
+            # Note: w1 and w3 are swapped!
+            w3_expert, w1_expert = w1[i][m // 2 :, :], w1[i][: m // 2, :]
+            inter = F.silu(a[mask] @ w1_expert.t()) * (a[mask] @ w3_expert.t())
+            inter_gs = torch.tensor(1.0).cuda()
+            inter_q, inter_blockscale = fp4_quantize(inter, inter_gs)
+            inter = dequantize_nvfp4_to_dtype(
+                inter_q,
+                inter_blockscale,
+                inter_gs,
+                dtype=inter.dtype,
+                device=inter.device,
+                block_size=16,
+            ).cuda()
+            out[mask] = inter @ w2[i].transpose(0, 1)
+    return (
+        out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
+    ).sum(dim=1)
+
+
+def flashinfer_cutedsl_grouped_gemm_nt_masked(
+    hidden_states: torch.Tensor,  # 3d
+    input_global_scale: torch.Tensor,  # (l,)
+    weights: torch.Tensor,
+    w_global_scale: torch.Tensor,  # (l,)
+    masked_m: torch.Tensor,
+):
+    from flashinfer.cute_dsl.blockscaled_gemm import grouped_gemm_nt_masked
+
+    # hidden_states: [l, m, k]
+    # weights: [l, n, k]
+    aq, aq_sf = scaled_fp4_grouped_quant(
+        hidden_states,
+        input_global_scale,
+        masked_m.to(hidden_states.device),
+    )
+    num_experts, n, k = weights.shape
+    bq, bq_sf = scaled_fp4_grouped_quant(
+        weights,
+        w_global_scale,
+        torch.ones(num_experts, device=weights.device, dtype=torch.int32) * n,
+    )
+
+    out = torch.zeros(
+        (num_experts, max(masked_m), n), dtype=weights.dtype, device=aq.device
+    )
+    out = out.permute(1, 2, 0)  # requirement of kernel
+    sf_vec_size = 16
+    ab_dtype = "float4_e2m1fn"
+    sf_dtype = "float8_e4m3fn"
+    c_dtype = "bfloat16"
+    alpha = 1.0 / (input_global_scale * w_global_scale).to(out.dtype).view(
+        1, 1, num_experts
+    )
+
+    def get_cute_dtype(input: torch.Tensor) -> str:
+        if input.dtype == torch.bfloat16:
+            return "bfloat16"
+        elif input.dtype == torch.float16:
+            return "float16"
+        elif input.dtype == torch.float32:
+            return "float32"
+        else:
+            raise ValueError(f"Unsupported cute dtype {input.dtype}")
+
+    grouped_gemm_nt_masked(
+        (aq, aq_sf),
+        (bq, bq_sf),
+        out,
+        masked_m.to(aq.device),
+        ab_dtype=ab_dtype,
+        sf_dtype=sf_dtype,
+        c_dtype=c_dtype,
+        sf_vec_size=sf_vec_size,
+        alpha=alpha,
+        alpha_dtype=get_cute_dtype(alpha),
+    )
+
+    return out
+
+
 def check_moe(
     m: int,
     n: int,
@@ -324,6 +451,248 @@ def test_flashinfer_fp4_moe_no_graph(
     check_moe(m, n, k, e, topk, dtype, flashinfer_moe_impl, flip_w13=True)
 
 
+@pytest.mark.parametrize("bs, hidden_dim, inter_dim", [(2, 128, 256), (16, 128, 512)])
+@pytest.mark.parametrize("topk", [1, 2, 4])
+@torch.inference_mode()
+def test_flashinfer_cutedsl_moe_masked(
+    bs: int, hidden_dim: int, inter_dim: int, topk: int
+):
+    torch.manual_seed(42)
+    device = "cuda"
+    dtype = torch.bfloat16
+    num_experts = 8
+    hidden_states = (
+        torch.randn(bs, hidden_dim, dtype=torch.bfloat16, device=device) / 5.0
+    )
+    w1 = (
+        torch.randn(
+            num_experts, 2 * inter_dim, hidden_dim, dtype=torch.bfloat16, device=device
+        )
+        / 10.0
+    )
+    w2 = (
+        torch.randn(
+            num_experts, hidden_dim, inter_dim, dtype=torch.bfloat16, device=device
+        )
+        / 10.0
+    )
+    router_logits = torch.randn(bs, num_experts, dtype=torch.float32)
+
+    hidden_states_expanded = (
+        hidden_states.view(bs, -1, hidden_dim)
+        .repeat(1, topk, 1)
+        .reshape(-1, hidden_dim)
+    )
+    hidden_states_3d, masked_m, topk_idx, routing_weights = prepare_inputs(
+        hidden_states_expanded, router_logits, num_experts, topk
+    )
+
+    w1_amax = w1.abs().amax(dim=(1, 2)).to(torch.float32).to(w1.device)
+    w2_amax = w2.abs().amax(dim=(1, 2)).to(torch.float32).to(w2.device)
+    input_global_scale = torch.ones(
+        (num_experts,), dtype=torch.float32, device=hidden_states.device
+    )
+
+    w1_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax
+    w2_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax
+    a2_global_scale = torch.ones(
+        (num_experts,), dtype=torch.float32, device=hidden_states.device
+    )  # assume intermediate scale is 1.0
+
+    w1_fp4, w1_blockscale = scaled_fp4_grouped_quant(
+        w1,
+        w1_global_scale,
+        torch.ones(num_experts, dtype=torch.int32, device=w1.device) * 2 * inter_dim,
+    )
+    w2_fp4, w2_blockscale = scaled_fp4_grouped_quant(
+        w2,
+        w2_global_scale,
+        torch.ones(num_experts, dtype=torch.int32, device=w2.device) * hidden_dim,
+    )
+
+    w1_alpha = 1.0 / (input_global_scale * w1_global_scale)
+    w2_alpha = 1.0 / (a2_global_scale * w2_global_scale)
+
+    out = flashinfer_cutedsl_moe_masked(
+        hidden_states_3d.to(hidden_states.device),
+        input_global_scale,
+        w1_fp4.permute(2, 0, 1),
+        w1_blockscale,
+        w1_alpha,
+        w2_fp4.permute(2, 0, 1),
+        a2_global_scale,
+        w2_blockscale,
+        w2_alpha,
+        masked_m.to(hidden_states.device),
+    )
+
+    # reference
+    a_fp4, a_scale_interleaved = fp4_quantize(hidden_states, input_global_scale)
+    a_in_dtype = dequantize_nvfp4_to_dtype(
+        a_fp4,
+        a_scale_interleaved,
+        input_global_scale,
+        dtype=hidden_states.dtype,
+        device=hidden_states.device,
+        block_size=16,
+    )
+    w1_d = torch.empty(
+        (num_experts, 2 * inter_dim, hidden_dim), device=w1.device, dtype=w1.dtype
+    )
+    w2_d = torch.empty(
+        (num_experts, hidden_dim, inter_dim), device=w2.device, dtype=w2.dtype
+    )
+
+    for idx in range(0, num_experts):
+        w1_fp4_sliced, w1_blockscale_sliced = fp4_quantize(
+            w1[idx], w1_global_scale[idx]
+        )
+        w2_fp4_sliced, w2_blockscale_sliced = fp4_quantize(
+            w2[idx], w2_global_scale[idx]
+        )
+        w1_d[idx] = dequantize_nvfp4_to_dtype(
+            w1_fp4_sliced,
+            w1_blockscale_sliced,
+            w1_global_scale[idx],
+            dtype=w1.dtype,
+            device=w1.device,
+            block_size=16,
+        )
+        w2_d[idx] = dequantize_nvfp4_to_dtype(
+            w2_fp4_sliced,
+            w2_blockscale_sliced,
+            w2_global_scale[idx],
+            dtype=w2.dtype,
+            device=w2.device,
+            block_size=16,
+        )
+
+    ref_output = torch_moe_nvfp4(
+        a_in_dtype,
+        w1_d,
+        w2_d,
+        topk,
+        routing_weights.to(a_in_dtype.device),
+        topk_idx.to(a_in_dtype.device),
+    )
+    out_weighted = torch.zeros_like(ref_output, device=out.device, dtype=out.dtype)
+
+    positions = torch.nonzero(masked_m[topk_idx], as_tuple=False)
+    rows, cols = positions[:, 0], positions[:, 1]
+    experts = topk_idx[rows, cols]
+    for i in range(num_experts):
+        mask = experts == i
+        if mask.any():
+            idx = torch.nonzero(mask, as_tuple=False).squeeze(-1)
+            r, c = rows[idx], cols[idx]
+            out_weighted[r] += out[i, : len(r), :] * routing_weights[r, c].to(
+                out.device
+            ).unsqueeze(-1)
+    torch.testing.assert_close(
+        out_weighted.cpu(), ref_output.cpu(), atol=5e-2, rtol=5e-2
+    )
+
+
+@pytest.mark.parametrize(
+    "bs, hidden_dim, inter_dim, topk", [(2, 128, 256, 2), (16, 128, 512, 5)]
+)
+@torch.inference_mode()
+def test_grouped_gemm_nt_masked(
+    bs: int, hidden_dim: int, inter_dim: int, topk: int
+) -> None:
+    torch.manual_seed(42)
+    B = bs
+    D = hidden_dim
+    N = inter_dim
+    num_experts = 8
+    hidden_states = torch.randn(B, D, dtype=torch.bfloat16, device="cuda")
+    weights = torch.randn(num_experts, N, D, dtype=torch.bfloat16, device="cuda")
+    router_logits = torch.randn(B, num_experts, dtype=torch.float32)
+
+    hidden_states_expanded = (
+        hidden_states.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
+    )
+    hidden_states_3d, masked_m, topk_idx, _ = prepare_inputs(
+        hidden_states_expanded, router_logits, num_experts, topk
+    )
+
+    # reference
+    out = torch.zeros(
+        (B * topk, weights.shape[1]), dtype=weights.dtype, device=weights.device
+    )
+    for i in range(num_experts):
+        mask = topk_idx.view(-1) == i
+        if mask.sum():
+            lhs = hidden_states_expanded[mask]
+            rhs = weights[i]
+            a_amax = lhs.abs().max().to(torch.float32).to(hidden_states.device)
+            b_amax = rhs.abs().amax().to(torch.float32).to(weights.device)
+            a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
+            b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
+
+            lhsq, lhsq_sf = fp4_quantize(
+                lhs,
+                a_gs,
+            )
+            rhsq, rhsq_sf = fp4_quantize(
+                rhs,
+                b_gs,
+            )
+
+            lhs_in_dtype = dequantize_nvfp4_to_dtype(
+                lhsq,
+                lhsq_sf,
+                a_gs,
+                dtype=hidden_states.dtype,
+                device=hidden_states.device,
+                block_size=16,
+            )
+
+            rhs_in_dtype = dequantize_nvfp4_to_dtype(
+                rhsq,
+                rhsq_sf,
+                b_gs,
+                dtype=hidden_states.dtype,
+                device=hidden_states.device,
+                block_size=16,
+            )
+            out[mask] = lhs_in_dtype @ rhs_in_dtype.t()
+
+    a_amax = (
+        hidden_states_3d.abs()
+        .amax(dim=(1, 2))
+        .to(torch.float32)
+        .to(hidden_states.device)
+    )
+    b_amax = weights.abs().amax(dim=(1, 2)).to(torch.float32).to(weights.device)
+    a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
+    b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
+    out_flashinfer = flashinfer_cutedsl_grouped_gemm_nt_masked(
+        hidden_states_3d.to(hidden_states.device), a_gs, weights, b_gs, masked_m
+    )
+
+    # re-pack out into [num_experts, max_m, n]
+    out_ref = torch.zeros(
+        (num_experts, max(masked_m), weights.shape[1]), dtype=out.dtype
+    )
+    expert_slot = [0] * num_experts
+    for i, expert_id in enumerate(topk_idx.view(-1).tolist()):
+        out_ref[expert_id, expert_slot[expert_id], :] = out[i]
+        expert_slot[expert_id] += 1
+
+    # Note: just to compare the masked position due to cutedsl may write nan
+    # into unmasked position.
+    for i in range(num_experts):
+        torch.testing.assert_close(
+            out_flashinfer.permute(2, 0, 1)[i, : masked_m[i]],
+            out_ref.to(out_flashinfer.device)[i, : masked_m[i]],
+            atol=1e-1,
+            rtol=5e-2,
+        )
+
+
 if __name__ == "__main__":
     test_cutlass_fp4_moe_no_graph(224, 1024, 1024, 256, 8, torch.half)
     test_flashinfer_fp4_moe_no_graph(224, 1024, 1024, 256, 8, torch.half)
+    test_flashinfer_cutedsl_moe_masked(16, 128, 512, 4)
+    test_grouped_gemm_nt_masked(16, 128, 512, 4)

sglang/test/test_utils.py

@@ -42,7 +42,8 @@ DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct"
 DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
 DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE = "meta-llama/Llama-3.2-1B"
 DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1"
-DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST = "Qwen/Qwen1.5-MoE-A2.7B"
+DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B"
+DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat"
 
 # MLA test models
 DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct"
@@ -52,6 +53,9 @@ DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instru
 DEFAULT_MODEL_NAME_FOR_TEST_MLA = "lmsys/sglang-ci-dsv3-test"
 DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = "lmsys/sglang-ci-dsv3-test-NextN"
 
+# NVFP4 models
+DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST = "nvidia/DeepSeek-R1-0528-FP4"
+
 # FP8 models
 DEFAULT_MODEL_NAME_FOR_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8"
 DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8"
@@ -72,6 +76,10 @@ DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE = "nytopop/Qwen3-30B-A3B.w8a8"
 DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf"
 DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B"
 DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B"
+DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST = (
+    "meta-llama/Llama-3.1-8B-Instruct"
+)
+DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
 
 # Other use cases
 DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION = (
@@ -466,6 +474,25 @@ def try_cached_model(model_repo: str):
     return model_dir if model_dir else model_repo
 
 
+def popen_with_error_check(command: list[str], allow_exit: bool = False):
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def _run_and_check():
+        stdout, stderr = process.communicate()
+
+        while process.poll() is None:
+            time.sleep(5)
+
+        if not allow_exit or process.returncode != 0:
+            raise Exception(
+                f"{command} exited with code {process.returncode}\n{stdout=}\n{stderr=}"
+            )
+
+    t = threading.Thread(target=_run_and_check)
+    t.start()
+    return process
+
+
 def popen_launch_server(
     model: str,
     base_url: str,

sglang/utils.py

@@ -472,11 +472,22 @@ def wait_for_server(base_url: str, timeout: int = None) -> None:
 class TypeBasedDispatcher:
     def __init__(self, mapping: List[Tuple[Type, Callable]]):
         self._mapping = mapping
+        self._fallback_fn = None
+
+    def add_fallback_fn(self, fallback_fn: Callable):
+        self._fallback_fn = fallback_fn
+
+    def __iadd__(self, other: "TypeBasedDispatcher"):
+        self._mapping.extend(other._mapping)
+        return self
 
     def __call__(self, obj: Any):
         for ty, fn in self._mapping:
             if isinstance(obj, ty):
                 return fn(obj)
+
+        if self._fallback_fn is not None:
+            return self._fallback_fn(obj)
         raise ValueError(f"Invalid object: {obj}")
 
 

sglang/version.py

@@ -1 +1 @@
-__version__ = "0.5.2rc2"
+__version__ = "0.5.3rc0"

{sglang-0.5.2rc2.dist-info → sglang-0.5.3rc0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: sglang
-Version: 0.5.2rc2
-Summary: SGLang is yet another fast serving framework for large language models and vision language models.
+Version: 0.5.3rc0
+Summary: SGLang is a fast serving framework for large language models and vision language models.
 License:                                  Apache License
                                    Version 2.0, January 2004
                                 http://www.apache.org/licenses/
@@ -217,84 +217,54 @@ Requires-Dist: tqdm
 Requires-Dist: numpy
 Requires-Dist: IPython
 Requires-Dist: setproctitle
-Provides-Extra: runtime-common
-Requires-Dist: blobfile==3.0.0; extra == "runtime-common"
-Requires-Dist: build; extra == "runtime-common"
-Requires-Dist: compressed-tensors; extra == "runtime-common"
-Requires-Dist: datasets; extra == "runtime-common"
-Requires-Dist: einops; extra == "runtime-common"
-Requires-Dist: fastapi; extra == "runtime-common"
-Requires-Dist: hf_transfer; extra == "runtime-common"
-Requires-Dist: huggingface_hub; extra == "runtime-common"
-Requires-Dist: interegular; extra == "runtime-common"
-Requires-Dist: llguidance<0.8.0,>=0.7.11; extra == "runtime-common"
-Requires-Dist: modelscope; extra == "runtime-common"
-Requires-Dist: msgspec; extra == "runtime-common"
-Requires-Dist: ninja; extra == "runtime-common"
-Requires-Dist: openai==1.99.1; extra == "runtime-common"
-Requires-Dist: openai-harmony==0.0.4; extra == "runtime-common"
-Requires-Dist: orjson; extra == "runtime-common"
-Requires-Dist: outlines==0.1.11; extra == "runtime-common"
-Requires-Dist: packaging; extra == "runtime-common"
-Requires-Dist: partial_json_parser; extra == "runtime-common"
-Requires-Dist: pillow; extra == "runtime-common"
-Requires-Dist: prometheus-client>=0.20.0; extra == "runtime-common"
-Requires-Dist: psutil; extra == "runtime-common"
-Requires-Dist: pybase64; extra == "runtime-common"
-Requires-Dist: pydantic; extra == "runtime-common"
-Requires-Dist: pynvml; extra == "runtime-common"
-Requires-Dist: python-multipart; extra == "runtime-common"
-Requires-Dist: pyzmq>=25.1.2; extra == "runtime-common"
-Requires-Dist: sentencepiece; extra == "runtime-common"
-Requires-Dist: soundfile==0.13.1; extra == "runtime-common"
-Requires-Dist: scipy; extra == "runtime-common"
-Requires-Dist: timm==1.0.16; extra == "runtime-common"
-Requires-Dist: tiktoken; extra == "runtime-common"
-Requires-Dist: torchao==0.9.0; extra == "runtime-common"
-Requires-Dist: transformers==4.56.0; extra == "runtime-common"
-Requires-Dist: uvicorn; extra == "runtime-common"
-Requires-Dist: uvloop; extra == "runtime-common"
-Requires-Dist: xgrammar==0.1.23; extra == "runtime-common"
-Provides-Extra: srt
-Requires-Dist: sglang[runtime_common]; extra == "srt"
-Requires-Dist: sgl-kernel==0.3.8; extra == "srt"
-Requires-Dist: torch==2.8.0; extra == "srt"
-Requires-Dist: torchaudio==2.8.0; extra == "srt"
-Requires-Dist: torchvision; extra == "srt"
-Requires-Dist: cuda-python; extra == "srt"
-Requires-Dist: flashinfer_python==0.3.0; extra == "srt"
-Provides-Extra: blackwell
-Requires-Dist: sglang[runtime_common]; extra == "blackwell"
-Requires-Dist: sgl-kernel; extra == "blackwell"
-Requires-Dist: torch==2.8.0; extra == "blackwell"
-Requires-Dist: torchaudio==2.8.0; extra == "blackwell"
-Requires-Dist: torchvision; extra == "blackwell"
-Requires-Dist: cuda-python; extra == "blackwell"
-Requires-Dist: flashinfer_python==0.3.0; extra == "blackwell"
-Provides-Extra: srt-hip
-Requires-Dist: sglang[runtime_common]; extra == "srt-hip"
-Requires-Dist: torch; extra == "srt-hip"
-Requires-Dist: petit_kernel==0.0.2; extra == "srt-hip"
-Requires-Dist: wave-lang==1.0.1; extra == "srt-hip"
-Provides-Extra: srt-cpu
-Requires-Dist: sglang[runtime_common]; extra == "srt-cpu"
-Provides-Extra: srt-npu
-Requires-Dist: sglang[runtime_common]; extra == "srt-npu"
-Provides-Extra: srt-xpu
-Requires-Dist: sglang[runtime_common]; extra == "srt-xpu"
-Provides-Extra: srt-hpu
-Requires-Dist: sglang[runtime_common]; extra == "srt-hpu"
-Provides-Extra: openai
-Requires-Dist: openai==1.99.1; extra == "openai"
-Requires-Dist: tiktoken; extra == "openai"
-Provides-Extra: anthropic
-Requires-Dist: anthropic>=0.20.0; extra == "anthropic"
-Provides-Extra: litellm
-Requires-Dist: litellm>=1.0.0; extra == "litellm"
-Provides-Extra: torch-memory-saver
-Requires-Dist: torch_memory_saver==0.0.8; extra == "torch-memory-saver"
-Provides-Extra: decord
-Requires-Dist: decord; extra == "decord"
+Requires-Dist: blobfile==3.0.0
+Requires-Dist: build
+Requires-Dist: compressed-tensors
+Requires-Dist: datasets
+Requires-Dist: einops
+Requires-Dist: fastapi
+Requires-Dist: hf_transfer
+Requires-Dist: huggingface_hub
+Requires-Dist: interegular
+Requires-Dist: llguidance<0.8.0,>=0.7.11
+Requires-Dist: modelscope
+Requires-Dist: msgspec
+Requires-Dist: ninja
+Requires-Dist: openai==1.99.1
+Requires-Dist: openai-harmony==0.0.4
+Requires-Dist: orjson
+Requires-Dist: outlines==0.1.11
+Requires-Dist: packaging
+Requires-Dist: partial_json_parser
+Requires-Dist: pillow
+Requires-Dist: prometheus-client>=0.20.0
+Requires-Dist: psutil
+Requires-Dist: pybase64
+Requires-Dist: pydantic
+Requires-Dist: pynvml
+Requires-Dist: python-multipart
+Requires-Dist: pyzmq>=25.1.2
+Requires-Dist: scipy
+Requires-Dist: sentencepiece
+Requires-Dist: soundfile==0.13.1
+Requires-Dist: timm==1.0.16
+Requires-Dist: tiktoken
+Requires-Dist: torchao==0.9.0
+Requires-Dist: transformers==4.56.1
+Requires-Dist: uvicorn
+Requires-Dist: uvloop
+Requires-Dist: xgrammar==0.1.24
+Requires-Dist: sgl-kernel==0.3.9.post2
+Requires-Dist: torch==2.8.0
+Requires-Dist: torchaudio==2.8.0
+Requires-Dist: torchvision
+Requires-Dist: cuda-python
+Requires-Dist: flashinfer_python==0.3.1
+Requires-Dist: openai==1.99.1
+Requires-Dist: tiktoken
+Requires-Dist: anthropic>=0.20.0
+Requires-Dist: torch_memory_saver==0.0.8
+Requires-Dist: decord
 Provides-Extra: test
 Requires-Dist: accelerate; extra == "test"
 Requires-Dist: expecttest; extra == "test"
@@ -305,52 +275,18 @@ Requires-Dist: peft; extra == "test"
 Requires-Dist: sentence_transformers; extra == "test"
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: tabulate; extra == "test"
+Provides-Extra: tracing
+Requires-Dist: opentelemetry-sdk; extra == "tracing"
+Requires-Dist: opentelemetry-api; extra == "tracing"
+Requires-Dist: opentelemetry-exporter-otlp; extra == "tracing"
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc; extra == "tracing"
 Provides-Extra: all
-Requires-Dist: sglang[srt]; extra == "all"
-Requires-Dist: sglang[openai]; extra == "all"
-Requires-Dist: sglang[anthropic]; extra == "all"
-Requires-Dist: sglang[torch_memory_saver]; extra == "all"
-Requires-Dist: sglang[decord]; extra == "all"
-Provides-Extra: all-hip
-Requires-Dist: sglang[srt_hip]; extra == "all-hip"
-Requires-Dist: sglang[openai]; extra == "all-hip"
-Requires-Dist: sglang[anthropic]; extra == "all-hip"
-Requires-Dist: sglang[decord]; extra == "all-hip"
-Provides-Extra: all-xpu
-Requires-Dist: sglang[srt_xpu]; extra == "all-xpu"
-Requires-Dist: sglang[openai]; extra == "all-xpu"
-Requires-Dist: sglang[anthropic]; extra == "all-xpu"
-Requires-Dist: sglang[decord]; extra == "all-xpu"
-Provides-Extra: all-hpu
-Requires-Dist: sglang[srt_hpu]; extra == "all-hpu"
-Requires-Dist: sglang[openai]; extra == "all-hpu"
-Requires-Dist: sglang[anthropic]; extra == "all-hpu"
-Requires-Dist: sglang[decord]; extra == "all-hpu"
-Provides-Extra: all-cpu
-Requires-Dist: sglang[srt_cpu]; extra == "all-cpu"
-Requires-Dist: sglang[openai]; extra == "all-cpu"
-Requires-Dist: sglang[anthropic]; extra == "all-cpu"
-Requires-Dist: sglang[decord]; extra == "all-cpu"
-Provides-Extra: all-npu
-Requires-Dist: sglang[srt_npu]; extra == "all-npu"
-Requires-Dist: sglang[openai]; extra == "all-npu"
-Requires-Dist: sglang[anthropic]; extra == "all-npu"
-Requires-Dist: sglang[decord]; extra == "all-npu"
+Requires-Dist: sglang[test]; extra == "all"
+Provides-Extra: blackwell
+Requires-Dist: nvidia-cutlass-dsl==4.1.0; extra == "blackwell"
+Requires-Dist: sglang[test]; extra == "blackwell"
 Provides-Extra: dev
-Requires-Dist: sglang[all]; extra == "dev"
 Requires-Dist: sglang[test]; extra == "dev"
-Provides-Extra: dev-hip
-Requires-Dist: sglang[all_hip]; extra == "dev-hip"
-Requires-Dist: sglang[test]; extra == "dev-hip"
-Provides-Extra: dev-xpu
-Requires-Dist: sglang[all_xpu]; extra == "dev-xpu"
-Requires-Dist: sglang[test]; extra == "dev-xpu"
-Provides-Extra: dev-hpu
-Requires-Dist: sglang[all_hpu]; extra == "dev-hpu"
-Requires-Dist: sglang[test]; extra == "dev-hpu"
-Provides-Extra: dev-cpu
-Requires-Dist: sglang[all_cpu]; extra == "dev-cpu"
-Requires-Dist: sglang[test]; extra == "dev-cpu"
 Dynamic: license-file
 
 <div align="center" id="sglangtop">