sglang 0.5.1.post2__py3-none-any.whl → 0.5.1.post3__py3-none-any.whl

sglang/srt/reasoning_parser.py

@@ -1,13 +1,19 @@
 import re
 from typing import Dict, Optional, Tuple, Type
 
+from sglang.srt.harmony_parser import HarmonyParser
+
 
 class StreamingParseResult:
     """Result of streaming incremental parsing."""
 
-    def __init__(self, normal_text: str = "", reasoning_text: str = ""):
-        self.normal_text = normal_text
-        self.reasoning_text = reasoning_text
+    def __init__(
+        self,
+        normal_text: Optional[str] = None,
+        reasoning_text: Optional[str] = None,
+    ):
+        self.normal_text = normal_text or ""
+        self.reasoning_text = reasoning_text or ""
 
 
 class BaseReasoningFormatDetector:
@@ -188,316 +194,60 @@ class KimiDetector(BaseReasoningFormatDetector):
 
 class GptOssDetector(BaseReasoningFormatDetector):
     """
-    Detector for T4-style reasoning format.
-
-    Assumes reasoning format with two channels:
-      <|channel|>analysis<|message|>...reasoning content...<|end|>
-      <|start|>assistant<|channel|>final<|message|>...final answer...<|return|>
-
-    Returns content from 'analysis' channel as reasoning_text
-    and content from 'final' channel as normal_text.
-
-    Args:
-        stream_reasoning (bool): If False, accumulates reasoning content until complete.
-            If True, streams reasoning content as it arrives.
+    Detector for T4-style reasoning format (GPT-OSS), using the HarmonyParser.
     """
 
     def __init__(self, stream_reasoning: bool = True, force_reasoning: bool = True):
-        # TypeScript uses channel tokens instead of simple start/end tokens
         super().__init__(
             "<|channel|>analysis<|message|>",
             "<|end|>",
-            force_reasoning=True,
+            force_reasoning=force_reasoning,
             stream_reasoning=stream_reasoning,
         )
-        self.final_channel_start = "<|start|>assistant<|channel|>final<|message|>"
-        self.final_channel_end = "<|return|>"
-        self._in_final_channel = False
-        self._analysis_complete = False
-        self._in_reasoning = True
+        self.parser = HarmonyParser()
 
     def detect_and_parse(self, text: str) -> StreamingParseResult:
-        """
-        One-time parsing: Detects and parses both analysis and final channels.
-        Tool call channels are preserved in normal_text for downstream processing.
+        events = self.parser.parse(text)
+        # Flush the buffer for one-shot parsing
+        events += self.parser.parse("")
 
-        HACK: Also handles simplified format where text starts with "analysis" and transitions
-        to "assistantfinal" without full channel markers.
-        """
-        # HACK: Handle simplified format (analysis...assistantfinal) without channel markers
-        if (
-            text.startswith("analysis")
-            and "assistantfinal" in text
-            and "<|channel|>" not in text
-        ):
-            # Split on "assistantfinal"
-            parts = text.split("assistantfinal", 1)
-            self._in_reasoning = False
-            if len(parts) == 2:
-                reasoning_text = parts[0][
-                    len("analysis") :
-                ].strip()  # Remove "analysis" prefix
-                normal_text = parts[1].strip()
-                return StreamingParseResult(
-                    normal_text=normal_text, reasoning_text=reasoning_text
-                )
-
-        reasoning_parts = []
-        normal_parts = []
-        current_pos = 0
-
-        # Process text sequentially to preserve tool calls between analysis sections
-        while current_pos < len(text):
-            # Look for next analysis channel
-            analysis_start_idx = text.find(self.think_start_token, current_pos)
-
-            if analysis_start_idx == -1:
-                # No more analysis channels, rest goes to remaining
-                break
-
-            # Preserve any content before this analysis channel (could include tool calls)
-            if analysis_start_idx > current_pos:
-                between_content = text[current_pos:analysis_start_idx]
-                # This content will be added to normal_parts later
-                normal_parts.append(between_content)
-
-            # Extract analysis content
-            analysis_content_start = analysis_start_idx + len(self.think_start_token)
-            analysis_end_idx = text.find(self.think_end_token, analysis_content_start)
-
-            if analysis_end_idx != -1:
-                reasoning_parts.append(
-                    text[analysis_content_start:analysis_end_idx].strip()
-                )
-                current_pos = analysis_end_idx + len(self.think_end_token)
-            else:
-                # Analysis not complete
-                reasoning_parts.append(text[analysis_content_start:].strip())
-                reasoning_text = "".join(reasoning_parts)
-                return StreamingParseResult(reasoning_text=reasoning_text)
-
-        # Add any remaining text after all analysis sections
-        if current_pos < len(text):
-            remaining = text[current_pos:]
-            normal_parts.append(remaining)
-
-        # Process non-analysis content for commentary sections
-        full_normal_text = "".join(normal_parts)
-
-        # Extract reasoning from non-tool-call commentary sections
-        # Tool calls have "to=" in their header, regular commentary does not
-        commentary_pattern = re.compile(
-            r"<\|start\|>assistant<\|channel\|>commentary<\|message\|>(.*?)(?:<\|end\|>|<\|call\|>)",
-            re.DOTALL,
+        reasoning_text = "".join(
+            [e.content for e in events if e.event_type == "reasoning"]
         )
-
-        cleaned_text = full_normal_text
-        for match in reversed(list(commentary_pattern.finditer(full_normal_text))):
-            # Check if this commentary is a tool call by looking at the text before <|message|>
-            match_start = match.start()
-            # Find where "<|channel|>commentary" starts within the matched pattern
-            # The pattern starts with "<|start|>assistant<|channel|>commentary"
-            # So we look for the text between "commentary" and "<|message|>" in the match
-            match_text = full_normal_text[match_start : match.end()]
-            commentary_idx = match_text.find("<|channel|>commentary")
-            if commentary_idx != -1:
-                message_idx = match_text.find("<|message|>", commentary_idx)
-                if message_idx != -1:
-                    between_text = match_text[commentary_idx:message_idx]
-                    # If no "to=" found, this is regular commentary (reasoning content)
-                    if " to=" not in between_text:
-                        content = match.group(1).strip()
-                        reasoning_parts.append(content)
-                        # Remove this commentary section from normal text
-                        cleaned_text = (
-                            cleaned_text[: match.start()] + cleaned_text[match.end() :]
-                        )
-
-        full_normal_text = cleaned_text
-
-        # Combine all reasoning parts
-        reasoning_text = "".join(reasoning_parts)
-
-        # Process full_normal_text for final output
-        normal_text = ""
-        if self.final_channel_start in full_normal_text:
-            final_start = full_normal_text.find(self.final_channel_start)
-            final_content_start = final_start + len(self.final_channel_start)
-            final_end = full_normal_text.find(
-                self.final_channel_end, final_content_start
-            )
-
-            if final_end != -1:
-                # Extract content before final channel (includes tool calls)
-                before_final = full_normal_text[:final_start].strip()
-                # Extract ONLY the final channel content (not the channel markers)
-                final_text = full_normal_text[final_content_start:final_end].strip()
-                # Extract content after final channel
-                after_final = full_normal_text[
-                    final_end + len(self.final_channel_end) :
-                ].strip()
-
-                # For tool calls + final answer: concatenate tool calls with final text
-                parts = []
-                if before_final:
-                    parts.append(before_final)
-                if final_text:
-                    parts.append(final_text)
-                if after_final:
-                    parts.append(after_final)
-                normal_text = " ".join(parts)
-            else:
-                # Final channel not complete - extract what we have
-                # Look for just <|channel|>final<|message|> without <|return|>
-                alt_final_start = full_normal_text.find("<|channel|>final<|message|>")
-                if alt_final_start != -1:
-                    before_alt_final = full_normal_text[:alt_final_start].strip()
-                    alt_final_content = full_normal_text[
-                        alt_final_start + len("<|channel|>final<|message|>") :
-                    ].strip()
-
-                    parts = []
-                    if before_alt_final:
-                        parts.append(before_alt_final)
-                    if alt_final_content:
-                        parts.append(alt_final_content)
-                    normal_text = " ".join(parts)
-                else:
-                    normal_text = full_normal_text.strip()
-        else:
-            # No final channel, treat all as normal text (includes tool calls)
-            normal_text = full_normal_text.strip()
+        normal_parts = []
+        for e in events:
+            if e.event_type == "normal":
+                normal_parts.append(e.content)
+            elif e.event_type == "tool_call":
+                # Use raw_text to preserve structural markers for function call detector
+                normal_parts.append(e.raw_text if e.raw_text else e.content)
+        normal_text = "".join(normal_parts)
+        # Tool call events preserve raw text with structural markers
 
         return StreamingParseResult(
-            normal_text=normal_text, reasoning_text=reasoning_text
+            normal_text=normal_text,
+            reasoning_text=reasoning_text,
         )
 
     def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
-        """
-        Streaming incremental parsing for GPT-OSS format.
+        events = self.parser.parse(new_text)
 
-        This is a simplified streaming implementation that accumulates content
-        and delegates to the non-streaming parser for complex multi-channel parsing.
-        TODO: Implement proper incremental parsing for better streaming performance.
-        """
-        self._buffer += new_text
-
-        if not self._in_reasoning:
-            return StreamingParseResult(normal_text=new_text)
-
-        # Check if we have complete sections to process
-        # For GPT-OSS, we need to wait for complete channel sections
-        # HACK: For now, use simplified approach - wait for key markers before processing
-        key_markers = ["<|end|>", "<|call|>", "<|return|>", "assistantfinal"]
-        has_complete_section = any(marker in self._buffer for marker in key_markers)
-
-        if not has_complete_section:
-            # Still accumulating, don't process yet
-            return StreamingParseResult()
-
-        # Handle simplified format (analysis...assistantfinal) with true incremental streaming
-        if (
-            "<|channel|>" not in self._buffer
-        ):  # Simplified format without channel markers
-            if self._buffer.startswith("analysis"):
-                # Check if we have the transition to assistantfinal
-                if "assistantfinal" in self._buffer:
-                    self._in_reasoning = False
-                    # Complete reasoning section - extract and stream it
-                    parts = self._buffer.split("assistantfinal", 1)
-                    reasoning_text = parts[0][len("analysis") :].strip()
-                    final_content = parts[1].strip()
-
-                    # Clear buffer and return both reasoning and final content
-                    self._buffer = ""
-                    return StreamingParseResult(
-                        reasoning_text=reasoning_text if self.stream_reasoning else "",
-                        normal_text=final_content,
-                    )
-                elif self.stream_reasoning:
-                    # Stream reasoning content incrementally as it arrives
-                    current_reasoning = self._buffer[len("analysis") :].strip()
-                    self._buffer = ""
-                    return StreamingParseResult(reasoning_text=current_reasoning)
-                else:
-                    # Wait for assistantfinal
-                    return StreamingParseResult()
-            elif self._buffer.startswith("assistantfinal"):
-                # Direct final content without analysis
-                final_content = self._buffer[len("assistantfinal") :].strip()
-                self._buffer = ""
-                return StreamingParseResult(normal_text=final_content)
-
-        # For full channel format, process sections as they complete
-        result = StreamingParseResult()
-
-        # Process complete analysis sections
-        while (
-            self.think_start_token in self._buffer
-            and self.think_end_token in self._buffer
-        ):
-            start_idx = self._buffer.find(self.think_start_token)
-            start_pos = start_idx + len(self.think_start_token)
-            end_pos = self._buffer.find(self.think_end_token, start_pos)
-
-            if end_pos != -1:
-                reasoning_content = self._buffer[start_pos:end_pos].strip()
-                if self.stream_reasoning and reasoning_content:
-                    result.reasoning_text += reasoning_content
-
-                # Remove processed analysis section
-                self._buffer = (
-                    self._buffer[:start_idx]
-                    + self._buffer[end_pos + len(self.think_end_token) :]
-                )
-            else:
-                break
-
-        # Process complete commentary sections
-        commentary_pattern = re.compile(
-            r"<\|start\|>assistant<\|channel\|>commentary<\|message\|>(.*?)(?:<\|end\|>|<\|call\|>)",
-            re.DOTALL,
+        reasoning_text = "".join(
+            [e.content for e in events if e.event_type == "reasoning"]
         )
+        normal_parts = []
+        for e in events:
+            if e.event_type == "normal":
+                normal_parts.append(e.content)
+            elif e.event_type == "tool_call":
+                # Use raw_text to preserve structural markers for function call detector
+                normal_parts.append(e.raw_text if e.raw_text else e.content)
+        normal_text = "".join(normal_parts)
 
-        for match in reversed(list(commentary_pattern.finditer(self._buffer))):
-            # Check if this is a tool call
-            start_pos = match.start()
-            commentary_content = match.group(1).strip()
-            if self.stream_reasoning and commentary_content:
-                result.reasoning_text += commentary_content
-
-            # Remove this commentary section
-            self._buffer = self._buffer[: match.start()] + self._buffer[match.end() :]
-            # Clean up any standalone <|start|>assistant
-            self._buffer = re.sub(
-                r"<\|start\|>assistant(?=<\|start\|>assistant)", "", self._buffer
-            )
-
-        # Handle final channel completion
-        if self.final_channel_start in self._buffer:
-            final_start = self._buffer.find(self.final_channel_start)
-            final_content_start = final_start + len(self.final_channel_start)
-
-            # Check if final channel is complete
-            final_end = self._buffer.find(self.final_channel_end, final_content_start)
-            if final_end != -1:
-                # Complete final channel - process everything
-                final_result = self.detect_and_parse(self._buffer)
-                self._buffer = ""
-                return StreamingParseResult(
-                    normal_text=final_result.normal_text,
-                    reasoning_text=result.reasoning_text + final_result.reasoning_text,
-                )
-            else:
-                # Extract content before final channel (e.g. tool calls)
-                before_final = self._buffer[:final_start]
-                if before_final:
-                    # Output tool calls for processing
-                    result.normal_text += before_final
-                    # Keep the final channel part in buffer
-                    self._buffer = self._buffer[final_start:]
-
-        return result
+        return StreamingParseResult(
+            normal_text=normal_text,
+            reasoning_text=reasoning_text,
+        )
 
 
 class ReasoningParser:
@@ -526,7 +276,7 @@ class ReasoningParser:
         self,
         model_type: Optional[str] = None,
         stream_reasoning: bool = True,
-        force_reasoning: bool = False,
+        force_reasoning: Optional[bool] = None,
     ):
         if not model_type:
             raise ValueError("Model type must be specified")
@@ -535,19 +285,25 @@ class ReasoningParser:
         if not detector_class:
             raise ValueError(f"Unsupported model type: {model_type}")
 
-        if model_type.lower() == "qwen3-thinking":
+        # Special cases where we override force_reasoning
+        if model_type.lower() in {"qwen3-thinking", "gpt-oss"}:
             force_reasoning = True
 
-        self.detector = detector_class(
-            stream_reasoning=stream_reasoning, force_reasoning=force_reasoning
-        )
+        # Only pass force_reasoning if explicitly set, let detectors use their defaults
+        kwargs = {"stream_reasoning": stream_reasoning}
+        if force_reasoning is not None:
+            kwargs["force_reasoning"] = force_reasoning
+
+        self.detector = detector_class(**kwargs)
 
-    def parse_non_stream(self, full_text: str) -> Tuple[str, str]:
+    def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]:
         """Non-streaming call: one-time parsing"""
         ret = self.detector.detect_and_parse(full_text)
         return ret.reasoning_text, ret.normal_text
 
-    def parse_stream_chunk(self, chunk_text: str) -> Tuple[str, str]:
+    def parse_stream_chunk(
+        self, chunk_text: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """Streaming call: incremental parsing"""
         ret = self.detector.parse_streaming_increment(chunk_text)
         return ret.reasoning_text, ret.normal_text

sglang/srt/server_args.py

@@ -25,7 +25,6 @@ from typing import List, Literal, Optional, Union
 
 from sglang.srt.function_call.function_call_parser import FunctionCallParser
 from sglang.srt.hf_transformers_utils import check_gguf_file, get_config
-from sglang.srt.layers.utils import is_sm90_supported, is_sm100_supported
 from sglang.srt.lora.lora_registry import LoRARef
 from sglang.srt.reasoning_parser import ReasoningParser
 from sglang.srt.utils import (
@@ -39,6 +38,8 @@ from sglang.srt.utils import (
     is_hip,
     is_port_available,
     is_remote_url,
+    is_sm90_supported,
+    is_sm100_supported,
     is_triton_kernels_available,
     is_valid_ipv6_address,
     nullable_str,
@@ -216,6 +217,7 @@ class ServerArgs:
     hicache_mem_layout: str = "layer_first"
     hicache_storage_backend: Optional[str] = None
     hicache_storage_prefetch_policy: str = "best_effort"
+    hicache_storage_backend_extra_config: Optional[str] = None
 
     # Double Sparsity
     enable_double_sparsity: bool = False
@@ -1641,6 +1643,12 @@ class ServerArgs:
             default=ServerArgs.hicache_storage_prefetch_policy,
             help="Control when prefetching from the storage backend should stop.",
         )
+        parser.add_argument(
+            "--hicache-storage-backend-extra-config",
+            type=str,
+            default=ServerArgs.hicache_storage_backend_extra_config,
+            help="A dictionary in JSON string format containing extra configuration for the storage backend.",
+        )
 
         # Double Sparsity
         parser.add_argument(
@@ -2271,6 +2279,7 @@ class ServerArgs:
             if is_mxfp4_quant_format:
                 # use bf16 for mxfp4 triton kernels
                 self.dtype = "bfloat16"
+
         elif "Llama4" in model_arch:
             assert self.attention_backend in {
                 "fa3",

sglang/srt/tokenizer/tiktoken_tokenizer.py

@@ -121,7 +121,12 @@ class TiktokenTokenizer:
         return self.tokenizer.decode_batch(batch)
 
     def apply_chat_template(
-        self, messages, tokenize, add_generation_prompt, tools=None
+        self,
+        messages,
+        tokenize,
+        add_generation_prompt,
+        tools=None,
+        reasoning_effort=None,
     ):
         ret = self.chat_template_jinja.render(
             messages=messages, add_generation_prompt=add_generation_prompt

sglang/srt/utils.py

@@ -172,6 +172,20 @@ def is_blackwell():
     return torch.cuda.get_device_capability()[0] == 10
 
 
+@lru_cache(maxsize=1)
+def is_sm100_supported(device=None) -> bool:
+    return (torch.cuda.get_device_capability(device)[0] == 10) and (
+        torch.version.cuda >= "12.8"
+    )
+
+
+@lru_cache(maxsize=1)
+def is_sm90_supported(device=None) -> bool:
+    return (torch.cuda.get_device_capability(device)[0] == 9) and (
+        torch.version.cuda >= "12.3"
+    )
+
+
 _warned_bool_env_var_keys = set()
 
 
@@ -1665,9 +1679,29 @@ def direct_register_custom_op(
     IMPORTANT: the lifetime of the operator is tied to the lifetime of the
     library object. If you want to bind the operator to a different library,
     make sure the library object is alive when the operator is used.
+
+    Note: This function will silently skip registration if the operator
+    with the same name is already registered to avoid RuntimeError in
+    multi-engine scenarios (e.g., VERL framework).
     """
     import torch.library
 
+    my_lib = target_lib or sglang_lib
+
+    # Check if operator is already registered to avoid duplicate registration
+    # This is important for scenarios where multiple SGLang engines run in the same process
+    try:
+        # Try to access the operator to see if it's already registered
+        lib_name = my_lib.m.name if hasattr(my_lib.m, "name") else "sglang"
+        if hasattr(torch.ops, lib_name) and hasattr(
+            getattr(torch.ops, lib_name), op_name
+        ):
+            # Operator already exists, skip registration
+            return
+    except (AttributeError, RuntimeError):
+        # Operator doesn't exist, proceed with registration
+        pass
+
     if hasattr(torch.library, "infer_schema"):
         schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
     else:
@@ -1676,11 +1710,22 @@ def direct_register_custom_op(
 
         schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
 
-    my_lib = target_lib or sglang_lib
-    my_lib.define(op_name + schema_str)
-    my_lib.impl(op_name, op_func, "CUDA")
-    if fake_impl is not None:
-        my_lib._register_fake(op_name, fake_impl)
+    try:
+        my_lib.define(op_name + schema_str)
+        my_lib.impl(op_name, op_func, "CUDA")
+        if fake_impl is not None:
+            my_lib._register_fake(op_name, fake_impl)
+    except RuntimeError as error:
+        if "Tried to register an operator" in str(e) and "multiple times" in str(e):
+            # Silently ignore duplicate registration errors
+            # This can happen in multi-engine scenarios
+            pass
+        else:
+            # Re-raise other RuntimeErrors
+            raise error
+    except AttributeError as error:
+        # Always re-raise AttributeError as it indicates missing dependencies
+        raise error
 
 
 def set_gpu_proc_affinity(
@@ -1919,6 +1964,15 @@ def get_ip() -> str:
     except Exception:
         pass
 
+    # try  using hostname
+    hostname = socket.gethostname()
+    try:
+        ip_addr = socket.gethostbyname(hostname)
+        warnings.warn("using local ip address: {}".format(ip_addr))
+        return ip_addr
+    except Exception:
+        pass
+
     warnings.warn(
         "Failed to get the IP address, using 0.0.0.0 by default."
         "The value can be set by the environment variable"

sglang/version.py

@@ -1 +1 @@
-__version__ = "0.5.1.post2"
+__version__ = "0.5.1.post3"

{sglang-0.5.1.post2.dist-info → sglang-0.5.1.post3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sglang
-Version: 0.5.1.post2
+Version: 0.5.1.post3
 Summary: SGLang is yet another fast serving framework for large language models and vision language models.
 License:                                  Apache License
                                    Version 2.0, January 2004
@@ -257,7 +257,7 @@ Requires-Dist: uvloop; extra == "runtime-common"
 Requires-Dist: xgrammar==0.1.23; extra == "runtime-common"
 Provides-Extra: srt
 Requires-Dist: sglang[runtime_common]; extra == "srt"
-Requires-Dist: sgl-kernel==0.3.5; extra == "srt"
+Requires-Dist: sgl-kernel==0.3.7; extra == "srt"
 Requires-Dist: torch==2.8.0; extra == "srt"
 Requires-Dist: torchaudio==2.8.0; extra == "srt"
 Requires-Dist: torchvision; extra == "srt"
@@ -304,6 +304,7 @@ Requires-Dist: pandas; extra == "test"
 Requires-Dist: peft; extra == "test"
 Requires-Dist: sentence_transformers; extra == "test"
 Requires-Dist: pytest; extra == "test"
+Requires-Dist: tabulate; extra == "test"
 Provides-Extra: all
 Requires-Dist: sglang[srt]; extra == "all"
 Requires-Dist: sglang[openai]; extra == "all"
@@ -374,7 +375,7 @@ Dynamic: license-file
 | [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) |
 
 ## News
-- [2025/08] 🔔 SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf)).
+- [2025/08] 🔔 SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf), [Highlights](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_highlights.pdf), [AITER/MoRI](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_aiter_mori.pdf), [Wave](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_wave.pdf)).
 - [2025/08] 🔥 SGLang provides day-0 support for OpenAI gpt-oss model ([instructions](https://github.com/sgl-project/sglang/issues/8833))
 - [2025/06] 🔥 SGLang, the high-performance serving infrastructure powering trillions of tokens daily, has been awarded the third batch of the Open Source AI Grant by a16z ([a16z blog](https://a16z.com/advancing-open-source-ai-through-benchmarks-and-bold-experimentation/)).
 - [2025/06] 🔥 Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part I): 2.7x Higher Decoding Throughput ([blog](https://lmsys.org/blog/2025-06-16-gb200-part-1/)).