sglang 0.5.2rc2__py3-none-any.whl → 0.5.3.post1__py3-none-any.whl

sglang/srt/eplb/expert_location.py

@@ -11,21 +11,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+
+from __future__ import annotations
+
 import json
 import logging
 import random
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional
+from typing import TYPE_CHECKING, List, Optional
 
 import torch
 import torch.distributed
 import torch.nn.functional as F
 
-from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.eplb import eplb_algorithms
 from sglang.srt.model_loader import get_model_architecture
-from sglang.srt.server_args import ServerArgs
+
+if TYPE_CHECKING:
+    from sglang.srt.configs.model_config import ModelConfig
+    from sglang.srt.server_args import ServerArgs
 
 logger = logging.getLogger(__name__)
 
@@ -226,6 +231,7 @@ class ExpertLocationMetadata:
             logical_to_all_physical_map_num_valid=logical_to_all_physical_map_num_valid,
             logical_to_rank_dispatch_physical_map=(
                 compute_logical_to_rank_dispatch_physical_map(
+                    server_args=server_args,
                     logical_to_all_physical_map=logical_to_all_physical_map,
                     num_gpus=ep_size,
                     num_physical_experts=num_physical_experts,
@@ -335,6 +341,7 @@ def _pad_nested_array(arr, pad_value):
 
 # TODO optimize performance (rewrite and/or run in separate process with overlap)
 def compute_logical_to_rank_dispatch_physical_map(
+    server_args: ServerArgs,
     logical_to_all_physical_map: torch.Tensor,
     num_gpus: int,
     num_physical_experts: int,
@@ -343,7 +350,9 @@ def compute_logical_to_rank_dispatch_physical_map(
 ):
     r = random.Random(seed)
 
-    num_local_physical_experts = num_physical_experts // num_gpus
+    num_local_gpu_physical_experts = num_physical_experts // num_gpus
+    num_gpus_per_node = server_args.ep_size // server_args.nnodes
+    num_local_node_physical_experts = num_local_gpu_physical_experts * num_gpus_per_node
     num_layers, num_logical_experts, _ = logical_to_all_physical_map.shape
     dtype = logical_to_all_physical_map.dtype
 
@@ -367,13 +376,28 @@ def compute_logical_to_rank_dispatch_physical_map(
                     physical_expert_id
                     for physical_expert_id in candidate_physical_expert_ids
                     if _compute_gpu_id_of_physical_expert(
-                        physical_expert_id, num_local_physical_experts
+                        physical_expert_id, num_local_gpu_physical_experts
                     )
                     == gpu_id
                 ]
                 if len(same_gpu_physical_expert_ids) > 0:
+                    # 1. Prefer same-GPU experts
                     output_partial[gpu_id] = same_gpu_physical_expert_ids[0]
-
+                else:
+                    # 2. Otherwise, prefer same-node experts
+                    node_id = gpu_id // num_gpus_per_node
+                    same_node_physical_expert_ids = [
+                        physical_expert_id
+                        for physical_expert_id in candidate_physical_expert_ids
+                        if _compute_node_id_of_physical_expert(
+                            physical_expert_id, num_local_node_physical_experts
+                        )
+                        == node_id
+                    ]
+                    if len(same_node_physical_expert_ids) > 0:
+                        output_partial[gpu_id] = same_node_physical_expert_ids[0]
+
+            # 3. Fill remaining slots with fair random choices
             num_remain = torch.sum(output_partial == -1).item()
             output_partial[output_partial == -1] = torch.tensor(
                 _fair_choices(candidate_physical_expert_ids, k=num_remain, r=r),
@@ -399,9 +423,15 @@ def _logical_to_all_physical_raw(
 
 
 def _compute_gpu_id_of_physical_expert(
-    physical_expert_id: int, num_local_physical_experts: int
+    physical_expert_id: int, num_local_gpu_physical_experts: int
+) -> int:
+    return physical_expert_id // num_local_gpu_physical_experts
+
+
+def _compute_node_id_of_physical_expert(
+    physical_expert_id: int, num_local_host_physical_experts: int
 ) -> int:
-    return physical_expert_id // num_local_physical_experts
+    return physical_expert_id // num_local_host_physical_experts
 
 
 def _fair_choices(arr: List, k: int, r: random.Random) -> List:

sglang/srt/eplb/expert_location_updater.py

@@ -47,7 +47,7 @@ class ExpertLocationUpdater:
     ):
         if self._first_execution:
             self._first_execution = False
-            torch.cuda.empty_cache()
+            torch.get_device_module().empty_cache()
 
         old_expert_location_metadata = get_global_expert_location_metadata()
         assert old_expert_location_metadata is not None

sglang/srt/function_call/base_format_detector.py

@@ -162,12 +162,9 @@ class BaseFormatDetector(ABC):
 
         try:
             try:
-                if current_text.startswith(self.bot_token):
-                    start_idx = len(self.bot_token)
-                elif self.current_tool_id > 0 and current_text.startswith(
-                    self.tool_call_separator + self.bot_token
-                ):
-                    start_idx = len(self.tool_call_separator + self.bot_token)
+                tool_call_pos = current_text.find(self.bot_token)
+                if tool_call_pos != -1:
+                    start_idx = tool_call_pos + len(self.bot_token)
                 elif self.current_tool_id > 0 and current_text.startswith(
                     self.tool_call_separator
                 ):

sglang/srt/function_call/ebnf_composer.py

@@ -50,19 +50,19 @@ class EBNFComposer:
 
     CALL_RULE_MAP = {
         "pythonic": 'call_{name} ::= "{name}" "(" {arguments_rule} ")"',
-        "json": 'call_{name} ::= "{{" "\\"name\\"" ":" "\\"{name}\\"" ", " "\\"arguments\\"" ":" {arguments_rule} "}}"',
+        "json": 'call_{name} ::= "{{" ws "\\"name\\"" ws ":" ws "\\"{name}\\"" ws "," ws "\\"arguments\\"" ws ":" ws {arguments_rule} ws "}}"',
         "xml": 'call_{name} ::= "<function={name}>\\n" {arguments_rule} "\\n</function>"',
     }
 
     ARGUMENTS_RULE_MAP = {
         "pythonic": "{arg_rules}",
-        "json": '"{{" {arg_rules} "}}"',
+        "json": '"{{" ws {arg_rules} ws "}}"',
         "xml": "{arg_rules}",
     }
 
     KEY_VALUE_RULE_MAP = {
         "pythonic": '"{key}" "=" {valrule}',
-        "json": '"\\"{key}\\"" ":" {valrule}',
+        "json": '"\\"{key}\\"" ws ":" ws {valrule}',
         "xml": '"<parameter={key}>\\n" {valrule} "\\n</parameter>"',
     }
 
@@ -165,7 +165,7 @@ class EBNFComposer:
         tool_call_separator: Optional[str] = None,
         call_rule_fmt: Optional[str] = None,
         key_value_rule_fmt: Optional[str] = None,
-        key_value_separator: str = ",",
+        key_value_separator: str = 'ws "," ws',
     ):
         """
         Generalized EBNF builder for all detectors.
@@ -183,6 +183,10 @@ class EBNFComposer:
             key_value_rule_fmt: Optional custom format string for key-value pairs. It should define how each parameter is formatted,
                 with placeholders {key} for the parameter name and {valrule} for the value rule. If None, a default format
                 based on function_format will be used.
+            key_value_separator: Raw EBNF fragment inserted between key-value pairs.
+                This string is used verbatim (not auto-quoted). Pass:
+                - Quoted terminals when you need a literal token (e.g. '","' or '"\\n"').
+                - Raw/non-terminals when you need grammar tokens (e.g. 'ws "," ws').
         """
         # =================================================================
         # Step 1: Determine the root tool calls rule
@@ -281,9 +285,7 @@ class EBNFComposer:
             # Add required properties joined by commas
             if required:
                 rule_parts.append(
-                    f' "{key_value_separator}" '.join(
-                        prop_kv_pairs[k] for k in required
-                    )
+                    f" {key_value_separator} ".join(prop_kv_pairs[k] for k in required)
                 )
 
             # Add optional properties with flexible ordering
@@ -298,14 +300,14 @@ class EBNFComposer:
                             opt_parts.append(prop_kv_pairs[optional[j]])
                         else:
                             opt_parts.append(
-                                f' ( "{key_value_separator}" {prop_kv_pairs[optional[j]]} )?'
+                                f" ( {key_value_separator} {prop_kv_pairs[optional[j]]} )?"
                             )
                     opt_alternatives.append("".join(opt_parts))
 
                 # Wrap with appropriate comma handling based on whether we have required properties
                 if required:
                     # Required properties exist, so optional group needs outer comma
-                    rule_parts.append(f' ( "{key_value_separator}" ( ')
+                    rule_parts.append(f" ( {key_value_separator} ( ")
                     rule_parts.append(" | ".join(opt_alternatives))
                     rule_parts.append(" ) )?")
                 else:

sglang/srt/function_call/function_call_parser.py

@@ -20,6 +20,7 @@ from sglang.srt.function_call.pythonic_detector import PythonicDetector
 from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
 from sglang.srt.function_call.qwen25_detector import Qwen25Detector
 from sglang.srt.function_call.step3_detector import Step3Detector
+from sglang.srt.function_call.utils import get_json_schema_constraint
 
 logger = logging.getLogger(__name__)
 
@@ -34,17 +35,19 @@ class FunctionCallParser:
     """
 
     ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = {
-        "llama3": Llama32Detector,
-        "qwen25": Qwen25Detector,
-        "mistral": MistralDetector,
         "deepseekv3": DeepSeekV3Detector,
         "deepseekv31": DeepSeekV31Detector,
-        "pythonic": PythonicDetector,
+        "glm": Glm4MoeDetector,
+        "glm45": Glm4MoeDetector,
+        "gpt-oss": GptOssDetector,
         "kimi_k2": KimiK2Detector,
+        "llama3": Llama32Detector,
+        "mistral": MistralDetector,
+        "pythonic": PythonicDetector,
+        "qwen": Qwen25Detector,
+        "qwen25": Qwen25Detector,
         "qwen3_coder": Qwen3CoderDetector,
-        "glm45": Glm4MoeDetector,
         "step3": Step3Detector,
-        "gpt-oss": GptOssDetector,
     }
 
     def __init__(self, tools: List[Tool], tool_call_parser: str):
@@ -69,6 +72,8 @@ class FunctionCallParser:
         Returns:
             True if the text contains a tool call, False otherwise
         """
+        if not self.tools:
+            return False
         return self.detector.has_tool_call(text)
 
     def parse_non_stream(self, full_text: str) -> Tuple[str, list[ToolCallItem]]:
@@ -83,6 +88,8 @@ class FunctionCallParser:
             - The remaining text after parsing that was not consumed by the detector (can be treated as normal text)
             - A list of tool calls parsed from the text
         """
+        if not self.tools:
+            return full_text, []
         parsed_result = self.detector.detect_and_parse(full_text, self.tools)
         tool_call_list = parsed_result.calls
         if tool_call_list:
@@ -102,6 +109,8 @@ class FunctionCallParser:
             - The normal text that should be displayed to the user
             - A list of tool calls parsed from the chunk
         """
+        if not self.tools:
+            return chunk_text, []
         final_normal_text = ""
         final_calls = []
 
@@ -172,8 +181,8 @@ class FunctionCallParser:
             strict_tag = self.get_structure_tag()
             return ("structural_tag", strict_tag)
         elif tool_choice == "required" or isinstance(tool_choice, ToolChoice):
-            ebnf = self.get_ebnf(tool_choice)
-            return ("ebnf", ebnf) if ebnf is not None else None
+            json_schema = get_json_schema_constraint(self.tools, tool_choice)
+            return ("json_schema", json_schema)
 
     def get_ebnf(
         self, tool_choice: Union[ToolChoice, Literal["required"]]

sglang/srt/function_call/glm4_moe_detector.py

@@ -39,7 +39,7 @@ def parse_arguments(json_value):
 
 class Glm4MoeDetector(BaseFormatDetector):
     """
-    Detector for GLM-4.5 models.
+    Detector for GLM-4.5 and GLM-4.6 models.
     Assumes function call format:
       <tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>北京</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2024-06-27</arg_value>\n</tool_call>\n<tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>上海</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2024-06-27</arg_value>\n</tool_call>
     """
@@ -53,7 +53,7 @@ class Glm4MoeDetector(BaseFormatDetector):
         self.func_arg_regex = r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>"
 
     def has_tool_call(self, text: str) -> bool:
-        """Check if the text contains a glm-4.5 format tool call."""
+        """Check if the text contains a glm-4.5 / glm-4.6 format tool call."""
         return self.bot_token in text
 
     def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
@@ -102,7 +102,7 @@ class Glm4MoeDetector(BaseFormatDetector):
         self, new_text: str, tools: List[Tool]
     ) -> StreamingParseResult:
         """
-        Streaming incremental parsing tool calls for GLM-4.5 format.
+        Streaming incremental parsing tool calls for GLM-4.5 and GLM-4.6 format.
         """
         self._buffer += new_text
         current_text = self._buffer
@@ -160,5 +160,5 @@ class Glm4MoeDetector(BaseFormatDetector):
             function_format="xml",
             call_rule_fmt='"{name}" "\\n" ( {arguments_rule} "\\n" )?',
             key_value_rule_fmt='"<arg_key>{key}</arg_key>" "\\n" "<arg_value>" {valrule} "</arg_value>"',
-            key_value_separator="\\n",
+            key_value_separator='"\\n"',
         )

sglang/srt/function_call/gpt_oss_detector.py

@@ -81,6 +81,29 @@ class GptOssDetector(BaseFormatDetector):
         # Always use HarmonyParser for parsing to ensure proper filtering
         events = self.harmony_parser.parse(new_text)
 
+        # If there are no parsed events and the chunk contains no Harmony structural
+        # markers, treat it as plain text and pass it through. This fixes a bug where
+        # normal content was held in the buffer when tools were provided but not used.
+        if not events:
+            has_harmony_markers = any(
+                marker in self._buffer
+                for marker in (
+                    "<|start|>",
+                    "<|channel|>",
+                    "<|message|>",
+                    "<|constrain|>",
+                    "<|end|>",
+                    "<|call|>",
+                    "<|return|>",
+                    "assistantfinal",
+                )
+            )
+            if not has_harmony_markers:
+                # Plain text with no tool markers — emit as normal content
+                out = self._buffer
+                self._buffer = ""
+                return StreamingParseResult(normal_text=out, calls=[])
+
         # Quick check if we might have tool calls
         if (
             "<|channel|>commentary to=" not in self._buffer

sglang/srt/function_call/json_array_parser.py

@@ -0,0 +1,63 @@
+import json
+import re
+from typing import List
+
+from sglang.srt.entrypoints.openai.protocol import Tool
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import StreamingParseResult
+
+
+class JsonArrayParser(BaseFormatDetector):
+    """
+    Parser for JSON array tool calls when JSON schema constraints are active.
+
+    This parser is used when tool_choice="required" or a specific tool is named,
+    bypassing model-specific parsers in favor of direct JSON array parsing.
+    """
+
+    def __init__(self):
+        super().__init__()
+        # Configure for JSON array parsing
+        self.bot_token = "["
+        self.eot_token = "]"
+        self.tool_call_separator = ","
+
+    def has_tool_call(self, text: str) -> bool:
+        """
+        Check if the given text contains a JSON tool call (array or single object).
+        """
+        return "[" in text or "{" in text
+
+    def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+        """
+        Parse JSON tool calls using the base class implementation.
+        """
+        raise NotImplementedError(
+            "Detect and parse not supported for JSON schema constraints."
+        )
+
+    def build_ebnf(self, tools: List[Tool]) -> str:
+        """
+        Build an EBNF grammar for constrained generation.
+        This is not used for JSON schema constraints as they are handled
+        by the constraint backends directly.
+        """
+        raise NotImplementedError(
+            "EBNF generation is not supported for JSON schema constraints."
+        )
+
+    def parse_streaming_increment(
+        self, new_text: str, tools: List[Tool]
+    ) -> StreamingParseResult:
+        """
+        Streaming incremental parsing with tool validation.
+        """
+        return super().parse_streaming_increment(new_text, tools)
+
+    def structure_info(self) -> callable:
+        """
+        Return a function that creates StructureInfo for constrained generation.
+        This is not used for JSON schema constraints as they are handled
+        by the constraint backends directly.
+        """
+        raise NotImplementedError("structure_info not used for JSON schema constraints")

sglang/srt/function_call/kimik2_detector.py

@@ -50,6 +50,11 @@ class KimiK2Detector(BaseFormatDetector):
 
         self._last_arguments = ""
 
+        # Robust parser for ids like "functions.search:0" or fallback "search:0"
+        self.tool_call_id_regex = re.compile(
+            r"^(?:functions\.)?(?P<name>[\w\.]+):(?P<index>\d+)$"
+        )
+
     def has_tool_call(self, text: str) -> bool:
         """Check if the text contains a KimiK2 format tool call."""
         return self.bot_token in text
@@ -76,14 +81,18 @@ class KimiK2Detector(BaseFormatDetector):
             tool_calls = []
             for match in function_call_tuples:
                 function_id, function_args = match
-                function_name = function_id.split(".")[1].split(":")[0]
-                function_idx = int(function_id.split(".")[1].split(":")[1])
+                m = self.tool_call_id_regex.match(function_id)
+                if not m:
+                    logger.warning("Unexpected tool_call_id format: %s", function_id)
+                    continue
+                function_name = m.group("name")
+                function_idx = int(m.group("index"))
 
                 logger.info(f"function_name {function_name}")
 
                 tool_calls.append(
                     ToolCallItem(
-                        tool_index=function_idx,  # Use the call index in the response, not tool position
+                        tool_index=function_idx,
                         name=function_name,
                         parameters=function_args,
                     )
@@ -128,7 +137,11 @@ class KimiK2Detector(BaseFormatDetector):
                 function_id = match.group("tool_call_id")
                 function_args = match.group("function_arguments")
 
-                function_name = function_id.split(".")[1].split(":")[0]
+                m = self.tool_call_id_regex.match(function_id)
+                if not m:
+                    logger.warning("Unexpected tool_call_id format: %s", function_id)
+                    return StreamingParseResult(normal_text="", calls=calls)
+                function_name = m.group("name")
 
                 # Initialize state if this is the first tool call
                 if self.current_tool_id == -1:

sglang/srt/function_call/qwen3_coder_detector.py

@@ -358,5 +358,5 @@ class Qwen3CoderDetector(BaseFormatDetector):
             function_format="xml",
             call_rule_fmt='"<function={name}>\\n" {arguments_rule} "\\n</function>"',
             key_value_rule_fmt='"<parameter={key}>\\n" {valrule} "\\n</parameter>"',
-            key_value_separator="\\n",
+            key_value_separator='"\\n"',
         )

sglang/srt/function_call/utils.py

@@ -1,10 +1,13 @@
 import json
 from json import JSONDecodeError, JSONDecoder
-from typing import Any, Tuple
+from json.decoder import WHITESPACE
+from typing import Any, List, Literal, Optional, Tuple, Union
 
 import partial_json_parser
 from partial_json_parser.core.options import Allow
 
+from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
+
 
 def _find_common_prefix(s1: str, s2: str) -> str:
     prefix = ""
@@ -37,10 +40,12 @@ def _partial_json_loads(input_str: str, flags: Allow) -> Tuple[Any, int]:
     """
     try:
         return (partial_json_parser.loads(input_str, flags), len(input_str))
-    except JSONDecodeError as e:
-        if "Extra data" in e.msg:
-            dec = JSONDecoder()
-            return dec.raw_decode(input_str)
+    except (JSONDecodeError, IndexError) as e:
+        msg = getattr(e, "msg", str(e))
+        if "Extra data" in msg or "pop from empty list" in msg:
+            start = WHITESPACE.match(input_str, 0).end()
+            obj, end = JSONDecoder().raw_decode(input_str, start)
+            return obj, end
         raise
 
 
@@ -50,3 +55,89 @@ def _is_complete_json(input_str: str) -> bool:
         return True
     except JSONDecodeError:
         return False
+
+
+def _get_tool_schema_defs(tools: List[Tool]) -> dict:
+    """
+    Get consolidated $defs from all tools, validating for conflicts.
+
+    Args:
+        tools: List of tools to process
+
+    Returns:
+        Dictionary of consolidated $defs from all tools
+
+    Raises:
+        ValueError: If conflicting $defs are found
+    """
+    all_defs = {}
+    for tool in tools:
+        if tool.function.parameters is None:
+            continue
+        defs = tool.function.parameters.get("$defs", {})
+        for def_name, def_schema in defs.items():
+            if def_name in all_defs and all_defs[def_name] != def_schema:
+                raise ValueError(
+                    f"Tool definition '{def_name}' has "
+                    "multiple schemas, which is not "
+                    "supported."
+                )
+            else:
+                all_defs[def_name] = def_schema
+    return all_defs
+
+
+def _get_tool_schema(tool: Tool) -> dict:
+    return {
+        "properties": {
+            "name": {"type": "string", "enum": [tool.function.name]},
+            "parameters": (
+                tool.function.parameters
+                if tool.function.parameters
+                else {"type": "object", "properties": {}}
+            ),
+        },
+        "required": ["name", "parameters"],
+    }
+
+
+def get_json_schema_constraint(
+    tools: List[Tool], tool_choice: Union[ToolChoice, Literal["required"]]
+) -> Optional[dict]:
+    """
+    Get the JSON schema constraint for the specified tool choice.
+
+    Args:
+        tool_choice: The tool choice specification
+
+    Returns:
+        JSON schema dict, or None if no valid tools found
+    """
+
+    if isinstance(tool_choice, ToolChoice):
+        # For specific function choice, return the user's parameters schema directly
+        fn_name = tool_choice.function.name
+        for tool in tools:
+            if tool.function.name == fn_name:
+                return {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 1,
+                    "items": _get_tool_schema(tool),
+                }
+        return None
+    elif tool_choice == "required":
+        json_schema = {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "type": "object",
+                "anyOf": [_get_tool_schema(tool) for tool in tools],
+            },
+        }
+        json_schema_defs = _get_tool_schema_defs(tools)
+        if json_schema_defs:
+            json_schema["$defs"] = json_schema_defs
+        return json_schema
+
+    return None

sglang/srt/grpc/__init__.py

@@ -0,0 +1 @@
+# SGLang gRPC module