sglang 0.4.10.post2__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

sglang/srt/entrypoints/openai/tool_server.py

@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import logging
+from abc import ABC, abstractmethod
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from typing import Any
+
+try:
+    from mcp import ClientSession
+    from mcp.client.sse import sse_client
+    from mcp.types import ListToolsResult
+except ImportError as e:
+    ClientSession = sse_client = ListToolsResult = e
+
+from openai_harmony import ToolDescription, ToolNamespaceConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def list_server_and_tools(server_url: str):
+
+    async with sse_client(url=server_url) as streams, ClientSession(
+        *streams
+    ) as session:
+        initialize_response = await session.initialize()
+        list_tools_response = await session.list_tools()
+        return initialize_response, list_tools_response
+
+
+def trim_schema(schema: dict) -> dict:
+    # Turn JSON Schema from MCP generated into Harmony's variant.
+    if "title" in schema:
+        del schema["title"]
+    if "default" in schema and schema["default"] is None:
+        del schema["default"]
+    if "anyOf" in schema:
+        # Turn "anyOf": [{"type": "type-1"}, {"type": "type-2"}]
+        # into "type": ["type-1", "type-2"]
+        # if there's more than 1 types, also remove "null" type as Harmony will
+        # just ignore it
+        types = [
+            type_dict["type"]
+            for type_dict in schema["anyOf"]
+            if type_dict["type"] != "null"
+        ]
+        schema["type"] = types
+        del schema["anyOf"]
+    if "properties" in schema:
+        schema["properties"] = {
+            k: trim_schema(v) for k, v in schema["properties"].items()
+        }
+    return schema
+
+
+def post_process_tools_description(
+    list_tools_result: "ListToolsResult",
+) -> "ListToolsResult":
+    # Adapt the MCP tool result for Harmony
+    for tool in list_tools_result.tools:
+        tool.inputSchema = trim_schema(tool.inputSchema)
+
+    # Some tools schema don't need to be part of the prompt (e.g. simple text
+    # in text out for Python)
+    list_tools_result.tools = [
+        tool
+        for tool in list_tools_result.tools
+        if getattr(tool.annotations, "include_in_prompt", True)
+    ]
+
+    return list_tools_result
+
+
+class ToolServer(ABC):
+
+    @abstractmethod
+    def has_tool(self, tool_name: str):
+        pass
+
+    @abstractmethod
+    def get_tool_description(self, tool_name: str):
+        pass
+
+    @abstractmethod
+    def get_tool_session(self, tool_name: str) -> AbstractAsyncContextManager[Any]: ...
+
+
+class MCPToolServer(ToolServer):
+
+    def __init__(self):
+        self.harmony_tool_descriptions = {}
+
+    async def add_tool_server(self, server_url: str):
+        tool_urls = server_url.split(",")
+        self.harmony_tool_descriptions = {}
+        self.urls: dict[str, str] = {}
+        for url in tool_urls:
+            url = f"http://{url}/sse"
+            initialize_response, list_tools_response = await list_server_and_tools(url)
+
+            list_tools_response = post_process_tools_description(list_tools_response)
+
+            tool_from_mcp = ToolNamespaceConfig(
+                name=initialize_response.serverInfo.name,
+                description=initialize_response.instructions,
+                tools=[
+                    ToolDescription.new(
+                        name=tool.name,
+                        description=tool.description,
+                        parameters=tool.inputSchema,
+                    )
+                    for tool in list_tools_response.tools
+                ],
+            )
+            self.harmony_tool_descriptions[tool_from_mcp.name] = tool_from_mcp
+            if tool_from_mcp.name not in self.urls:
+                self.urls[tool_from_mcp.name] = url
+            else:
+                logger.warning(
+                    "Tool %s already exists. Ignoring duplicate tool server %s",
+                    tool_from_mcp.name,
+                    url,
+                )
+
+    def has_tool(self, tool_name: str):
+        return tool_name in self.harmony_tool_descriptions
+
+    def get_tool_description(self, tool_name: str):
+        return self.harmony_tool_descriptions.get(tool_name)
+
+    @asynccontextmanager
+    async def get_tool_session(self, tool_name: str):
+        url = self.urls.get(tool_name)
+        if url:
+            async with sse_client(url=url) as streams, ClientSession(
+                *streams
+            ) as session:
+                await session.initialize()
+                yield session
+        else:
+            logger.warning("Tool %s not found", tool_name)
+
+
+class DemoToolServer(ToolServer):
+
+    def __init__(self):
+        from sglang.srt.entrypoints.tool import (
+            HarmonyBrowserTool,
+            HarmonyPythonTool,
+            Tool,
+        )
+
+        self.tools: dict[str, Tool] = {}
+        browser_tool = HarmonyBrowserTool()
+        if browser_tool.enabled:
+            self.tools["browser"] = browser_tool
+        python_tool = HarmonyPythonTool()
+        if python_tool.enabled:
+            self.tools["python"] = python_tool
+
+    def has_tool(self, tool_name: str):
+        return tool_name in self.tools
+
+    def get_tool_description(self, tool_name: str):
+        if tool_name not in self.tools:
+            return None
+        if tool_name == "browser":
+            return ToolNamespaceConfig.browser()
+        elif tool_name == "python":
+            return ToolNamespaceConfig.python()
+        else:
+            raise ValueError(f"Unknown tool {tool_name}")
+
+    @asynccontextmanager
+    async def get_tool_session(self, tool_name: str):
+        yield self.tools[tool_name]

sglang/srt/entrypoints/tool.py

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    # Avoid circular import.
+    from sglang.srt.entrypoints.context import ConversationContext
+
+logger = logging.getLogger(__name__)
+
+
+class Tool(ABC):
+
+    @abstractmethod
+    async def get_result(self, context: "ConversationContext") -> Any:
+        pass
+
+
+class HarmonyBrowserTool(Tool):
+
+    def __init__(self):
+        self.enabled = True
+        exa_api_key = os.getenv("EXA_API_KEY")
+        if not exa_api_key:
+            self.enabled = False
+            logger.warning_once("EXA_API_KEY is not set, browsing is disabled")
+            return
+
+        try:
+            from gpt_oss.tools.simple_browser import SimpleBrowserTool
+            from gpt_oss.tools.simple_browser.backend import ExaBackend
+        except ImportError:
+            self.enabled = False
+            logger.warning_once("gpt_oss is not installed, browsing is disabled")
+            return
+
+        browser_backend = ExaBackend(source="web", api_key=exa_api_key)
+        self.browser_tool = SimpleBrowserTool(backend=browser_backend)
+        logger.info_once("Browser tool initialized")
+
+    async def get_result(self, context: "ConversationContext") -> Any:
+        from sglang.srt.entrypoints.context import HarmonyContext
+
+        assert isinstance(context, HarmonyContext)
+        last_msg = context.messages[-1]
+        tool_output_msgs = []
+        async for msg in self.browser_tool.process(last_msg):
+            tool_output_msgs.append(msg)
+        return tool_output_msgs
+
+    @property
+    def tool_config(self) -> Any:
+        return self.browser_tool.tool_config
+
+
+class HarmonyPythonTool(Tool):
+
+    def __init__(self):
+        self.enabled = True
+
+        try:
+            from gpt_oss.tools.python_docker.docker_tool import PythonTool
+        except ImportError:
+            self.enabled = False
+            logger.warning_once(
+                "gpt_oss is not installed, code interpreter is disabled"
+            )
+            return
+
+        self.python_tool = PythonTool()
+        logger.info_once("Code interpreter tool initialized")
+
+    async def get_result(self, context: "ConversationContext") -> Any:
+        from sglang.srt.entrypoints.context import HarmonyContext
+
+        assert isinstance(context, HarmonyContext)
+        last_msg = context.messages[-1]
+        tool_output_msgs = []
+        async for msg in self.python_tool.process(last_msg):
+            tool_output_msgs.append(msg)
+        return tool_output_msgs
+
+    @property
+    def tool_config(self) -> Any:
+        return self.python_tool.tool_config

sglang/srt/eplb/expert_location.py

@@ -35,6 +35,7 @@ class ExpertLocationMetadata:
     physical_to_logical_map: torch.Tensor  # (layers, num_physical_experts)
     physical_to_logical_map_cpu: torch.Tensor
     logical_to_all_physical_map: torch.Tensor  # (layers, num_logical_experts, X)
+    logical_to_all_physical_map_cpu: torch.Tensor  # CPU copy for performance
     logical_to_all_physical_map_num_valid: torch.Tensor  # (layers, num_logical_experts)
     # (layers, num_logical_experts)
     logical_to_rank_dispatch_physical_map: Optional[torch.Tensor]
@@ -221,6 +222,7 @@ class ExpertLocationMetadata:
             physical_to_logical_map=physical_to_logical_map,
             physical_to_logical_map_cpu=physical_to_logical_map.cpu(),
             logical_to_all_physical_map=logical_to_all_physical_map_padded,
+            logical_to_all_physical_map_cpu=logical_to_all_physical_map_padded.cpu(),
             logical_to_all_physical_map_num_valid=logical_to_all_physical_map_num_valid,
             logical_to_rank_dispatch_physical_map=(
                 compute_logical_to_rank_dispatch_physical_map(
@@ -251,6 +253,7 @@ class ExpertLocationMetadata:
             "physical_to_logical_map",
             "physical_to_logical_map_cpu",
             "logical_to_all_physical_map",
+            "logical_to_all_physical_map_cpu",
             "logical_to_all_physical_map_num_valid",
             "logical_to_rank_dispatch_physical_map",
         ]:
@@ -270,9 +273,10 @@ class ExpertLocationMetadata:
     def logical_to_all_physical(
         self, layer_id: int, logical_expert_id: int
     ) -> List[int]:
+        # Use CPU copy to avoid GPU→CPU sync on every call, which is expensive in update weights scenario
         return [
             physical_expert_id
-            for physical_expert_id in self.logical_to_all_physical_map[
+            for physical_expert_id in self.logical_to_all_physical_map_cpu[
                 layer_id, logical_expert_id
             ].tolist()
             if physical_expert_id != -1

sglang/srt/function_call/ebnf_composer.py

@@ -316,6 +316,7 @@ class EBNFComposer:
 
             combined_args = "".join(rule_parts)
             arguments_rule = args_template.format(arg_rules=combined_args)
+            arguments_rule = arguments_rule or '""'
 
             # Add the function call rule and its arguments rule
             ebnf_lines.append(

sglang/srt/function_call/function_call_parser.py

@@ -11,6 +11,7 @@ from sglang.srt.function_call.base_format_detector import BaseFormatDetector
 from sglang.srt.function_call.core_types import ToolCallItem
 from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector
 from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector
+from sglang.srt.function_call.gpt_oss_detector import GptOssDetector
 from sglang.srt.function_call.kimik2_detector import KimiK2Detector
 from sglang.srt.function_call.llama32_detector import Llama32Detector
 from sglang.srt.function_call.mistral_detector import MistralDetector
@@ -41,6 +42,7 @@ class FunctionCallParser:
         "qwen3_coder": Qwen3CoderDetector,
         "glm45": Glm4MoeDetector,
         "step3": Step3Detector,
+        "gpt-oss": GptOssDetector,
     }
 
     def __init__(self, tools: List[Tool], tool_call_parser: str):

sglang/srt/function_call/glm4_moe_detector.py

@@ -158,7 +158,7 @@ class Glm4MoeDetector(BaseFormatDetector):
             individual_call_end_token=self.eot_token,
             tool_call_separator="\\n",
             function_format="xml",
-            call_rule_fmt='"{name}" "\\n" {arguments_rule} "\\n"',
+            call_rule_fmt='"{name}" "\\n" ( {arguments_rule} "\\n" )?',
             key_value_rule_fmt='"<arg_key>{key}</arg_key>" "\\n" "<arg_value>" {valrule} "</arg_value>"',
             key_value_separator="\\n",
         )

sglang/srt/function_call/gpt_oss_detector.py

@@ -0,0 +1,331 @@
+import json
+import logging
+import re
+from typing import List
+
+from sglang.srt.entrypoints.openai.protocol import Tool
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import (
+    StreamingParseResult,
+    ToolCallItem,
+    _GetInfoFunc,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GptOssDetector(BaseFormatDetector):
+    """
+    Detector for T4-style function calls with channel format.
+
+    Supports two formats:
+    1. Direct function call: <|channel|>commentary to={namespace.function}<|constrain|>json<|message|>{args}<|call|>
+    2. Commentary with action plan: <|channel|>commentary<|message|>{content}<|end|>
+
+    For parallel function calls, each call is self-contained and starts with its own channel:
+    <|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location":"SF"}<|call|>
+    <|channel|>commentary to=functions.search<|constrain|>json<|message|>{"query":"SF attractions"}<|call|>
+
+    Examples:
+    Single: <|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location":"San Francisco"}<|call|>commentary
+    Multiple: <|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location":"Paris"}<|call|>commentary<|channel|>commentary to=functions.search<|constrain|>json<|message|>{"query":"Paris tourism"}<|call|>
+    With Action Plan: <|channel|>commentary<|message|>**Action plan**: 1. Do X 2. Do Y<|end|><|start|>assistant<|channel|>commentary to=functions.x<|constrain|>json<|message|>{"template": "basic_html", "path": "index.html"}<|call|>
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.bot_token = "<|start|>assistant<|channel|>commentary"
+        self.eot_token = "<|call|>"
+        # TODO: no clear indication how parallel tool call response format is
+        self.tool_call_separator = ""
+
+        # Pattern for complete function calls with to= parameter
+        # Handles both <|call|> and <|call|>commentary endings
+        # Also handles optional <|start|>assistant prefix and whitespace after function name
+        self.function_call_pattern = re.compile(
+            r"(?:<\|start\|>assistant)?<\|channel\|>commentary to=([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)\s*"
+            r"<\|constrain\|>json<\|message\|>(.*?)<\|call\|>(?:commentary)?",
+            re.DOTALL,
+        )
+
+        # Pattern for streaming function calls (incomplete)
+        # Also handles optional whitespace after function name
+        self.streaming_pattern = re.compile(
+            r"(?:<\|start\|>assistant)?<\|channel\|>commentary to=([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)\s*"
+            r"<\|constrain\|>json<\|message\|>(.*)",
+            re.DOTALL,
+        )
+
+        # Pattern for commentary with action plan (no to= parameter)
+        self.commentary_pattern = re.compile(
+            r"<\|channel\|>commentary<\|message\|>(.*?)<\|end\|>",
+            re.DOTALL,
+        )
+
+        self._last_arguments = ""
+
+    def has_tool_call(self, text: str) -> bool:
+        """Check if text contains TypeScript-style function call markers."""
+        return self.bot_token in text
+
+    def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+        """Parse TypeScript-style function calls from complete text."""
+        if not self.has_tool_call(text):
+            return StreamingParseResult(normal_text=text, calls=[])
+
+        tool_indices = self._get_tool_indices(tools)
+
+        calls = []
+        tool_index = 0
+
+        # Process the entire text to handle mixed commentary and tool calls
+        normal_text_parts = []
+
+        # Find all commentary sections (both with and without to=)
+        all_commentary_pattern = re.compile(
+            r"<\|channel\|>commentary(?:\s+to=[^<]*)?<\|message\|>(.*?)(?:<\|end\|>|<\|call\|>)",
+            re.DOTALL,
+        )
+
+        # Track processed positions to avoid double-processing
+        processed_ranges = []
+
+        # First, extract all tool calls
+        for match in self.function_call_pattern.finditer(text):
+            full_function_name = match.group(1)
+            args_content = match.group(2)
+            processed_ranges.append((match.start(), match.end()))
+
+            function_name = (
+                full_function_name.split(".")[-1]
+                if "." in full_function_name
+                else full_function_name
+            )
+
+            try:
+                arguments = json.loads(args_content) if args_content.strip() else {}
+            except json.JSONDecodeError:
+                continue
+
+            if function_name in tool_indices:
+                calls.append(
+                    ToolCallItem(
+                        tool_index=tool_index,
+                        name=function_name,
+                        parameters=json.dumps(arguments, ensure_ascii=False),
+                    )
+                )
+                tool_index += 1
+
+        # Then, find non-tool-call commentary sections for normal text
+        for match in all_commentary_pattern.finditer(text):
+            # Check if this match overlaps with any processed tool call
+            match_start, match_end = match.start(), match.end()
+            is_tool_call = any(
+                start <= match_start < end or start < match_end <= end
+                for start, end in processed_ranges
+            )
+
+            # If this commentary is not part of a tool call, include it in normal text
+            if not is_tool_call:
+                content = match.group(1).strip()
+                if content:
+                    normal_text_parts.append(content)
+
+        # Handle remaining text after all matches
+        if processed_ranges:
+            last_match_end = max(end for _, end in processed_ranges)
+            if last_match_end < len(text):
+                remaining_text = text[last_match_end:]
+
+            # Clean up <|start|>assistant prefixes and extract final content
+            # Remove standalone <|start|>assistant prefixes
+            remaining_text = re.sub(r"<\|start\|>assistant(?!\w)", "", remaining_text)
+
+            # Extract content from final channel if present
+            final_pattern = re.compile(
+                r"<\|channel\|>final<\|message\|>(.*?)(?:<\|return\|>|$)", re.DOTALL
+            )
+            final_match = final_pattern.search(remaining_text)
+
+            if final_match:
+                # Get everything before final channel + final channel content
+                before_final = remaining_text[: final_match.start()].strip()
+                final_content = final_match.group(1).strip()
+
+                parts = []
+                if before_final:
+                    parts.append(before_final)
+                if final_content:
+                    parts.append(final_content)
+                remaining_text = " ".join(parts) if parts else ""
+
+            remaining_text = remaining_text.strip()
+
+            if remaining_text:
+                normal_text_parts.append(remaining_text)
+
+        # Combine all normal text parts
+        final_normal_text = " ".join(part for part in normal_text_parts if part).strip()
+        return StreamingParseResult(normal_text=final_normal_text, calls=calls)
+
+    def parse_streaming_increment(
+        self, new_text: str, tools: List[Tool]
+    ) -> StreamingParseResult:
+        """Parse incremental streaming text for TypeScript-style function calls."""
+        self._buffer += new_text
+        current_text = self._buffer
+
+        # Check if we have a tool call
+        has_tool_call = "<|channel|>commentary to=" in current_text
+
+        if not has_tool_call and current_text:
+            # Check for commentary without function calls
+            commentary_match = self.commentary_pattern.search(current_text)
+            if commentary_match:
+                commentary_content = commentary_match.group(1)
+                self._buffer = current_text[commentary_match.end() :]
+                return StreamingParseResult(normal_text=commentary_content, calls=[])
+
+            # Check for final channel content
+            final_pattern = re.compile(
+                r"<\|channel\|>final<\|message\|>(.*?)(?:<\|return\|>|$)",
+                re.DOTALL,
+            )
+            final_match = final_pattern.search(current_text)
+            if final_match:
+                final_content = final_match.group(1).strip()
+                self._buffer = ""
+                return StreamingParseResult(normal_text=final_content, calls=[])
+
+            self._buffer = ""
+            return StreamingParseResult(normal_text=new_text, calls=[])
+
+        if not hasattr(self, "_tool_indices"):
+            self._tool_indices = self._get_tool_indices(tools)
+
+        calls = []
+        try:
+            # Check for streaming function call
+            match = self.streaming_pattern.search(current_text)
+            if match:
+                full_function_name = match.group(1)
+                args_content = match.group(2)
+
+                function_name = (
+                    full_function_name.split(".")[-1]
+                    if "." in full_function_name
+                    else full_function_name
+                )
+
+                # Initialize state if this is the first tool call
+                if self.current_tool_id == -1:
+                    self.current_tool_id = 0
+                    self.prev_tool_call_arr = []
+                    self.streamed_args_for_tool = [""]
+
+                # Ensure we have enough entries in tracking arrays
+                while len(self.prev_tool_call_arr) <= self.current_tool_id:
+                    self.prev_tool_call_arr.append({})
+                while len(self.streamed_args_for_tool) <= self.current_tool_id:
+                    self.streamed_args_for_tool.append("")
+
+                if not self.current_tool_name_sent:
+                    calls.append(
+                        ToolCallItem(
+                            tool_index=self.current_tool_id,
+                            name=function_name,
+                            parameters="",
+                        )
+                    )
+                    self.current_tool_name_sent = True
+                    # Store the tool call info
+                    self.prev_tool_call_arr[self.current_tool_id] = {
+                        "name": function_name,
+                        "arguments": {},
+                    }
+                    self.streamed_args_for_tool[self.current_tool_id] = ""
+
+                # Check if we have a complete function call
+                complete_match = self.function_call_pattern.search(current_text)
+                if complete_match:
+                    args_content = complete_match.group(2)
+
+                    try:
+                        parsed_args = json.loads(args_content)
+                        self.prev_tool_call_arr[self.current_tool_id][
+                            "arguments"
+                        ] = parsed_args
+
+                        # Send complete arguments if we haven't sent them yet
+                        if not self.streamed_args_for_tool[self.current_tool_id]:
+                            # Send the complete arguments as JSON string
+                            calls.append(
+                                ToolCallItem(
+                                    tool_index=self.current_tool_id,
+                                    name=None,
+                                    parameters=json.dumps(
+                                        parsed_args, ensure_ascii=False
+                                    ),
+                                )
+                            )
+                            self.streamed_args_for_tool[self.current_tool_id] = (
+                                json.dumps(parsed_args, ensure_ascii=False)
+                            )
+                    except json.JSONDecodeError:
+                        pass
+
+                    # Remove the completed function call from buffer
+                    remaining_after_call = current_text[complete_match.end() :]
+
+                    # Clean up <|start|>assistant prefixes and extract final content
+                    remaining_after_call = re.sub(
+                        r"<\|start\|>assistant(?!\w)", "", remaining_after_call
+                    )
+
+                    # Extract content from final channel if present
+                    final_pattern = re.compile(
+                        r"<\|channel\|>final<\|message\|>(.*?)(?:<\|return\|>|$)",
+                        re.DOTALL,
+                    )
+                    final_match = final_pattern.search(remaining_after_call)
+
+                    if final_match:
+                        before_final = remaining_after_call[
+                            : final_match.start()
+                        ].strip()
+                        final_content = final_match.group(1).strip()
+
+                        parts = []
+                        if before_final:
+                            parts.append(before_final)
+                        if final_content:
+                            parts.append(final_content)
+                        remaining_after_call = " ".join(parts) if parts else ""
+
+                    self._buffer = remaining_after_call.strip()
+
+                    # Reset state for next tool call
+                    self.current_tool_name_sent = False
+                    self.current_tool_id += 1
+
+                    # Return final content if available
+                    final_text = ""
+                    if final_match and final_content:
+                        final_text = final_content
+                    elif remaining_after_call:
+                        final_text = remaining_after_call
+
+                    return StreamingParseResult(normal_text=final_text, calls=calls)
+
+            return StreamingParseResult(normal_text="", calls=calls)
+
+        except Exception as e:
+            logger.error(f"Error in parse_streaming_increment: {e}")
+            return StreamingParseResult(normal_text=current_text, calls=[])
+
+    def structure_info(self) -> _GetInfoFunc:
+        raise NotImplementedError()
+
+    def build_ebnf(self, tools: List[Tool]) -> str:
+        raise NotImplementedError()