sglang 0.5.3rc0__py3-none-any.whl → 0.5.4__py3-none-any.whl

sglang/srt/function_call/function_call_parser.py

@@ -1,10 +1,11 @@
 import logging
-from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Type, Union
+from typing import Dict, List, Literal, Optional, Set, Tuple, Type, Union
 
 from sglang.srt.entrypoints.openai.protocol import (
-    StructuralTagResponseFormat,
+    LegacyStructuralTagResponseFormat,
     StructuresResponseFormat,
     Tool,
+    ToolCallConstraint,
     ToolChoice,
 )
 from sglang.srt.function_call.base_format_detector import BaseFormatDetector
@@ -20,6 +21,7 @@ from sglang.srt.function_call.pythonic_detector import PythonicDetector
 from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
 from sglang.srt.function_call.qwen25_detector import Qwen25Detector
 from sglang.srt.function_call.step3_detector import Step3Detector
+from sglang.srt.function_call.utils import get_json_schema_constraint
 
 logger = logging.getLogger(__name__)
 
@@ -34,21 +36,22 @@ class FunctionCallParser:
     """
 
     ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = {
-        "llama3": Llama32Detector,
-        "qwen25": Qwen25Detector,
-        "mistral": MistralDetector,
         "deepseekv3": DeepSeekV3Detector,
         "deepseekv31": DeepSeekV31Detector,
-        "pythonic": PythonicDetector,
+        "glm": Glm4MoeDetector,
+        "glm45": Glm4MoeDetector,
+        "gpt-oss": GptOssDetector,
         "kimi_k2": KimiK2Detector,
+        "llama3": Llama32Detector,
+        "mistral": MistralDetector,
+        "pythonic": PythonicDetector,
+        "qwen": Qwen25Detector,
+        "qwen25": Qwen25Detector,
         "qwen3_coder": Qwen3CoderDetector,
-        "glm45": Glm4MoeDetector,
         "step3": Step3Detector,
-        "gpt-oss": GptOssDetector,
     }
 
     def __init__(self, tools: List[Tool], tool_call_parser: str):
-        detector: Type[BaseFormatDetector] = None
         detector_class = self.ToolCallParserEnum.get(tool_call_parser)
         if detector_class:
             detector = detector_class()
@@ -120,7 +123,7 @@ class FunctionCallParser:
 
         return final_normal_text, final_calls
 
-    def get_structure_tag(self) -> StructuralTagResponseFormat:
+    def get_structure_tag(self) -> LegacyStructuralTagResponseFormat:
         """
         Generate a structural tag response format for all available tools.
 
@@ -148,7 +151,9 @@ class FunctionCallParser:
             )
             tool_trigger_set.add(info.trigger)
 
-        return StructuralTagResponseFormat(
+        # TODO(dark): move this into new structural tag format
+        # This requires all grammar backend support the new format
+        return LegacyStructuralTagResponseFormat(
             type="structural_tag",
             structures=tool_structures,
             triggers=list(tool_trigger_set),
@@ -156,7 +161,7 @@ class FunctionCallParser:
 
     def get_structure_constraint(
         self, tool_choice: Union[ToolChoice, Literal["auto", "required"]]
-    ) -> Optional[Tuple[str, Any]]:
+    ) -> Optional[ToolCallConstraint]:
         """
         Returns the appropriate structure constraint for tool calls based on the tool_choice.
         The constraint is used to guide the model's output format.
@@ -175,11 +180,11 @@ class FunctionCallParser:
             and tool_choice == "auto"
             and any(tool.function.strict for tool in self.tools)
         ):
-            strict_tag = self.get_structure_tag()
-            return ("structural_tag", strict_tag)
+            tag = self.get_structure_tag()
+            return ("structural_tag", tag)
         elif tool_choice == "required" or isinstance(tool_choice, ToolChoice):
-            ebnf = self.get_ebnf(tool_choice)
-            return ("ebnf", ebnf) if ebnf is not None else None
+            json_schema = get_json_schema_constraint(self.tools, tool_choice)
+            return ("json_schema", json_schema)
 
     def get_ebnf(
         self, tool_choice: Union[ToolChoice, Literal["required"]]

sglang/srt/function_call/glm4_moe_detector.py

@@ -6,11 +6,7 @@ from typing import List
 
 from sglang.srt.entrypoints.openai.protocol import Tool
 from sglang.srt.function_call.base_format_detector import BaseFormatDetector
-from sglang.srt.function_call.core_types import (
-    StreamingParseResult,
-    StructureInfo,
-    _GetInfoFunc,
-)
+from sglang.srt.function_call.core_types import StreamingParseResult, _GetInfoFunc
 from sglang.srt.function_call.ebnf_composer import EBNFComposer
 
 logger = logging.getLogger(__name__)
@@ -39,7 +35,7 @@ def parse_arguments(json_value):
 
 class Glm4MoeDetector(BaseFormatDetector):
     """
-    Detector for GLM-4.5 models.
+    Detector for GLM-4.5 and GLM-4.6 models.
     Assumes function call format:
       <tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>北京</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2024-06-27</arg_value>\n</tool_call>\n<tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>上海</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2024-06-27</arg_value>\n</tool_call>
     """
@@ -53,7 +49,7 @@ class Glm4MoeDetector(BaseFormatDetector):
         self.func_arg_regex = r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>"
 
     def has_tool_call(self, text: str) -> bool:
-        """Check if the text contains a glm-4.5 format tool call."""
+        """Check if the text contains a glm-4.5 / glm-4.6 format tool call."""
         return self.bot_token in text
 
     def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
@@ -102,7 +98,7 @@ class Glm4MoeDetector(BaseFormatDetector):
         self, new_text: str, tools: List[Tool]
     ) -> StreamingParseResult:
         """
-        Streaming incremental parsing tool calls for GLM-4.5 format.
+        Streaming incremental parsing tool calls for GLM-4.5 and GLM-4.6 format.
         """
         self._buffer += new_text
         current_text = self._buffer

sglang/srt/function_call/gpt_oss_detector.py

@@ -31,7 +31,7 @@ class GptOssDetector(BaseFormatDetector):
 
         # Pattern to extract function name and JSON from tool_call event content
         self.tool_extract_pattern = re.compile(
-            r"to=([a-zA-Z_][a-zA-Z0-9_.]*)\s*<\|constrain\|>json<\|message\|>(.*?)(?:<\|call\|>|$)",
+            r"to=([a-zA-Z_][a-zA-Z0-9_.-]*)\s*<\|constrain\|>json<\|message\|>(.*?)(?:<\|call\|>|$)",
             re.DOTALL,
         )
 
@@ -81,6 +81,29 @@ class GptOssDetector(BaseFormatDetector):
         # Always use HarmonyParser for parsing to ensure proper filtering
         events = self.harmony_parser.parse(new_text)
 
+        # If there are no parsed events and the chunk contains no Harmony structural
+        # markers, treat it as plain text and pass it through. This fixes a bug where
+        # normal content was held in the buffer when tools were provided but not used.
+        if not events:
+            has_harmony_markers = any(
+                marker in self._buffer
+                for marker in (
+                    "<|start|>",
+                    "<|channel|>",
+                    "<|message|>",
+                    "<|constrain|>",
+                    "<|end|>",
+                    "<|call|>",
+                    "<|return|>",
+                    "assistantfinal",
+                )
+            )
+            if not has_harmony_markers:
+                # Plain text with no tool markers — emit as normal content
+                out = self._buffer
+                self._buffer = ""
+                return StreamingParseResult(normal_text=out, calls=[])
+
         # Quick check if we might have tool calls
         if (
             "<|channel|>commentary to=" not in self._buffer

sglang/srt/function_call/json_array_parser.py

@@ -0,0 +1,61 @@
+from typing import List
+
+from sglang.srt.entrypoints.openai.protocol import Tool
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import StreamingParseResult
+
+
+class JsonArrayParser(BaseFormatDetector):
+    """
+    Parser for JSON array tool calls when JSON schema constraints are active.
+
+    This parser is used when tool_choice="required" or a specific tool is named,
+    bypassing model-specific parsers in favor of direct JSON array parsing.
+    """
+
+    def __init__(self):
+        super().__init__()
+        # Configure for JSON array parsing
+        self.bot_token = "["
+        self.eot_token = "]"
+        self.tool_call_separator = ","
+
+    def has_tool_call(self, text: str) -> bool:
+        """
+        Check if the given text contains a JSON tool call (array or single object).
+        """
+        return "[" in text or "{" in text
+
+    def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+        """
+        Parse JSON tool calls using the base class implementation.
+        """
+        raise NotImplementedError(
+            "Detect and parse not supported for JSON schema constraints."
+        )
+
+    def build_ebnf(self, tools: List[Tool]) -> str:
+        """
+        Build an EBNF grammar for constrained generation.
+        This is not used for JSON schema constraints as they are handled
+        by the constraint backends directly.
+        """
+        raise NotImplementedError(
+            "EBNF generation is not supported for JSON schema constraints."
+        )
+
+    def parse_streaming_increment(
+        self, new_text: str, tools: List[Tool]
+    ) -> StreamingParseResult:
+        """
+        Streaming incremental parsing with tool validation.
+        """
+        return super().parse_streaming_increment(new_text, tools)
+
+    def structure_info(self) -> callable:
+        """
+        Return a function that creates StructureInfo for constrained generation.
+        This is not used for JSON schema constraints as they are handled
+        by the constraint backends directly.
+        """
+        raise NotImplementedError("structure_info not used for JSON schema constraints")

sglang/srt/function_call/kimik2_detector.py

@@ -50,6 +50,11 @@ class KimiK2Detector(BaseFormatDetector):
 
         self._last_arguments = ""
 
+        # Robust parser for ids like "functions.search:0" or fallback "search:0"
+        self.tool_call_id_regex = re.compile(
+            r"^(?:functions\.)?(?P<name>[\w\.]+):(?P<index>\d+)$"
+        )
+
     def has_tool_call(self, text: str) -> bool:
         """Check if the text contains a KimiK2 format tool call."""
         return self.bot_token in text
@@ -76,14 +81,18 @@ class KimiK2Detector(BaseFormatDetector):
             tool_calls = []
             for match in function_call_tuples:
                 function_id, function_args = match
-                function_name = function_id.split(".")[1].split(":")[0]
-                function_idx = int(function_id.split(".")[1].split(":")[1])
+                m = self.tool_call_id_regex.match(function_id)
+                if not m:
+                    logger.warning("Unexpected tool_call_id format: %s", function_id)
+                    continue
+                function_name = m.group("name")
+                function_idx = int(m.group("index"))
 
                 logger.info(f"function_name {function_name}")
 
                 tool_calls.append(
                     ToolCallItem(
-                        tool_index=function_idx,  # Use the call index in the response, not tool position
+                        tool_index=function_idx,
                         name=function_name,
                         parameters=function_args,
                     )
@@ -128,7 +137,11 @@ class KimiK2Detector(BaseFormatDetector):
                 function_id = match.group("tool_call_id")
                 function_args = match.group("function_arguments")
 
-                function_name = function_id.split(".")[1].split(":")[0]
+                m = self.tool_call_id_regex.match(function_id)
+                if not m:
+                    logger.warning("Unexpected tool_call_id format: %s", function_id)
+                    return StreamingParseResult(normal_text="", calls=calls)
+                function_name = m.group("name")
 
                 # Initialize state if this is the first tool call
                 if self.current_tool_id == -1:

sglang/srt/function_call/utils.py

@@ -1,10 +1,13 @@
-import json
 from json import JSONDecodeError, JSONDecoder
-from typing import Any, Tuple
+from json.decoder import WHITESPACE
+from typing import Any, List, Literal, Optional, Tuple, Union
 
+import orjson
 import partial_json_parser
 from partial_json_parser.core.options import Allow
 
+from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
+
 
 def _find_common_prefix(s1: str, s2: str) -> str:
     prefix = ""
@@ -37,16 +40,104 @@ def _partial_json_loads(input_str: str, flags: Allow) -> Tuple[Any, int]:
     """
     try:
         return (partial_json_parser.loads(input_str, flags), len(input_str))
-    except JSONDecodeError as e:
-        if "Extra data" in e.msg:
-            dec = JSONDecoder()
-            return dec.raw_decode(input_str)
+    except (JSONDecodeError, IndexError) as e:
+        msg = getattr(e, "msg", str(e))
+        if "Extra data" in msg or "pop from empty list" in msg:
+            start = WHITESPACE.match(input_str, 0).end()
+            obj, end = JSONDecoder().raw_decode(input_str, start)
+            return obj, end
         raise
 
 
 def _is_complete_json(input_str: str) -> bool:
     try:
-        json.loads(input_str)
+        orjson.loads(input_str)
         return True
     except JSONDecodeError:
         return False
+
+
+def _get_tool_schema_defs(tools: List[Tool]) -> dict:
+    """
+    Get consolidated $defs from all tools, validating for conflicts.
+
+    Args:
+        tools: List of tools to process
+
+    Returns:
+        Dictionary of consolidated $defs from all tools
+
+    Raises:
+        ValueError: If conflicting $defs are found
+    """
+    all_defs = {}
+    for tool in tools:
+        if tool.function.parameters is None:
+            continue
+        defs = tool.function.parameters.get("$defs", {})
+        for def_name, def_schema in defs.items():
+            if def_name in all_defs and all_defs[def_name] != def_schema:
+                raise ValueError(
+                    f"Tool definition '{def_name}' has "
+                    "multiple schemas, which is not "
+                    "supported."
+                )
+            else:
+                all_defs[def_name] = def_schema
+    return all_defs
+
+
+def _get_tool_schema(tool: Tool) -> dict:
+    return {
+        "properties": {
+            "name": {"type": "string", "enum": [tool.function.name]},
+            "parameters": (
+                tool.function.parameters
+                if tool.function.parameters
+                else {"type": "object", "properties": {}}
+            ),
+        },
+        "required": ["name", "parameters"],
+    }
+
+
+def get_json_schema_constraint(
+    tools: List[Tool], tool_choice: Union[ToolChoice, Literal["required"]]
+) -> Optional[dict]:
+    """
+    Get the JSON schema constraint for the specified tool choice.
+
+    Args:
+        tool_choice: The tool choice specification
+
+    Returns:
+        JSON schema dict, or None if no valid tools found
+    """
+
+    if isinstance(tool_choice, ToolChoice):
+        # For specific function choice, return the user's parameters schema directly
+        fn_name = tool_choice.function.name
+        for tool in tools:
+            if tool.function.name == fn_name:
+                return {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 1,
+                    "items": _get_tool_schema(tool),
+                }
+        return None
+    elif tool_choice == "required":
+        json_schema = {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "type": "object",
+                "anyOf": [_get_tool_schema(tool) for tool in tools],
+            },
+        }
+        json_schema_defs = _get_tool_schema_defs(tools)
+        if json_schema_defs:
+            json_schema["$defs"] = json_schema_defs
+        return json_schema
+
+    return None

sglang/srt/grpc/compile_proto.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+Compile protobuf files for SGLang gRPC server.
+
+This script compiles .proto files to Python code using grpc_tools.protoc.
+It generates:
+- *_pb2.py (protobuf message classes)
+- *_pb2_grpc.py (gRPC service classes)
+- *_pb2.pyi (type hints for mypy/IDEs)
+
+Usage:
+    python compile_proto.py [--check] [--proto-file PROTO_FILE]
+
+Options:
+    --check         Check if regeneration is needed (exit 1 if needed)
+    --proto-file    Specify proto file (default: sglang_scheduler.proto)
+
+### Install Dependencies
+pip install "grpcio==1.75.1" "grpcio-tools==1.75.1"
+
+### Run Script
+cd python/sglang/srt/grpc
+python compile_proto.py
+"""
+
+
+import argparse
+import subprocess
+import sys
+from importlib.metadata import version
+from pathlib import Path
+
+GRPC_VERSION = "1.75.1"
+
+
+def get_file_mtime(path: Path) -> float:
+    """Get file modification time, return 0 if file doesn't exist."""
+    try:
+        return path.stat().st_mtime
+    except FileNotFoundError:
+        return 0.0
+
+
+def check_regeneration_needed(proto_file: Path, output_dir: Path) -> bool:
+    """Check if proto files are newer than generated files."""
+    proto_mtime = get_file_mtime(proto_file)
+
+    generated_files = [
+        output_dir / f"{proto_file.stem}_pb2.py",
+        output_dir / f"{proto_file.stem}_pb2_grpc.py",
+        output_dir / f"{proto_file.stem}_pb2.pyi",
+    ]
+
+    for gen_file in generated_files:
+        if get_file_mtime(gen_file) < proto_mtime:
+            return True
+
+    return False
+
+
+def compile_proto(proto_file: Path, output_dir: Path, verbose: bool = True) -> bool:
+    """Compile the protobuf file to Python."""
+
+    if not proto_file.exists():
+        print(f"Error: Proto file not found: {proto_file}")
+        return False
+
+    if verbose:
+        print(f"Found proto file: {proto_file}")
+
+    # Check if grpc_tools is available
+    try:
+        import grpc_tools.protoc  # noqa: F401
+    except ImportError:
+        print("Error: grpcio-tools not installed")
+        print(
+            f'Install with: pip install "grpcio-tools=={GRPC_VERSION}" "grpcio=={GRPC_VERSION}"'
+        )
+        return False
+
+    grpc_tools_version = version("grpcio-tools")
+    grpc_version = version("grpcio")
+    if grpc_tools_version != GRPC_VERSION or grpc_version != GRPC_VERSION:
+        raise RuntimeError(
+            f"Error: grpcio-tools version {grpc_tools_version} and grpcio version {grpc_version} detected, but {GRPC_VERSION} is required."
+        )
+
+    # Compile command
+    cmd = [
+        sys.executable,
+        "-m",
+        "grpc_tools.protoc",
+        f"-I{proto_file.parent}",
+        f"--python_out={output_dir}",
+        f"--grpc_python_out={output_dir}",
+        f"--pyi_out={output_dir}",  # Generate type stubs
+        str(proto_file.name),
+    ]
+
+    if verbose:
+        print(f"Running: {' '.join(cmd)}")
+
+    # Run protoc
+    result = subprocess.run(cmd, capture_output=True, text=True, cwd=proto_file.parent)
+
+    if result.returncode != 0:
+        print(f"Error compiling proto:")
+        print(result.stderr)
+        if result.stdout:
+            print(result.stdout)
+        return False
+
+    # Verify generated files exist
+    generated_files = [
+        f"{proto_file.stem}_pb2.py",
+        f"{proto_file.stem}_pb2_grpc.py",
+        f"{proto_file.stem}_pb2.pyi",
+    ]
+
+    missing_files = []
+    for gen_file in generated_files:
+        if not (output_dir / gen_file).exists():
+            missing_files.append(gen_file)
+
+    if missing_files:
+        print(f"Error: Expected generated files not found: {missing_files}")
+        return False
+
+    if verbose:
+        print("Successfully compiled protobuf files:")
+        for gen_file in generated_files:
+            print(f"  - {output_dir}/{gen_file}")
+
+    # Fix imports in generated files
+    fix_imports(output_dir, proto_file.stem, verbose)
+
+    return True
+
+
+def fix_imports(output_dir: Path, proto_stem: str, verbose: bool = True) -> None:
+    """Fix imports in generated files to use relative imports."""
+    grpc_file = output_dir / f"{proto_stem}_pb2_grpc.py"
+
+    if grpc_file.exists():
+        content = grpc_file.read_text()
+        # Change absolute import to relative import
+        old_import = f"import {proto_stem}_pb2"
+        new_import = f"from . import {proto_stem}_pb2"
+
+        if old_import in content:
+            content = content.replace(old_import, new_import)
+            grpc_file.write_text(content)
+            if verbose:
+                print("Fixed imports in generated files")
+
+
+def add_generation_header(output_dir: Path, proto_stem: str) -> None:
+    """Add header to generated files indicating they are auto-generated."""
+    header = """# This file is auto-generated. Do not edit manually.
+# Regenerate with: python compile_proto.py
+
+"""
+
+    files_to_update = [f"{proto_stem}_pb2.py", f"{proto_stem}_pb2_grpc.py"]
+
+    for filename in files_to_update:
+        file_path = output_dir / filename
+        if file_path.exists():
+            content = file_path.read_text()
+            if not content.startswith("# This file is auto-generated"):
+                file_path.write_text(header + content)
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(
+        description="Compile protobuf files for SGLang gRPC server",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Check if regeneration is needed (exit 1 if needed)",
+    )
+
+    parser.add_argument(
+        "--proto-file",
+        type=str,
+        default="sglang_scheduler.proto",
+        help="Proto file to compile (default: sglang_scheduler.proto)",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=True,
+        help="Verbose output (default: True)",
+    )
+
+    parser.add_argument(
+        "-q", "--quiet", action="store_true", help="Quiet mode (overrides verbose)"
+    )
+
+    args = parser.parse_args()
+
+    # Handle verbosity
+    verbose = args.verbose and not args.quiet
+
+    # Get paths
+    script_dir = Path(__file__).parent
+    proto_file = script_dir / args.proto_file
+    output_dir = script_dir
+
+    # Check mode
+    if args.check:
+        if check_regeneration_needed(proto_file, output_dir):
+            if verbose:
+                print("Proto files need regeneration")
+            sys.exit(1)
+        else:
+            if verbose:
+                print("Generated files are up to date")
+            sys.exit(0)
+
+    # Compile mode
+    success = compile_proto(proto_file, output_dir, verbose)
+
+    if success:
+        # Add generation headers
+        add_generation_header(output_dir, proto_file.stem)
+
+        if verbose:
+            print("\n✅ Protobuf compilation successful!")
+            print("Generated files are ready for use")
+    else:
+        if verbose:
+            print("\n❌ Protobuf compilation failed!")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()