sglang 0.4.6.post3__py3-none-any.whl → 0.4.6.post5__py3-none-any.whl

sglang/srt/function_call/ebnf_composer.py

@@ -0,0 +1,234 @@
+from typing import Literal, Optional
+
+
+class EBNFComposer:
+    # Adapted from https://xgrammar.mlc.ai/docs/how_to/ebnf_guided_generation.html#try-out-via-hf-transformers
+    json_grammar_ebnf_str = r"""
+        json ::= basic_array | basic_object
+        basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object
+        basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"?
+        basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)?
+        basic_string ::= (([\"] basic_string_1 [\"]))
+        basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1
+        escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]
+        basic_boolean ::= "true" | "false"
+        basic_null ::= "null"
+        basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]"
+        basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}"
+        ws ::= [ \n\t]*
+        """
+
+    pythonic_grammar_ebnf_str = r"""
+        pythonic ::= basic_number | basic_string | basic_array | "True" | "False" | "None"
+        basic_any ::= basic_number | basic_string | basic_array | basic_object
+        basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)?
+        basic_string ::= (([\"] basic_string_1 [\"]))
+        basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1
+        escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]
+        basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]"
+        basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}"
+        ws ::= [ \n\t]*
+    """
+
+    TOOL_CALLS_MAP = {
+        "pythonic": '"[" function_call ("," function_call)* "]"',
+        "json": "function_call",
+    }
+
+    CALL_RULE_MAP = {
+        "pythonic": 'call_{name} ::= "{name}" "(" {arguments_rule} ")"',
+        "json": 'call_{name} ::= "{{" "\\"name\\"" ":" "\\"{name}\\"" ", " "\\"arguments\\"" ":" {arguments_rule} "}}"',
+    }
+
+    ARGUMENTS_RULE_MAP = {
+        "pythonic": "{arg_rules}",
+        "json": '"{{" {arg_rules} "}}"',
+    }
+
+    KEY_VALUE_RULE_MAP = {
+        "pythonic": '"{key}" "=" {valrule}',
+        "json": '"\\"{key}\\"" ":" {valrule}',
+    }
+
+    JSON_TYPE_MAPPING = {
+        "string": "basic_string",
+        "number": "basic_number",
+        "integer": "basic_number",
+        "boolean": "basic_boolean",
+        "null": "basic_null",
+        "array": "basic_array",
+        "object": "basic_object",
+    }
+
+    PYTHONIC_TYPE_MAPPING = {
+        "string": "basic_string",
+        "number": "basic_number",
+        "integer": "basic_number",
+        "boolean": '"True" | "False"',
+        "null": '"None"',
+        "array": "basic_array",
+        "object": "basic_object",
+    }
+
+    @staticmethod
+    def get_value_rule(
+        prop: dict, function_format: Literal["pythonic", "json"] = "json"
+    ) -> str:
+        if "enum" in prop:
+            return EBNFComposer._handle_enum(prop, function_format)
+
+        if "type" in prop:
+            return EBNFComposer._handle_type(prop, function_format)
+
+        return function_format
+
+    @staticmethod
+    def _handle_enum(prop: dict, function_format: str) -> str:
+        """Handle enum properties by formatting each value according to type and format."""
+        enum_values = prop["enum"]
+        prop_type = prop.get("type", "string")
+
+        # Define formatters for different type/format combinations
+        formatters = {
+            ("string", "json"): lambda v: f'"\\"{v}\\""',
+            ("string", "pythonic"): lambda v: f'"\\"{v}\\""',
+            ("number", "json"): str,
+            ("number", "pythonic"): str,
+            ("integer", "json"): str,
+            ("integer", "pythonic"): str,
+            ("boolean", "json"): lambda v: "true" if v else "false",
+            ("boolean", "pythonic"): lambda v: "True" if v else "False",
+        }
+
+        # Get the formatter or default to string handling
+        formatter = formatters.get(
+            (prop_type, function_format),
+            formatters[("string", function_format)],  # Default to string handling
+        )
+
+        formatted_values = [formatter(value) for value in enum_values]
+        enum_rule = " | ".join(formatted_values)
+
+        # Wrap in parentheses if there are multiple values to ensure correct EBNF precedence
+        if len(formatted_values) > 1:
+            enum_rule = f"({enum_rule})"
+
+        return enum_rule
+
+    @staticmethod
+    def _handle_type(prop: dict, function_format: str) -> str:
+        """Handle type properties using the appropriate type mapping."""
+        prop_type = prop["type"]
+        type_mapping = (
+            EBNFComposer.PYTHONIC_TYPE_MAPPING
+            if function_format == "pythonic"
+            else EBNFComposer.JSON_TYPE_MAPPING
+        )
+
+        if isinstance(prop_type, list):
+            type_rules = [
+                type_mapping[single_type]
+                for single_type in prop_type
+                if single_type in type_mapping
+            ]
+            return " | ".join(type_rules) if type_rules else function_format
+
+        return type_mapping.get(prop_type, function_format)
+
+    @staticmethod
+    def build_ebnf(
+        tools,
+        *,
+        call_rule_fmt: Optional[str] = None,
+        function_format: Literal["pythonic", "json"] = "json",
+        bot_token: Optional[str] = None,
+        eot_token: Optional[str] = None,
+        tool_call_separator: Optional[str] = None,
+    ):
+        """
+        Generalized EBNF builder for all detectors.
+        Args:
+            tools: List of Tool objects to generate EBNF grammar for
+            call_rule_fmt: Optional custom format string for call_{name} rule. It should define each function call's format, with
+                the placeholders {name} for the function name and {arguments_rule} for the arguments rule. If None, a default
+                format based on function_format will be used.
+            function_format: The format of function calls, either "pythonic" or "json"
+            bot_token: The token that indicates the start of a tool call section
+            eot_token: The token that indicates the end of a tool call section
+            tool_call_separator: The separator between multiple tool calls
+        """
+        # =================================================================
+        # Step 1: Determine the root tool calls rule
+        # =================================================================
+        if bot_token and eot_token:
+            if tool_call_separator:
+                root_rule = f'"{bot_token}" function_call ( "{tool_call_separator}" function_call )* "{eot_token}"'
+            else:
+                root_rule = f'"{bot_token}" function_call "{eot_token}"'
+        else:
+            root_rule = EBNFComposer.TOOL_CALLS_MAP[function_format]
+
+        # =================================================================
+        # Step 2: Build the header rules
+        # =================================================================
+        ebnf_lines = [
+            f"root ::= {root_rule}",
+            "function_call ::= "
+            + " | ".join([f"call_{tool.function.name}" for tool in tools]),
+        ]
+
+        # =================================================================
+        # Step 3: Set up formatting templates
+        # =================================================================
+        call_template = (
+            f"call_{{name}} ::= {call_rule_fmt}"
+            if call_rule_fmt
+            else EBNFComposer.CALL_RULE_MAP[function_format]
+        )
+        args_template = EBNFComposer.ARGUMENTS_RULE_MAP[function_format]
+        key_value_template = EBNFComposer.KEY_VALUE_RULE_MAP[function_format]
+
+        # =================================================================
+        # Step 4: Build rules for each tool
+        # =================================================================
+        for tool in tools:
+            tool_name = tool.function.name
+            params = tool.function.parameters or {}
+            properties = params.get("properties", {})
+            required_props = set(params.get("required", []))
+
+            # Build argument rules for this tool
+            arg_rules = []
+            for prop_name, prop_schema in properties.items():
+                value_rule = EBNFComposer.get_value_rule(prop_schema, function_format)
+                # Create key=value pair
+                pair = key_value_template.format(key=prop_name, valrule=value_rule)
+
+                if prop_name not in required_props:
+                    pair = f"[ {pair} ]"
+
+                arg_rules.append(pair)
+
+            # Combine all argument rules
+            combined_args = ' "," '.join(arg_rules) if arg_rules else ""
+            arguments_rule = args_template.format(arg_rules=combined_args)
+
+            # Add the function call rule and its arguments rule
+            ebnf_lines.append(
+                call_template.format(
+                    name=tool_name, arguments_rule=f"arguments_{tool_name}"
+                )
+            )
+            ebnf_lines.append(f"arguments_{tool_name} ::= {arguments_rule}")
+
+        # =================================================================
+        # Step 5: Add base grammar rules
+        # =================================================================
+        base_grammar = (
+            EBNFComposer.pythonic_grammar_ebnf_str
+            if function_format == "pythonic"
+            else EBNFComposer.json_grammar_ebnf_str
+        )
+        ebnf_lines.append(base_grammar)
+
+        return "\n".join(ebnf_lines)

sglang/srt/function_call/function_call_parser.py

@@ -0,0 +1,175 @@
+from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Type, Union
+
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import ToolCallItem
+from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector
+from sglang.srt.function_call.llama32_detector import Llama32Detector
+from sglang.srt.function_call.mistral_detector import MistralDetector
+from sglang.srt.function_call.pythonic_detector import PythonicDetector
+from sglang.srt.function_call.qwen25_detector import Qwen25Detector
+from sglang.srt.openai_api.protocol import (
+    StructuralTagResponseFormat,
+    StructuresResponseFormat,
+    Tool,
+    ToolChoice,
+)
+
+
+class FunctionCallParser:
+    """
+    Parser for function/tool calls in model outputs.
+
+    This class handles both streaming and non-streaming parsing of function calls using a detector.
+    In streaming scenarios, each time new_text is received, it calls detector.parse_streaming_increment
+    and returns the resulting normal_text and calls to the upper layer (or SSE).
+    """
+
+    ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = {
+        "llama3": Llama32Detector,
+        "qwen25": Qwen25Detector,
+        "mistral": MistralDetector,
+        "deepseekv3": DeepSeekV3Detector,
+        "pythonic": PythonicDetector,
+    }
+
+    def __init__(self, tools: List[Tool], tool_call_parser: str):
+        detector: Type[BaseFormatDetector] = None
+        detector_class = self.ToolCallParserEnum.get(tool_call_parser)
+        if detector_class:
+            detector = detector_class()
+        else:
+            raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
+
+        self.detector = detector
+        self.tools = tools
+
+    def has_tool_call(self, text: str) -> bool:
+        """
+        Check if the given text contains a tool call in the format supported by this parser.
+        This delegates to the detector's implementation.
+
+        Args:
+            text: The text to check for tool calls
+
+        Returns:
+            True if the text contains a tool call, False otherwise
+        """
+        return self.detector.has_tool_call(text)
+
+    def parse_non_stream(self, full_text: str) -> Tuple[str, list[ToolCallItem]]:
+        """
+        One-time parsing of the full text to extract tool calls.
+
+        Args:
+            full_text: The complete text to parse
+
+        Returns:
+            A tuple containing:
+            - The remaining text after parsing that was not consumed by the detector (can be treated as normal text)
+            - A list of tool calls parsed from the text
+        """
+        parsed_result = self.detector.detect_and_parse(full_text, self.tools)
+        tool_call_list = parsed_result.calls
+        if tool_call_list:
+            return parsed_result.normal_text, tool_call_list
+        else:
+            return full_text, []
+
+    def parse_stream_chunk(self, chunk_text: str) -> Tuple[str, list[ToolCallItem]]:
+        """
+        Streaming incremental parsing of chunks of text as they arrive.
+
+        Args:
+            chunk_text: The new chunk of text to parse
+
+        Returns:
+            A tuple containing:
+            - The normal text that should be displayed to the user
+            - A list of tool calls parsed from the chunk
+        """
+        final_normal_text = ""
+        final_calls = []
+
+        sp_result = self.detector.parse_streaming_increment(chunk_text, self.tools)
+        if sp_result.normal_text:
+            final_normal_text = sp_result.normal_text
+        if sp_result.calls:
+            final_calls.extend(sp_result.calls)
+            final_normal_text = sp_result.normal_text
+
+        return final_normal_text, final_calls
+
+    def get_structure_tag(self) -> StructuralTagResponseFormat:
+        """
+        Generate a structural tag response format for all available tools.
+
+        This creates the necessary structural tags that guide the model's output format.
+        """
+        tool_structures: List[StructuresResponseFormat] = list()
+        tool_trigger_set: Set[str] = set()
+
+        get_structure_info = self.detector.structure_info()
+        for tool in self.tools:
+            function = tool.function
+            name = function.name
+            assert name is not None
+            info = get_structure_info(name)
+
+            # accept all if not strict, otherwise only accept the schema
+            schema = function.parameters if function.strict else {}
+
+            tool_structures.append(
+                StructuresResponseFormat(
+                    begin=info.begin,
+                    schema=schema,  # type: ignore
+                    end=info.end,
+                )
+            )
+            tool_trigger_set.add(info.trigger)
+
+        return StructuralTagResponseFormat(
+            type="structural_tag",
+            structures=tool_structures,
+            triggers=list(tool_trigger_set),
+        )
+
+    def get_structure_constraint(
+        self, tool_choice: Union[ToolChoice, Literal["auto", "required"]]
+    ) -> Optional[Tuple[str, Any]]:
+        """
+        Returns the appropriate structure constraint for tool calls based on the tool_choice.
+        The constraint is used to guide the model's output format.
+
+        Args:
+            tool_choice: The tool choice setting from the request
+
+        Returns:
+            A tuple of (constraint_type, constraint_value) to be added to sampling parameters,
+            or None if no constraint applies.
+        """
+        # NOTE: structural_tag only supports JSON-compatible content between the begin and end.
+        # It cannot parse or validate Python syntax like function calls.
+        if (
+            not isinstance(self.detector, PythonicDetector)
+            and tool_choice == "auto"
+            and any(tool.function.strict for tool in self.tools)
+        ):
+            strict_tag = self.get_structure_tag()
+            return ("structural_tag", strict_tag)
+        elif tool_choice == "required" or isinstance(tool_choice, ToolChoice):
+            ebnf = self.get_ebnf(tool_choice)
+            return ("ebnf", ebnf) if ebnf is not None else None
+
+    def get_ebnf(
+        self, tool_choice: Union[ToolChoice, Literal["required"]]
+    ) -> Optional[str]:
+        """
+        Get the EBNF grammar for the specified tool choice.
+        """
+        filtered_tools = []
+        if isinstance(tool_choice, ToolChoice):
+            fn_name = tool_choice.function.name
+            filtered_tools = [t for t in self.tools if t.function.name == fn_name]
+        else:
+            filtered_tools = self.tools
+        return self.detector.build_ebnf(filtered_tools)

sglang/srt/function_call/llama32_detector.py

@@ -0,0 +1,74 @@
+import json
+import logging
+from typing import List
+
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import (
+    StreamingParseResult,
+    StructureInfo,
+    _GetInfoFunc,
+)
+from sglang.srt.function_call.ebnf_composer import EBNFComposer
+from sglang.srt.openai_api.protocol import Tool
+
+logger = logging.getLogger(__name__)
+
+
+class Llama32Detector(BaseFormatDetector):
+    """
+    Detector for Llama 3.2 models.
+    Assumes function call format:
+      <|python_tag|>{"name":"xxx", "arguments":{...}}
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.bot_token = "<|python_tag|>"
+
+    def has_tool_call(self, text: str) -> bool:
+        """Check if the text contains a Llama 3.2 format tool call."""
+        # depending on the prompt format the Llama model may or may not
+        # prefix the output with the <|python_tag|> token
+        return "<|python_tag|>" in text or text.startswith("{")
+
+    def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+        """Parse function calls from text, handling multiple JSON objects."""
+        if "<|python_tag|>" not in text and not text.startswith("{"):
+            return StreamingParseResult(normal_text=text, calls=[])
+
+        if "<|python_tag|>" in text:
+            normal_text, action_text = text.split("<|python_tag|>")
+        else:
+            normal_text, action_text = "", text
+
+        # Split by semicolon and process each part
+        json_parts = [part.strip() for part in action_text.split(";") if part.strip()]
+        all_actions = []
+        for part in json_parts:
+            try:
+                # Parse each individual JSON object
+                action = json.loads(part)
+                all_actions.append(action)
+            except json.JSONDecodeError as e:
+                logger.warning(f"Failed to parse JSON part: {part}")
+                logger.warning(f"JSON parse error: {str(e)}")
+                continue
+        calls = []
+        # Only process if we found valid JSON objects
+        if all_actions:
+            calls = self.parse_base_json(all_actions, tools)
+        return StreamingParseResult(normal_text=normal_text, calls=calls)
+
+    def structure_info(self) -> _GetInfoFunc:
+        return lambda name: StructureInfo(
+            begin='<|python_tag|>{"name":"' + name + '", "arguments":',
+            end="}",
+            trigger="<|python_tag|>",
+        )
+
+    def build_ebnf(self, tools: List[Tool]):
+        return EBNFComposer.build_ebnf(
+            tools,
+            function_format="json",
+            tool_call_separator=",",
+        )

sglang/srt/function_call/mistral_detector.py

@@ -0,0 +1,84 @@
+import json
+import re
+from typing import List
+
+from sglang.srt.function_call.base_format_detector import BaseFormatDetector
+from sglang.srt.function_call.core_types import (
+    StreamingParseResult,
+    StructureInfo,
+    _GetInfoFunc,
+)
+from sglang.srt.function_call.ebnf_composer import EBNFComposer
+from sglang.srt.openai_api.protocol import Tool
+
+
+class MistralDetector(BaseFormatDetector):
+    """
+    Detector for Mistral models.
+    Assumes function call format:
+      [TOOL_CALLS] [{"name":"xxx", "arguments":{...}}]
+    """
+
+    def __init__(self):
+        """
+        Initializes the detector with necessary state variables.
+        """
+        super().__init__()
+        self.bot_token = "[TOOL_CALLS] ["
+        self.eot_token = "]"
+        self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
+
+    def has_tool_call(self, text: str) -> bool:
+        """Check if the text contains a Mistral format tool call."""
+        return self.bot_token in text
+
+    def _clean_text(self, text: str) -> str:
+        """
+        clean text to only leave ''[TOOL_CALLS] [{"name": xxx, "arguments": {xxx}}]'
+        for example,
+            text = '[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"location": "Boston, MA", "unit": "fahrenheit"}}]\n\nToday\'s weather in Boston is :{function call result} (in Fahrenheit)\n\nIf you prefer Celsius, please let me know.'
+            return '[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"location": "Boston, MA", "unit": "fahrenheit"}}]'
+        The key pattern is [TOOL_CALLS] [...]
+        """
+        # TODO: check if Mistral supports multiple tool calls, currently assume only support one tool call
+        find_results = re.findall(r"\[TOOL_CALLS\] \[.*?\]", text, re.DOTALL)
+        if len(find_results) > 0:
+            return find_results[0]
+        else:
+            return ""
+
+    def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+        """
+        One-time parsing: Detects and parses tool calls in the provided text.
+
+        :param text: The complete text to parse.
+        :param tools: List of available tools.
+        :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls.
+        """
+        idx = text.find(self.bot_token)
+        normal_text = text[:idx].strip() if idx != -1 else text
+        text = self._clean_text(text)
+        tool_content = text.replace("[TOOL_CALLS]", "").strip()
+        raw_tool_calls = self.tool_call_regex.findall(tool_content)
+        calls = []
+        if len(raw_tool_calls) > 0:
+            raw_tool_call = raw_tool_calls[0]
+            function_call_arr = json.loads(raw_tool_call)
+            for match_result in function_call_arr:
+                calls.extend(self.parse_base_json(match_result, tools))
+        return StreamingParseResult(normal_text=normal_text, calls=calls)
+
+    def structure_info(self) -> _GetInfoFunc:
+        return lambda name: StructureInfo(
+            begin='[TOOL_CALLS] [{"name":"' + name + '", "arguments":',
+            end="}]",
+            trigger="[TOOL_CALLS]",
+        )
+
+    def build_ebnf(self, tools: List[Tool]):
+        return EBNFComposer.build_ebnf(
+            tools,
+            bot_token=self.bot_token,
+            eot_token=self.eot_token,
+            function_format="json",
+        )