xinference 1.9.1__py3-none-any.whl → 1.10.0__py3-none-any.whl

xinference/model/llm/tool_parsers/__init__.py

@@ -0,0 +1,58 @@
+from functools import wraps
+from typing import Any, Callable, Dict, Type
+
+# Global registry for tool parsers, mapping parser names to their classes
+TOOL_PARSERS: Dict[str, Type[Any]] = {}
+
+
+def register_tool_parser(name: str):
+    """
+    Decorator for registering ToolParser classes to the TOOL_PARSERS registry.
+
+    This decorator allows tool parser classes to be automatically registered
+    when they are defined, making them available for dynamic lookup.
+
+    Args:
+        name (str): The name to register the tool parser under. This should
+                   typically match the model family name (e.g., "qwen", "glm4").
+
+    Returns:
+        Callable: The decorator function that registers the class.
+
+    Example:
+        @register_tool_parser("qwen")
+        class QwenToolParser(ToolParser):
+            def parse_tool_calls(self, text: str) -> List[ToolCall]:
+                # Implementation for parsing Qwen model tool calls
+                pass
+
+    Note:
+        The registered class should implement the ToolParser interface
+        and provide methods for parsing tool calls from model outputs.
+    """
+
+    def decorator(cls: Type[Any]) -> Type[Any]:
+        """
+        The actual decorator that performs the registration.
+
+        Args:
+            cls: The tool parser class to register.
+
+        Returns:
+            The same class (unmodified) after registration.
+        """
+        TOOL_PARSERS[name] = cls
+        return cls
+
+    return decorator
+
+
+# Import all tool parser modules to trigger decorator registration
+# This ensures all tool parsers are automatically registered when this module is imported
+from . import (
+    deepseek_r1_tool_parser,
+    deepseek_v3_tool_parser,
+    glm4_tool_parser,
+    llama3_tool_parser,
+    qwen_tool_parser,
+)

xinference/model/llm/tool_parsers/abstract_tool_parser.py

@@ -0,0 +1,33 @@
+class ToolParser:
+    """
+    Abstract ToolParser class that should not be used directly. Provided
+    properties and methods should be used in
+    derived classes.
+    """
+
+    def extract_tool_calls(self, model_output: str):
+        """
+        Static method that should be implemented for extracting tool calls from
+        a complete model-generated string.
+        Used for non-streaming responses where we have the entire model response
+        available before sending to the client.
+        Static because it's stateless.
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls has not been implemented!"
+        )
+
+    def extract_tool_calls_streaming(
+        self, previous_text, current_text: str, delta_text: str
+    ):
+        """
+        Instance method that should be implemented for extracting tool calls
+        from an incomplete response; for use when handling tool calls and
+        streaming. Has to be an instance method because  it requires state -
+        the current tokens/diffs, but also the information about what has
+        previously been parsed and extracted (see constructor)
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls_streaming has not been "
+            "implemented!"
+        )

xinference/model/llm/tool_parsers/deepseek_r1_tool_parser.py

@@ -0,0 +1,128 @@
+import json
+import logging
+import re
+from typing import Any, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("deepseek-r1")
+class DeepseekR1ToolParser(ToolParser):
+    """
+    Tool parser implementation for DeepSeek R1 model.
+
+    This parser handles the specific format used by DeepSeek R1 for tool calls,
+    which includes special Unicode tokens and JSON-formatted function arguments.
+    """
+
+    def __init__(self):
+        """
+        Initialize the DeepSeek R1 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match DeepSeek R1 tool call format
+        self.tool_calls_regex = (
+            r"<\｜tool▁call▁begin｜>function<\｜tool▁sep｜>([^\n]+)\n"
+            r"```json\n(.*?)\n```<\｜tool▁call▁end｜>"
+        )
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find tool call patterns and extracts
+        function names and arguments. Handles JSON parsing errors gracefully
+        and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - arguments (dict or None): Parsed function arguments
+
+        Example:
+            >>> parser = DeepseekR1ToolParser()
+            >>> output = '<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_weather\n```json\n{"location": "上海", "unit": "celsius"}\n```<｜tool▁call▁end｜>'
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_current_weather', {'location': 'Beijing'})]
+        """
+        matches = re.findall(self.tool_calls_regex, model_output, re.DOTALL)
+        if not matches:
+            # No tool calls found, return the original output as content
+            return [(model_output, None, None)]
+
+        # Use set for deduplication of identical tool calls
+        tool_calls = set()
+        results: List[Tuple[Optional[str], Optional[str], Optional[dict]]] = []
+
+        for func_name, raw_json in matches:
+            func_and_args = None
+            try:
+                # Parse JSON arguments
+                func_and_args = json.loads(raw_json)
+                # Create hashable representation for deduplication
+                arguments_hashable = frozenset(func_and_args.items())
+                tool_call_tuple = (
+                    None,  # No content error
+                    func_name,
+                    func_and_args,
+                )
+            except Exception as e:
+                # JSON parsing failed, treat as raw content
+                logger.warning(
+                    f"Failed to parse tool call JSON: {raw_json}, error: {e}"
+                )
+                tool_call_tuple = (raw_json, None, None)
+                arguments_hashable = None
+
+            # Create deduplication key
+            dedup_key = (
+                (func_name, arguments_hashable)
+                if func_and_args is not None
+                else raw_json
+            )
+
+            # Add to results if not already seen
+            if dedup_key not in tool_calls:
+                tool_calls.add(dedup_key)
+                results.append(tool_call_tuple)
+
+        return results
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for DeepSeek R1 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+
+        Note:
+            DeepSeek R1 model does not currently support streaming tool call
+            extraction. Use extract_tool_calls() with complete output instead.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "DeepSeek R1 does not support streaming tool call extraction."
+        )

xinference/model/llm/tool_parsers/deepseek_v3_tool_parser.py

@@ -0,0 +1,145 @@
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("deepseek-v3")
+class DeepseekV3ToolParser(ToolParser):
+    """
+    Tool parser implementation for DeepSeek V3 model.
+
+    This parser handles the specific format used by DeepSeek V3 for tool calls,
+    which uses JSON code blocks wrapped in markdown format.
+
+    """
+
+    def __init__(self):
+        """
+        Initialize the DeepSeek V3 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match JSON code blocks
+        self.tool_calls_regex = r"\s*```json\s*(.*?)\s*```"
+
+    def _parse_json_function_call(
+        self,
+        function_call_str: str,
+    ) -> str:
+        """
+        Parse JSON function call from string.
+
+        Args:
+            function_call_str (str): The function call string to parse.
+
+        Returns:
+            str: Parsed result or original string if no match found.
+
+        """
+        match = self.tool_calls_regex.search(function_call_str)
+        if match:
+            result = match.group(1)
+            return result
+        return function_call_str
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find JSON code blocks containing tool calls
+        and extracts function names and parameters. Handles JSON parsing errors
+        gracefully and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - parameters (dict or None): Function parameters
+
+        Example:
+            >>> parser = DeepseekV3ToolParser()
+            >>> output = '```json\n{"name": "get_weather", "parameters": {"location": "Beijing"}}\n```'
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_weather', {'location': 'Beijing'})]
+        """
+        matches = re.findall(self.tool_calls_regex, model_output, re.DOTALL)
+
+        if not matches:
+            # No tool calls found, return the original output as content
+            return [(model_output, None, None)]
+
+        # Use set for deduplication of identical tool calls
+        tool_calls = set()
+        results: List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]] = (
+            []
+        )
+
+        for raw_json in matches:
+            func_and_args = None
+            try:
+                # Parse JSON to extract function call information
+                func_and_args = json.loads(raw_json)
+                # Convert dictionary to frozenset for deduplication
+                arguments_hashable = frozenset(func_and_args["parameters"])
+                tool_call_tuple = (
+                    None,  # No content error
+                    func_and_args["name"],
+                    func_and_args["parameters"],
+                )
+            except json.JSONDecodeError:
+                tool_call_tuple = (
+                    raw_json,
+                    None,
+                    None,
+                )  # If parsing fails, treat as raw content
+                arguments_hashable = None  # No need for hashing
+
+            # Avoid duplicate entries
+            dedup_key = (
+                (func_and_args["name"], arguments_hashable)
+                if func_and_args is not None
+                else (raw_json)
+            )
+
+            # Add to results if not already seen
+            if dedup_key not in tool_calls:
+                tool_calls.add(dedup_key)
+                results.append(tool_call_tuple)
+
+        return results
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for DeepSeek V3 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "DeepSeek V3 does not support streaming tool call extraction."
+        )

xinference/model/llm/tool_parsers/glm4_tool_parser.py

@@ -0,0 +1,123 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("glm4")
+class Glm4ToolParser(ToolParser):
+    """
+    Tool parser implementation for GLM4 model.
+
+    This parser handles the specific format used by GLM4 for tool calls,
+    which uses JSON code blocks wrapped in markdown format.
+
+    """
+
+    def __init__(self):
+        """
+        Initialize the GLM4 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match JSON code blocks
+        self.tool_calls_regex = r"\s*```json\s*(.*?)\s*```"
+
+    def _parse_json_function_call(
+        self,
+        function_call_str: str,
+    ) -> str:
+        """
+        Parse JSON function call from string.
+
+        Args:
+            function_call_str (str): The function call string to parse.
+
+        Returns:
+            str: Parsed result or original string if no match found.
+
+        """
+        match = self.tool_calls_regex.search(function_call_str)
+        if match:
+            result = match.group(1)
+            return result
+        return function_call_str
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find JSON code blocks containing tool calls
+        and extracts function names and parameters. Handles JSON parsing errors
+        gracefully and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - parameters (dict or None): Function parameters
+
+        Example:
+            >>> parser = Glm4ToolParser()
+            >>> output = {"name": "get_weather", "parameters": {"location": "Beijing"}}
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_weather', {'location': 'Beijing'})]
+        """
+        try:
+            if isinstance(model_output, dict):
+                try:
+                    return [
+                        (
+                            None,
+                            model_output["name"],
+                            json.loads(model_output["arguments"]),
+                        )
+                    ]
+                except Exception:
+                    return [(None, model_output["name"], model_output["arguments"])]
+        except KeyError:
+            logger.error("Can't parse glm output: %s", model_output)
+            return [(str(model_output), None, None)]
+        else:
+            return [(str(model_output), None, None)]
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently has limited support for GLM4 model streaming. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+        """
+        try:
+            if isinstance(current_text, dict):
+                try:
+                    return (
+                        None,
+                        current_text["name"],
+                        json.loads(current_text["arguments"]),
+                    )
+                except Exception:
+                    return (None, current_text["name"], current_text["arguments"])
+        except KeyError:
+            logger.error("Can't parse glm output: %s", current_text)
+            return (str(current_text), None, None)
+        else:
+            return (str(current_text), None, None)

xinference/model/llm/tool_parsers/llama3_tool_parser.py

@@ -0,0 +1,77 @@
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("llama3")
+class Llama3ToolParser(ToolParser):
+    """
+    Tool parser implementation for Llama3 model.
+
+    This parser handles the specific format used by Llama3 for tool calls,
+    which uses Python dictionary format that needs to be evaluated safely.
+
+    """
+
+    def __init__(self):
+        """
+        Initialize the Llama3 tool parser.
+        """
+        super().__init__()
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output using eval() to extract tool call information.
+        This method expects the output to be a valid Python dictionary format.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - parameters (dict or None): Function parameters
+        """
+        try:
+            data = eval(model_output, {}, {})
+            return [(None, data["name"], data["parameters"])]
+        except Exception:
+            return [(model_output, None, None)]
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for Llama3 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+
+        Note:
+            Llama3 model does not currently support streaming tool call
+            extraction. Use extract_tool_calls() with complete output instead.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "Llama3 does not support streaming tool call extraction."
+        )