xinference 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

xinference/model/llm/llm_family.py

@@ -154,6 +154,7 @@ class LLMFamilyV2(BaseModel, ModelInstanceInfoMixin):
     reasoning_end_tag: Optional[str]
     cache_config: Optional[dict]
     virtualenv: Optional[VirtualEnvSettings]
+    tool_parser: Optional[str]
 
     class Config:
         extra = "allow"

xinference/model/llm/mlx/core.py

@@ -148,6 +148,16 @@ class MLXModel(LLM):
         # to call aynsc method with asyncio.run_coroutine_threadsafe
         self._loop = loop  # type: ignore
 
+    def _cleanup_memory(self):
+        import gc
+
+        import mlx.core as mx
+
+        # mandatory recycling
+        gc.collect()
+        # clear the MLX cache
+        mx.clear_cache()
+
     @property
     def driver_info(self) -> Optional[dict]:
         return self._driver_info
@@ -333,6 +343,7 @@ class MLXModel(LLM):
         self.prepare_parse_reasoning_content(
             reasoning_content, enable_thinking=enable_thinking
         )
+        self.prepare_parse_tool_calls()
 
         kwargs = {}
         kwargs["revision"] = self._model_config.get(
@@ -458,14 +469,18 @@ class MLXModel(LLM):
             repetition_penalty=kwargs.pop("repetition_penalty"),
             repetition_context_size=kwargs.pop("repetition_context_size"),
         )
-        yield from stream_generate(
-            self._model,
-            self._tokenizer,
-            prompt_token_ids,
-            sampler=sampler,
-            logits_processors=logits_processors,
-            **kwargs,
-        )
+        try:
+            yield from stream_generate(
+                self._model,
+                self._tokenizer,
+                prompt_token_ids,
+                sampler=sampler,
+                logits_processors=logits_processors,
+                **kwargs,
+            )
+        finally:
+            # after completing the inference, clear the memory.
+            self._cleanup_memory()
 
     def _prepare_inputs(
         self, prompt: Union[str, Dict[str, Any]], kwargs
@@ -755,7 +770,7 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             assert not isinstance(c, Iterator)
             if tools:
                 return self._post_process_completion(
-                    self.model_family, self.model_uid, c, self.reasoning_parser
+                    self.model_family, self.model_uid, c
                 )
             return self._to_chat_completion(c, self.reasoning_parser)
 
@@ -831,18 +846,32 @@ class MLXVisionModel(MLXModel, ChatModelMixin):
 
         detokenizer.reset()
         tic = time.perf_counter()
-        for n, (token, logprobs) in enumerate(
-            generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
-        ):
-            if n == 0:
-                prompt_time = time.perf_counter() - tic
-                prompt_tps = len(input_ids) / prompt_time
-                tic = time.perf_counter()
-            if token == tokenizer.eos_token_id:
-                break
-            detokenizer.add_token(token)
+        try:
+            for n, (token, logprobs) in enumerate(
+                generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
+            ):
+                if n == 0:
+                    prompt_time = time.perf_counter() - tic
+                    prompt_tps = len(input_ids) / prompt_time
+                    tic = time.perf_counter()
+                if token == tokenizer.eos_token_id:
+                    break
+                detokenizer.add_token(token)
+
+                # Yield the last segment if streaming
+                yield GenerationResponse(
+                    text=detokenizer.last_segment,
+                    token=token,
+                    logprobs=logprobs,
+                    from_draft=False,
+                    prompt_tokens=len(input_ids),
+                    prompt_tps=prompt_tps,
+                    generation_tokens=n + 1,
+                    generation_tps=(n + 1) / (time.perf_counter() - tic),
+                    peak_memory=mx.metal.get_peak_memory() / 1e9,
+                )
 
-            # Yield the last segment if streaming
+            detokenizer.finalize()
             yield GenerationResponse(
                 text=detokenizer.last_segment,
                 token=token,
@@ -854,19 +883,9 @@ class MLXVisionModel(MLXModel, ChatModelMixin):
                 generation_tps=(n + 1) / (time.perf_counter() - tic),
                 peak_memory=mx.metal.get_peak_memory() / 1e9,
             )
-
-        detokenizer.finalize()
-        yield GenerationResponse(
-            text=detokenizer.last_segment,
-            token=token,
-            logprobs=logprobs,
-            from_draft=False,
-            prompt_tokens=len(input_ids),
-            prompt_tps=prompt_tps,
-            generation_tokens=n + 1,
-            generation_tps=(n + 1) / (time.perf_counter() - tic),
-            peak_memory=mx.metal.get_peak_memory() / 1e9,
-        )
+        finally:
+            # after completing the inference, clear the memory
+            self._cleanup_memory()
 
     def _prepare_inputs(
         self, prompt: Union[str, Dict[str, Any]], kwargs

xinference/model/llm/sglang/core.py

@@ -175,6 +175,7 @@ class SGLANGModel(LLM):
         self.prepare_parse_reasoning_content(
             reasoning_content, enable_thinking=enable_thinking
         )
+        self.prepare_parse_tool_calls()
 
         # Fix: GH#2169
         if sgl.__version__ >= "0.2.14":
@@ -646,49 +647,6 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
     def is_tool_call_chunk_end(chunk):
         return chunk["choices"][0]["text"].endswith(QWEN_TOOL_CALL_SYMBOLS[1])
 
-    async def _async_to_tool_completion_chunks(
-        self,
-        chunks: AsyncGenerator[CompletionChunk, None],
-    ) -> AsyncGenerator[ChatCompletionChunk, None]:
-        i = 0
-        previous_texts = [""]
-        tool_call = False
-        tool_call_texts = [""]
-        if self.reasoning_parser:
-            chunks = self.reasoning_parser.prepare_reasoning_content_streaming(chunks)
-        async for chunk in chunks:
-            if i == 0:
-                for first_chunk in self._get_first_chat_completion_chunk(
-                    chunk, self.reasoning_parser
-                ):
-                    yield first_chunk
-            # usage
-            choices = chunk.get("choices")
-            if not choices:
-                yield self._get_final_chat_completion_chunk(chunk)
-            else:
-                if self.is_tool_call_chunk_start(chunk):
-                    tool_call = True
-                if tool_call:
-                    tool_call_text = tool_call_texts[-1]
-                    tool_call_text += chunk["choices"][0]["text"]
-                    tool_call_texts.append(tool_call_text)
-                    if self.is_tool_call_chunk_end(chunk):
-                        yield self._post_process_completion_chunk(
-                            self.model_family,
-                            self.model_uid,
-                            chunk,
-                            reasoning_parser=self.reasoning_parser,
-                            tool_call_text=tool_call_text,
-                        )
-                        tool_call = False
-                        tool_call_texts = [""]
-                else:
-                    yield self._to_chat_completion_chunk(
-                        chunk, self.reasoning_parser, previous_texts
-                    )
-            i += 1
-
     async def async_chat(
         self,
         messages: List[Dict],
@@ -731,7 +689,7 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
             assert not isinstance(c, AsyncGenerator)
             if tools:
                 return self._post_process_completion(
-                    self.model_family, self.model_uid, c, self.reasoning_parser
+                    self.model_family, self.model_uid, c
                 )
             return self._to_chat_completion(c, self.reasoning_parser)
 

xinference/model/llm/tool_parsers/__init__.py

@@ -0,0 +1,58 @@
+from functools import wraps
+from typing import Any, Callable, Dict, Type
+
+# Global registry for tool parsers, mapping parser names to their classes
+TOOL_PARSERS: Dict[str, Type[Any]] = {}
+
+
+def register_tool_parser(name: str):
+    """
+    Decorator for registering ToolParser classes to the TOOL_PARSERS registry.
+
+    This decorator allows tool parser classes to be automatically registered
+    when they are defined, making them available for dynamic lookup.
+
+    Args:
+        name (str): The name to register the tool parser under. This should
+                   typically match the model family name (e.g., "qwen", "glm4").
+
+    Returns:
+        Callable: The decorator function that registers the class.
+
+    Example:
+        @register_tool_parser("qwen")
+        class QwenToolParser(ToolParser):
+            def parse_tool_calls(self, text: str) -> List[ToolCall]:
+                # Implementation for parsing Qwen model tool calls
+                pass
+
+    Note:
+        The registered class should implement the ToolParser interface
+        and provide methods for parsing tool calls from model outputs.
+    """
+
+    def decorator(cls: Type[Any]) -> Type[Any]:
+        """
+        The actual decorator that performs the registration.
+
+        Args:
+            cls: The tool parser class to register.
+
+        Returns:
+            The same class (unmodified) after registration.
+        """
+        TOOL_PARSERS[name] = cls
+        return cls
+
+    return decorator
+
+
+# Import all tool parser modules to trigger decorator registration
+# This ensures all tool parsers are automatically registered when this module is imported
+from . import (
+    deepseek_r1_tool_parser,
+    deepseek_v3_tool_parser,
+    glm4_tool_parser,
+    llama3_tool_parser,
+    qwen_tool_parser,
+)

xinference/model/llm/tool_parsers/abstract_tool_parser.py

@@ -0,0 +1,33 @@
+class ToolParser:
+    """
+    Abstract ToolParser class that should not be used directly. Provided
+    properties and methods should be used in
+    derived classes.
+    """
+
+    def extract_tool_calls(self, model_output: str):
+        """
+        Static method that should be implemented for extracting tool calls from
+        a complete model-generated string.
+        Used for non-streaming responses where we have the entire model response
+        available before sending to the client.
+        Static because it's stateless.
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls has not been implemented!"
+        )
+
+    def extract_tool_calls_streaming(
+        self, previous_text, current_text: str, delta_text: str
+    ):
+        """
+        Instance method that should be implemented for extracting tool calls
+        from an incomplete response; for use when handling tool calls and
+        streaming. Has to be an instance method because  it requires state -
+        the current tokens/diffs, but also the information about what has
+        previously been parsed and extracted (see constructor)
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls_streaming has not been "
+            "implemented!"
+        )

xinference/model/llm/tool_parsers/deepseek_r1_tool_parser.py

@@ -0,0 +1,128 @@
+import json
+import logging
+import re
+from typing import Any, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("deepseek-r1")
+class DeepseekR1ToolParser(ToolParser):
+    """
+    Tool parser implementation for DeepSeek R1 model.
+
+    This parser handles the specific format used by DeepSeek R1 for tool calls,
+    which includes special Unicode tokens and JSON-formatted function arguments.
+    """
+
+    def __init__(self):
+        """
+        Initialize the DeepSeek R1 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match DeepSeek R1 tool call format
+        self.tool_calls_regex = (
+            r"<\｜tool▁call▁begin｜>function<\｜tool▁sep｜>([^\n]+)\n"
+            r"```json\n(.*?)\n```<\｜tool▁call▁end｜>"
+        )
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find tool call patterns and extracts
+        function names and arguments. Handles JSON parsing errors gracefully
+        and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - arguments (dict or None): Parsed function arguments
+
+        Example:
+            >>> parser = DeepseekR1ToolParser()
+            >>> output = '<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_weather\n```json\n{"location": "上海", "unit": "celsius"}\n```<｜tool▁call▁end｜>'
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_current_weather', {'location': 'Beijing'})]
+        """
+        matches = re.findall(self.tool_calls_regex, model_output, re.DOTALL)
+        if not matches:
+            # No tool calls found, return the original output as content
+            return [(model_output, None, None)]
+
+        # Use set for deduplication of identical tool calls
+        tool_calls = set()
+        results: List[Tuple[Optional[str], Optional[str], Optional[dict]]] = []
+
+        for func_name, raw_json in matches:
+            func_and_args = None
+            try:
+                # Parse JSON arguments
+                func_and_args = json.loads(raw_json)
+                # Create hashable representation for deduplication
+                arguments_hashable = frozenset(func_and_args.items())
+                tool_call_tuple = (
+                    None,  # No content error
+                    func_name,
+                    func_and_args,
+                )
+            except Exception as e:
+                # JSON parsing failed, treat as raw content
+                logger.warning(
+                    f"Failed to parse tool call JSON: {raw_json}, error: {e}"
+                )
+                tool_call_tuple = (raw_json, None, None)
+                arguments_hashable = None
+
+            # Create deduplication key
+            dedup_key = (
+                (func_name, arguments_hashable)
+                if func_and_args is not None
+                else raw_json
+            )
+
+            # Add to results if not already seen
+            if dedup_key not in tool_calls:
+                tool_calls.add(dedup_key)
+                results.append(tool_call_tuple)
+
+        return results
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for DeepSeek R1 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+
+        Note:
+            DeepSeek R1 model does not currently support streaming tool call
+            extraction. Use extract_tool_calls() with complete output instead.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "DeepSeek R1 does not support streaming tool call extraction."
+        )

xinference/model/llm/tool_parsers/deepseek_v3_tool_parser.py

@@ -0,0 +1,145 @@
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("deepseek-v3")
+class DeepseekV3ToolParser(ToolParser):
+    """
+    Tool parser implementation for DeepSeek V3 model.
+
+    This parser handles the specific format used by DeepSeek V3 for tool calls,
+    which uses JSON code blocks wrapped in markdown format.
+
+    """
+
+    def __init__(self):
+        """
+        Initialize the DeepSeek V3 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match JSON code blocks
+        self.tool_calls_regex = r"\s*```json\s*(.*?)\s*```"
+
+    def _parse_json_function_call(
+        self,
+        function_call_str: str,
+    ) -> str:
+        """
+        Parse JSON function call from string.
+
+        Args:
+            function_call_str (str): The function call string to parse.
+
+        Returns:
+            str: Parsed result or original string if no match found.
+
+        """
+        match = self.tool_calls_regex.search(function_call_str)
+        if match:
+            result = match.group(1)
+            return result
+        return function_call_str
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find JSON code blocks containing tool calls
+        and extracts function names and parameters. Handles JSON parsing errors
+        gracefully and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - parameters (dict or None): Function parameters
+
+        Example:
+            >>> parser = DeepseekV3ToolParser()
+            >>> output = '```json\n{"name": "get_weather", "parameters": {"location": "Beijing"}}\n```'
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_weather', {'location': 'Beijing'})]
+        """
+        matches = re.findall(self.tool_calls_regex, model_output, re.DOTALL)
+
+        if not matches:
+            # No tool calls found, return the original output as content
+            return [(model_output, None, None)]
+
+        # Use set for deduplication of identical tool calls
+        tool_calls = set()
+        results: List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]] = (
+            []
+        )
+
+        for raw_json in matches:
+            func_and_args = None
+            try:
+                # Parse JSON to extract function call information
+                func_and_args = json.loads(raw_json)
+                # Convert dictionary to frozenset for deduplication
+                arguments_hashable = frozenset(func_and_args["parameters"])
+                tool_call_tuple = (
+                    None,  # No content error
+                    func_and_args["name"],
+                    func_and_args["parameters"],
+                )
+            except json.JSONDecodeError:
+                tool_call_tuple = (
+                    raw_json,
+                    None,
+                    None,
+                )  # If parsing fails, treat as raw content
+                arguments_hashable = None  # No need for hashing
+
+            # Avoid duplicate entries
+            dedup_key = (
+                (func_and_args["name"], arguments_hashable)
+                if func_and_args is not None
+                else (raw_json)
+            )
+
+            # Add to results if not already seen
+            if dedup_key not in tool_calls:
+                tool_calls.add(dedup_key)
+                results.append(tool_call_tuple)
+
+        return results
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for DeepSeek V3 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "DeepSeek V3 does not support streaming tool call extraction."
+        )

xinference/model/llm/tool_parsers/glm4_tool_parser.py

@@ -0,0 +1,123 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("glm4")
+class Glm4ToolParser(ToolParser):
+    """
+    Tool parser implementation for GLM4 model.
+
+    This parser handles the specific format used by GLM4 for tool calls,
+    which uses JSON code blocks wrapped in markdown format.
+
+    """
+
+    def __init__(self):
+        """
+        Initialize the GLM4 tool parser.
+        """
+        super().__init__()
+        # Regex pattern to match JSON code blocks
+        self.tool_calls_regex = r"\s*```json\s*(.*?)\s*```"
+
+    def _parse_json_function_call(
+        self,
+        function_call_str: str,
+    ) -> str:
+        """
+        Parse JSON function call from string.
+
+        Args:
+            function_call_str (str): The function call string to parse.
+
+        Returns:
+            str: Parsed result or original string if no match found.
+
+        """
+        match = self.tool_calls_regex.search(function_call_str)
+        if match:
+            result = match.group(1)
+            return result
+        return function_call_str
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find JSON code blocks containing tool calls
+        and extracts function names and parameters. Handles JSON parsing errors
+        gracefully and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - parameters (dict or None): Function parameters
+
+        Example:
+            >>> parser = Glm4ToolParser()
+            >>> output = {"name": "get_weather", "parameters": {"location": "Beijing"}}
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_weather', {'location': 'Beijing'})]
+        """
+        try:
+            if isinstance(model_output, dict):
+                try:
+                    return [
+                        (
+                            None,
+                            model_output["name"],
+                            json.loads(model_output["arguments"]),
+                        )
+                    ]
+                except Exception:
+                    return [(None, model_output["name"], model_output["arguments"])]
+        except KeyError:
+            logger.error("Can't parse glm output: %s", model_output)
+            return [(str(model_output), None, None)]
+        else:
+            return [(str(model_output), None, None)]
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently has limited support for GLM4 model streaming. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+        """
+        try:
+            if isinstance(current_text, dict):
+                try:
+                    return (
+                        None,
+                        current_text["name"],
+                        json.loads(current_text["arguments"]),
+                    )
+                except Exception:
+                    return (None, current_text["name"], current_text["arguments"])
+        except KeyError:
+            logger.error("Can't parse glm output: %s", current_text)
+            return (str(current_text), None, None)
+        else:
+            return (str(current_text), None, None)