xinference 1.9.1__py3-none-any.whl → 1.10.1__py3-none-any.whl

xinference/model/llm/llm_family.py

@@ -154,6 +154,7 @@ class LLMFamilyV2(BaseModel, ModelInstanceInfoMixin):
     reasoning_end_tag: Optional[str]
     cache_config: Optional[dict]
     virtualenv: Optional[VirtualEnvSettings]
+    tool_parser: Optional[str]
 
     class Config:
         extra = "allow"

xinference/model/llm/mlx/core.py

@@ -148,6 +148,16 @@ class MLXModel(LLM):
         # to call aynsc method with asyncio.run_coroutine_threadsafe
         self._loop = loop  # type: ignore
 
+    def _cleanup_memory(self):
+        import gc
+
+        import mlx.core as mx
+
+        # mandatory recycling
+        gc.collect()
+        # clear the MLX cache
+        mx.clear_cache()
+
     @property
     def driver_info(self) -> Optional[dict]:
         return self._driver_info
@@ -333,6 +343,7 @@ class MLXModel(LLM):
         self.prepare_parse_reasoning_content(
             reasoning_content, enable_thinking=enable_thinking
         )
+        self.prepare_parse_tool_calls()
 
         kwargs = {}
         kwargs["revision"] = self._model_config.get(
@@ -458,14 +469,18 @@ class MLXModel(LLM):
             repetition_penalty=kwargs.pop("repetition_penalty"),
             repetition_context_size=kwargs.pop("repetition_context_size"),
         )
-        yield from stream_generate(
-            self._model,
-            self._tokenizer,
-            prompt_token_ids,
-            sampler=sampler,
-            logits_processors=logits_processors,
-            **kwargs,
-        )
+        try:
+            yield from stream_generate(
+                self._model,
+                self._tokenizer,
+                prompt_token_ids,
+                sampler=sampler,
+                logits_processors=logits_processors,
+                **kwargs,
+            )
+        finally:
+            # after completing the inference, clear the memory.
+            self._cleanup_memory()
 
     def _prepare_inputs(
         self, prompt: Union[str, Dict[str, Any]], kwargs
@@ -755,7 +770,7 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             assert not isinstance(c, Iterator)
             if tools:
                 return self._post_process_completion(
-                    self.model_family, self.model_uid, c, self.reasoning_parser
+                    self.model_family, self.model_uid, c
                 )
             return self._to_chat_completion(c, self.reasoning_parser)
 
@@ -831,18 +846,32 @@ class MLXVisionModel(MLXModel, ChatModelMixin):
 
         detokenizer.reset()
         tic = time.perf_counter()
-        for n, (token, logprobs) in enumerate(
-            generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
-        ):
-            if n == 0:
-                prompt_time = time.perf_counter() - tic
-                prompt_tps = len(input_ids) / prompt_time
-                tic = time.perf_counter()
-            if token == tokenizer.eos_token_id:
-                break
-            detokenizer.add_token(token)
+        try:
+            for n, (token, logprobs) in enumerate(
+                generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
+            ):
+                if n == 0:
+                    prompt_time = time.perf_counter() - tic
+                    prompt_tps = len(input_ids) / prompt_time
+                    tic = time.perf_counter()
+                if token == tokenizer.eos_token_id:
+                    break
+                detokenizer.add_token(token)
+
+                # Yield the last segment if streaming
+                yield GenerationResponse(
+                    text=detokenizer.last_segment,
+                    token=token,
+                    logprobs=logprobs,
+                    from_draft=False,
+                    prompt_tokens=len(input_ids),
+                    prompt_tps=prompt_tps,
+                    generation_tokens=n + 1,
+                    generation_tps=(n + 1) / (time.perf_counter() - tic),
+                    peak_memory=mx.metal.get_peak_memory() / 1e9,
+                )
 
-            # Yield the last segment if streaming
+            detokenizer.finalize()
             yield GenerationResponse(
                 text=detokenizer.last_segment,
                 token=token,
@@ -854,19 +883,9 @@ class MLXVisionModel(MLXModel, ChatModelMixin):
                 generation_tps=(n + 1) / (time.perf_counter() - tic),
                 peak_memory=mx.metal.get_peak_memory() / 1e9,
             )
-
-        detokenizer.finalize()
-        yield GenerationResponse(
-            text=detokenizer.last_segment,
-            token=token,
-            logprobs=logprobs,
-            from_draft=False,
-            prompt_tokens=len(input_ids),
-            prompt_tps=prompt_tps,
-            generation_tokens=n + 1,
-            generation_tps=(n + 1) / (time.perf_counter() - tic),
-            peak_memory=mx.metal.get_peak_memory() / 1e9,
-        )
+        finally:
+            # after completing the inference, clear the memory
+            self._cleanup_memory()
 
     def _prepare_inputs(
         self, prompt: Union[str, Dict[str, Any]], kwargs

xinference/model/llm/sglang/core.py

@@ -73,6 +73,7 @@ class SGLANGGenerateConfig(TypedDict, total=False):
     stream: bool
     stream_options: Optional[Union[dict, None]]
     json_schema: Optional[dict]
+    response_format: dict
 
 
 try:
@@ -175,6 +176,7 @@ class SGLANGModel(LLM):
         self.prepare_parse_reasoning_content(
             reasoning_content, enable_thinking=enable_thinking
         )
+        self.prepare_parse_tool_calls()
 
         # Fix: GH#2169
         if sgl.__version__ >= "0.2.14":
@@ -316,13 +318,16 @@ class SGLANGModel(LLM):
         stream_options = generate_config.get("stream_options")
         generate_config.setdefault("stream_options", stream_options)
         generate_config.setdefault("ignore_eos", False)
-        json_schema = (
-            generate_config.pop("response_format", {})  # type: ignore
-            .pop("json_schema", {})
-            .pop("schema", {})
-        )
-        if json_schema:
-            generate_config.setdefault("json_schema", json.dumps(json_schema))  # type: ignore
+        response_format = generate_config.pop("response_format", None)
+        if response_format:
+            json_schema_config = response_format.pop("json_schema", None)
+            json_schema = None
+            if "schema_" in json_schema_config:
+                json_schema = json_schema_config.pop("schema_")
+            elif "schema" in json_schema_config:
+                json_schema = json_schema_config.pop("schema")
+            if json_schema:
+                generate_config.setdefault("json_schema", json.dumps(json_schema))  # type: ignore
 
         return generate_config
 
@@ -355,22 +360,31 @@ class SGLANGModel(LLM):
 
     @staticmethod
     def _convert_state_to_completion_chunk(
-        request_id: str, model: str, output_text: str
+        request_id: str, model: str, output_text: str, meta_info: Dict
     ) -> CompletionChunk:
+        finish_reason = meta_info.get("finish_reason", None)
+        if isinstance(finish_reason, dict) and "type" in finish_reason:
+            finish_reason = finish_reason["type"]
         choices: List[CompletionChoice] = [
             CompletionChoice(
                 text=output_text,
                 index=0,
                 logprobs=None,
-                finish_reason=None,
+                finish_reason=finish_reason,
             )
         ]
+        usage = CompletionUsage(
+            prompt_tokens=meta_info["prompt_tokens"],
+            completion_tokens=meta_info["completion_tokens"],
+            total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"],
+        )
         chunk = CompletionChunk(
             id=request_id,
             object="text_completion",
             created=int(time.time()),
             model=model,
             choices=choices,
+            usage=usage,
         )
         return chunk
 
@@ -378,12 +392,15 @@ class SGLANGModel(LLM):
     def _convert_state_to_completion(
         request_id: str, model: str, output_text: str, meta_info: Dict
     ) -> Completion:
+        finish_reason = meta_info.get("finish_reason", None)
+        if isinstance(finish_reason, dict) and "type" in finish_reason:
+            finish_reason = finish_reason["type"]
         choices = [
             CompletionChoice(
                 text=output_text,
                 index=0,
                 logprobs=None,
-                finish_reason=None,
+                finish_reason=finish_reason,
             )
         ]
 
@@ -512,7 +529,10 @@ class SGLANGModel(LLM):
                     prompt, image_data, **sanitized_generate_config
                 ):
                     chunk = self._convert_state_to_completion_chunk(
-                        request_id, self.model_uid, output_text=out
+                        request_id,
+                        self.model_uid,
+                        output_text=out,
+                        meta_info=meta_info,
                     )
                     complete_response += out
                     finish_reason = meta_info["finish_reason"]
@@ -646,49 +666,6 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
     def is_tool_call_chunk_end(chunk):
         return chunk["choices"][0]["text"].endswith(QWEN_TOOL_CALL_SYMBOLS[1])
 
-    async def _async_to_tool_completion_chunks(
-        self,
-        chunks: AsyncGenerator[CompletionChunk, None],
-    ) -> AsyncGenerator[ChatCompletionChunk, None]:
-        i = 0
-        previous_texts = [""]
-        tool_call = False
-        tool_call_texts = [""]
-        if self.reasoning_parser:
-            chunks = self.reasoning_parser.prepare_reasoning_content_streaming(chunks)
-        async for chunk in chunks:
-            if i == 0:
-                for first_chunk in self._get_first_chat_completion_chunk(
-                    chunk, self.reasoning_parser
-                ):
-                    yield first_chunk
-            # usage
-            choices = chunk.get("choices")
-            if not choices:
-                yield self._get_final_chat_completion_chunk(chunk)
-            else:
-                if self.is_tool_call_chunk_start(chunk):
-                    tool_call = True
-                if tool_call:
-                    tool_call_text = tool_call_texts[-1]
-                    tool_call_text += chunk["choices"][0]["text"]
-                    tool_call_texts.append(tool_call_text)
-                    if self.is_tool_call_chunk_end(chunk):
-                        yield self._post_process_completion_chunk(
-                            self.model_family,
-                            self.model_uid,
-                            chunk,
-                            reasoning_parser=self.reasoning_parser,
-                            tool_call_text=tool_call_text,
-                        )
-                        tool_call = False
-                        tool_call_texts = [""]
-                else:
-                    yield self._to_chat_completion_chunk(
-                        chunk, self.reasoning_parser, previous_texts
-                    )
-            i += 1
-
     async def async_chat(
         self,
         messages: List[Dict],
@@ -731,7 +708,7 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
             assert not isinstance(c, AsyncGenerator)
             if tools:
                 return self._post_process_completion(
-                    self.model_family, self.model_uid, c, self.reasoning_parser
+                    self.model_family, self.model_uid, c
                 )
             return self._to_chat_completion(c, self.reasoning_parser)
 

xinference/model/llm/tool_parsers/__init__.py

@@ -0,0 +1,58 @@
+from functools import wraps
+from typing import Any, Callable, Dict, Type
+
+# Global registry for tool parsers, mapping parser names to their classes
+TOOL_PARSERS: Dict[str, Type[Any]] = {}
+
+
+def register_tool_parser(name: str):
+    """
+    Decorator for registering ToolParser classes to the TOOL_PARSERS registry.
+
+    This decorator allows tool parser classes to be automatically registered
+    when they are defined, making them available for dynamic lookup.
+
+    Args:
+        name (str): The name to register the tool parser under. This should
+                   typically match the model family name (e.g., "qwen", "glm4").
+
+    Returns:
+        Callable: The decorator function that registers the class.
+
+    Example:
+        @register_tool_parser("qwen")
+        class QwenToolParser(ToolParser):
+            def parse_tool_calls(self, text: str) -> List[ToolCall]:
+                # Implementation for parsing Qwen model tool calls
+                pass
+
+    Note:
+        The registered class should implement the ToolParser interface
+        and provide methods for parsing tool calls from model outputs.
+    """
+
+    def decorator(cls: Type[Any]) -> Type[Any]:
+        """
+        The actual decorator that performs the registration.
+
+        Args:
+            cls: The tool parser class to register.
+
+        Returns:
+            The same class (unmodified) after registration.
+        """
+        TOOL_PARSERS[name] = cls
+        return cls
+
+    return decorator
+
+
+# Import all tool parser modules to trigger decorator registration
+# This ensures all tool parsers are automatically registered when this module is imported
+from . import (
+    deepseek_r1_tool_parser,
+    deepseek_v3_tool_parser,
+    glm4_tool_parser,
+    llama3_tool_parser,
+    qwen_tool_parser,
+)

xinference/model/llm/tool_parsers/abstract_tool_parser.py

@@ -0,0 +1,33 @@
+class ToolParser:
+    """
+    Abstract ToolParser class that should not be used directly. Provided
+    properties and methods should be used in
+    derived classes.
+    """
+
+    def extract_tool_calls(self, model_output: str):
+        """
+        Static method that should be implemented for extracting tool calls from
+        a complete model-generated string.
+        Used for non-streaming responses where we have the entire model response
+        available before sending to the client.
+        Static because it's stateless.
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls has not been implemented!"
+        )
+
+    def extract_tool_calls_streaming(
+        self, previous_text, current_text: str, delta_text: str
+    ):
+        """
+        Instance method that should be implemented for extracting tool calls
+        from an incomplete response; for use when handling tool calls and
+        streaming. Has to be an instance method because  it requires state -
+        the current tokens/diffs, but also the information about what has
+        previously been parsed and extracted (see constructor)
+        """
+        raise NotImplementedError(
+            "AbstractToolParser.extract_tool_calls_streaming has not been "
+            "implemented!"
+        )

xinference/model/llm/tool_parsers/deepseek_r1_tool_parser.py

@@ -0,0 +1,190 @@
+import json
+import logging
+import re
+from typing import Any, List, Optional, Tuple
+
+from . import register_tool_parser
+from .abstract_tool_parser import ToolParser
+
+logger = logging.getLogger(__name__)
+
+
+@register_tool_parser("deepseek-r1")
+class DeepseekR1ToolParser(ToolParser):
+    """
+    Tool parser implementation for DeepSeek R1 model.
+
+    This parser handles the specific format used by DeepSeek R1 for tool calls,
+    which includes special Unicode tokens and JSON-formatted function arguments.
+    """
+
+    def __init__(self):
+        """
+        Initialize the DeepSeek R1 tool parser.
+        """
+        super().__init__()
+
+        # Sentinel tokens for streaming mode
+        self.think_start_token: str = "<think>"
+        self.think_end_token: str = "</think>"
+        self.tool_call_start_token: str = "<｜tool▁call▁begin｜>"
+        self.tool_call_end_token: str = "<｜tool▁call▁end｜>"
+
+        # Regex pattern to match DeepSeek R1 tool call format
+        self.tool_calls_regex = (
+            r"<\｜tool▁call▁begin｜>function<\｜tool▁sep｜>([^\n]+)\n"
+            r"```json\n(.*?)\n```<\｜tool▁call▁end｜>"
+        )
+
+        # Regex pattern to match the entire tool-calls wrapper block.
+        # We intentionally do NOT match <think> blocks here so that the
+        # "text before" chunk will include both the think block and any
+        # narrative text up to the tool calls wrapper, yielding exactly two
+        # blocks when there is a single tool calls section:
+        # [before_text_including_think, tool_calls_wrapper_block]
+        self.content_regex = r"(<\｜tool▁calls▁begin｜>.*?<\｜tool▁calls▁end｜>)"
+
+    def extract_tool_calls(
+        self, model_output: str
+    ) -> List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+        """
+        Extract tool calls from complete model output.
+
+        Parses the model output to find tool call patterns and extracts
+        function names and arguments. Handles JSON parsing errors gracefully
+        and deduplicates identical tool calls.
+
+        Args:
+            model_output (str): The complete output string from the model.
+
+        Returns:
+            List[Tuple[Optional[str], Optional[str], Optional[dict]]]:
+            A list of tuples where each tuple contains:
+            - content (str or None): Raw content if parsing failed, None if successful
+            - function_name (str or None): Name of the function to call
+            - arguments (dict or None): Parsed function arguments
+
+        Example:
+            >>> parser = DeepseekR1ToolParser()
+            >>> output = '<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_weather\n```json\n{"location": "上海", "unit": "celsius"}\n```<｜tool▁call▁end｜>'
+            >>> result = parser.extract_tool_calls(output)
+            >>> print(result)
+            [(None, 'get_current_weather', {'location': 'Beijing'})]
+        """
+        # If no tool call tokens, return original output as content
+        if self.tool_call_start_token not in model_output:
+            return [(model_output, None, None)]
+
+        # Get all content blocks (text, thinking blocks, tool calls)
+        function_calls = self._get_function_calls(model_output)
+
+        # Use set for deduplication of identical tool calls
+        tool_calls = set()
+        results: List[Tuple[Optional[str], Optional[str], Optional[dict]]] = []
+
+        for content_block in function_calls:
+            # Check if this block is a tool call
+            if (
+                self.tool_call_start_token in content_block
+                and self.tool_call_end_token in content_block
+            ):
+                # Extract function name and arguments from tool call block
+                matches = re.findall(self.tool_calls_regex, content_block, re.DOTALL)
+                if not matches:
+                    # Malformed tool call, treat as regular content
+                    results.append((content_block, None, None))
+                    continue
+
+                func_name, raw_json = matches[0]  # Take the first match
+
+                func_and_args = None
+                try:
+                    # Parse JSON arguments
+                    func_and_args = json.loads(raw_json)
+                    # Create hashable representation for deduplication
+                    arguments_hashable = frozenset(func_and_args.items())
+                    tool_call_tuple = (
+                        None,  # No content error
+                        func_name,
+                        func_and_args,
+                    )
+                except Exception as e:
+                    # JSON parsing failed, treat as raw content
+                    logger.warning(
+                        f"Failed to parse tool call JSON: {raw_json}, error: {e}"
+                    )
+                    tool_call_tuple = (raw_json, None, None)
+                    arguments_hashable = None
+
+                # Create deduplication key
+                dedup_key = (
+                    (func_name, arguments_hashable)
+                    if func_and_args is not None
+                    else raw_json
+                )
+
+                # Add to results if not already seen
+                if dedup_key not in tool_calls:
+                    tool_calls.add(dedup_key)
+                    results.append(tool_call_tuple)
+            else:
+                # This is regular content (text or thinking block), add as-is
+                if content_block.strip():  # Only add non-empty content
+                    results.append((content_block, None, None))
+
+        return results
+
+    def _get_function_calls(self, model_output: str) -> List[str]:
+        """
+        Extract all function calls and content blocks from model output.
+
+        Parses the model output to separate thinking blocks, tool calls,
+        and regular content into individual components.
+
+        Args:
+            model_output (str): The complete model output to parse.
+
+        Returns:
+            List[str]: List of content blocks (text, thinking blocks, tool calls).
+        """
+        functions_calls = []
+        last_end = 0
+        for m in re.finditer(self.content_regex, model_output, re.DOTALL):
+            # Add any text before the current match
+            if m.start() > last_end:
+                functions_calls.append(model_output[last_end : m.start()])
+            # Add the matched content (think or tool_call block)
+            functions_calls.append(m.group(0))
+            last_end = m.end()
+        # Add any remaining text after the last match
+        if last_end < len(model_output):
+            functions_calls.append(model_output[last_end:])
+        return functions_calls
+
+    def extract_tool_calls_streaming(
+        self, previous_text: List[str], current_text: str, delta_text: str
+    ) -> Optional[Any]:
+        """
+        Extract tool calls from streaming output.
+
+        Currently not supported for DeepSeek R1 model. This method raises
+        a ValueError indicating that streaming tool call extraction is only
+        available for specific model/backend combinations.
+
+        Args:
+            previous_text (List[str]): Previous text chunks from the stream.
+            current_text (str): Current accumulated text.
+            delta_text (str): New text delta in this chunk.
+
+        Raises:
+            ValueError: Always raised as streaming is not supported.
+
+        Note:
+            DeepSeek R1 model does not currently support streaming tool call
+            extraction. Use extract_tool_calls() with complete output instead.
+        """
+        raise NotImplementedError(
+            "Streaming support for tool calls is available only when using "
+            "Qwen models with vLLM backend or GLM4-chat models without vLLM backend. "
+            "DeepSeek R1 does not support streaming tool call extraction."
+        )