xinference 0.15.2__py3-none-any.whl → 0.15.4__py3-none-any.whl

xinference/model/llm/transformers/yi_vl.py

@@ -29,6 +29,7 @@ from ..utils import (
     parse_messages,
 )
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -99,6 +100,7 @@ class YiVLChatModel(PytorchChatModel):
                 raise RuntimeError("Only one image per message is supported by Yi VL.")
         return content
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/utils.py

@@ -29,6 +29,7 @@ from ...types import (
     ChatCompletion,
     ChatCompletionChoice,
     ChatCompletionChunk,
+    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -50,6 +51,7 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen1.5-moe-chat",
     "qwen2-instruct",
     "qwen2-moe-instruct",
+    "qwen2.5-instruct",
 ]
 
 GLM4_TOOL_CALL_FAMILY = [
@@ -57,6 +59,10 @@ GLM4_TOOL_CALL_FAMILY = [
     "glm4-chat-1m",
 ]
 
+LLAMA3_TOOL_CALL_FAMILY = [
+    "llama-3.1-instruct",
+]
+
 QWEN_TOOL_CALL_SYMBOLS = ["<tool_call>", "</tool_call>"]
 
 
@@ -113,7 +119,7 @@ class ChatModelMixin:
             return self._build_from_raw_template(messages, chat_template, **kwargs)
 
     @staticmethod
-    def get_specific_prompt(model_family: str, messages: List[Dict]):
+    def get_specific_prompt(model_family: str, messages: List[ChatCompletionMessage]):
         """
         Inspired by FastChat. Format chat history into a prompt according to the prompty style of
         different models.
@@ -129,7 +135,7 @@ class ChatModelMixin:
             ret = (
                 "<s>"
                 if system_prompt == ""
-                else "<s><|im_start|>system\n"
+                else "<s><|im_start|>system\n"  # type: ignore
                 + system_prompt
                 + intra_message_sep
                 + "\n"
@@ -159,14 +165,25 @@ class ChatModelMixin:
                         for image_url in image_urls:
                             fut = executor.submit(_decode_image, image_url)
                             image_futures.append(fut)
-                    images = [fut.result() for fut in image_futures]
+                    images.extend([fut.result() for fut in image_futures])
                     if len(image_futures) == 0:
                         ret += role + "\n" + text + intra_message_sep + "\n"
                     else:
+                        placeholders = "\n".join(
+                            f"Image-{i+1}: <image>\n"
+                            for i in range(
+                                len(images) - len(image_futures), len(images)
+                            )
+                        )
                         ret += (
-                            role + "\n" + f"<image>\n{text}" + intra_message_sep + "\n"
+                            role
+                            + "\n"
+                            + f"{placeholders}\n{text}"
+                            + intra_message_sep
+                            + "\n"
                         )
-
+            if len(images) == 1:
+                ret = ret.replace("Image-1: <image>\n", "<image>\n")
             return ret, images
         else:
             raise ValueError(f"Invalid model family: {model_family}")
@@ -322,8 +339,9 @@ class ChatModelMixin:
         for content in contents:
             content = content.strip()
             if content:
-                if content.startswith(QWEN_TOOL_CALL_SYMBOLS[0]):
-                    content = content[len(QWEN_TOOL_CALL_SYMBOLS[0]) :]
+                pos = content.find(QWEN_TOOL_CALL_SYMBOLS[0])
+                if pos != -1:
+                    content = content[pos + len(QWEN_TOOL_CALL_SYMBOLS[0]) :]
                 content = content.strip()
                 try:
                     res = json.loads(content)
@@ -342,6 +360,15 @@ class ChatModelMixin:
         text = c["choices"][0]["text"]
         return cls._handle_qwen_tool_result(text)
 
+    @classmethod
+    def _eval_llama3_chat_arguments(cls, c) -> List[Tuple]:
+        text = c["choices"][0]["text"]
+        try:
+            data = eval(text, {}, {})
+            return [(None, data["name"], data["parameters"])]
+        except Exception:
+            return [(text, None, None)]
+
     @classmethod
     def _eval_tool_arguments(cls, model_family, c):
         family = model_family.model_family or model_family.model_name
@@ -349,6 +376,8 @@ class ChatModelMixin:
             result = cls._eval_glm_chat_arguments(c)
         elif family in QWEN_TOOL_CALL_FAMILY:
             result = cls._eval_qwen_chat_arguments(c)
+        elif family in LLAMA3_TOOL_CALL_FAMILY:
+            result = cls._eval_llama3_chat_arguments(c)
         else:
             raise Exception(
                 f"Model {model_family.model_name} is not support tool calls."
@@ -365,16 +394,14 @@ class ChatModelMixin:
         for content, func, args in tool_result:
             if func:
                 tool_calls.append(
-                    [
-                        {
-                            "id": f"call_{_id}",
-                            "type": "function",
-                            "function": {
-                                "name": func,
-                                "arguments": json.dumps(args, ensure_ascii=False),
-                            },
-                        }
-                    ]
+                    {
+                        "id": f"call_{_id}",
+                        "type": "function",
+                        "function": {
+                            "name": func,
+                            "arguments": json.dumps(args, ensure_ascii=False),
+                        },
+                    }
                 )
             else:
                 failed_contents.append(content)
@@ -460,6 +487,34 @@ class ChatModelMixin:
             "usage": usage,
         }
 
+    def _transform_messages(
+        self,
+        messages: List[ChatCompletionMessage],
+    ):
+        transformed_messages = []
+        for msg in messages:
+            new_content = []
+            role = msg["role"]
+            content = msg["content"]
+            if isinstance(content, str):
+                new_content.append({"type": "text", "text": content})
+            elif isinstance(content, List):
+                for item in content:  # type: ignore
+                    if "text" in item:
+                        new_content.append({"type": "text", "text": item["text"]})
+                    elif "image_url" in item:
+                        new_content.append(
+                            {"type": "image", "image": item["image_url"]["url"]}
+                        )
+                    elif "video_url" in item:
+                        new_content.append(
+                            {"type": "video", "video": item["video_url"]["url"]}
+                        )
+            new_message = {"role": role, "content": new_content}
+            transformed_messages.append(new_message)
+
+        return transformed_messages
+
 
 def get_file_location(
     llm_family: LLMFamilyV1, spec: LLMSpecV1, quantization: str

xinference/model/llm/vllm/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import json
 import logging
 import multiprocessing
 import os
@@ -33,6 +34,7 @@ from typing import (
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
+    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -47,6 +49,7 @@ from ..utils import (
     ChatModelMixin,
     generate_completion_chunk,
 )
+from .utils import vllm_check
 
 logger = logging.getLogger(__name__)
 
@@ -65,6 +68,7 @@ class VLLMModelConfig(TypedDict, total=False):
     max_num_seqs: int
     quantization: Optional[str]
     max_model_len: Optional[int]
+    limit_mm_per_prompt: Optional[Dict[str, int]]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -90,9 +94,7 @@ try:
 except ImportError:
     VLLM_INSTALLED = False
 
-VLLM_SUPPORTED_VISION_MODEL_LIST: List[str] = [
-    "internvl2",
-]
+VLLM_SUPPORTED_VISION_MODEL_LIST: List[str] = []
 VLLM_SUPPORTED_MODELS = [
     "llama-2",
     "llama-3",
@@ -171,6 +173,12 @@ if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
+
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -304,7 +312,7 @@ class VLLMModel(LLM):
         model_config.setdefault("gpu_memory_utilization", 0.90)
         model_config.setdefault("max_num_seqs", 256)
         model_config.setdefault("quantization", None)
-        model_config.setdefault("max_model_len", 4096)
+        model_config.setdefault("max_model_len", None)
 
         return model_config
 
@@ -434,6 +442,7 @@ class VLLMModel(LLM):
             usage=usage,
         )
 
+    @vllm_check
     async def async_generate(
         self,
         prompt: Union[str, Dict[str, Any]],
@@ -665,6 +674,7 @@ class VLLMChatModel(VLLMModel, ChatModelMixin):
                     yield self._to_chat_completion_chunk(chunk)
             i += 1
 
+    @vllm_check
     async def async_chat(
         self,
         messages: List[Dict],
@@ -722,6 +732,33 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
             return False
         return VLLM_INSTALLED
 
+    def _sanitize_model_config(
+        self, model_config: Optional[VLLMModelConfig]
+    ) -> VLLMModelConfig:
+        if model_config is None:
+            model_config = VLLMModelConfig()
+
+        cuda_count = self._get_cuda_count()
+
+        model_config.setdefault("tokenizer_mode", "auto")
+        model_config.setdefault("trust_remote_code", True)
+        model_config.setdefault("tensor_parallel_size", cuda_count)
+        model_config.setdefault("block_size", 16)
+        model_config.setdefault("swap_space", 4)
+        model_config.setdefault("gpu_memory_utilization", 0.90)
+        model_config.setdefault("max_num_seqs", 256)
+        model_config.setdefault("quantization", None)
+        model_config.setdefault("max_model_len", None)
+        model_config["limit_mm_per_prompt"] = (
+            json.loads(model_config.get("limit_mm_per_prompt"))  # type: ignore
+            if model_config.get("limit_mm_per_prompt")
+            else {
+                "image": 2,  # default 2 images all chat
+            }
+        )
+
+        return model_config
+
     def _sanitize_chat_config(
         self,
         generate_config: Optional[Dict] = None,
@@ -741,25 +778,48 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
                     )
         return generate_config
 
+    @vllm_check
     async def async_chat(
         self,
-        messages: List[Dict],
+        messages: List[ChatCompletionMessage],  # type: ignore
         generate_config: Optional[Dict] = None,
         request_id: Optional[str] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
-        # only support single image, waiting vllm support multi images
+        messages = self._transform_messages(messages)
+        tools = generate_config.pop("tools", []) if generate_config else None
+
         model_family = self.model_family.model_family or self.model_family.model_name
-        prompt, images = self.get_specific_prompt(model_family, messages)
 
-        if len(images) == 0:
+        if "internvl2" not in model_family.lower():
+            from qwen_vl_utils import process_vision_info
+
+            full_context_kwargs = {}
+            if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            assert self.model_family.chat_template is not None
+            prompt = self.get_full_context(
+                messages, self.model_family.chat_template, **full_context_kwargs
+            )
+            images, video_inputs = process_vision_info(messages)
+            if video_inputs:
+                raise ValueError("Not support video input now.")
+        else:
+            prompt, images = self.get_specific_prompt(model_family, messages)
+
+        if not images:
             inputs = {
                 "prompt": prompt,
             }
-        else:
+        elif len(images) == 1:
             inputs = {
                 "prompt": prompt,
                 "multi_modal_data": {"image": images[-1]},  # type: ignore
             }
+        else:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images},  # type: ignore
+            }
         generate_config = self._sanitize_chat_config(generate_config)
 
         stream = generate_config.get("stream", None)

xinference/model/llm/vllm/utils.py

@@ -0,0 +1,41 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import functools
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def vllm_check(fn):
+    try:
+        from vllm.engine.async_llm_engine import AsyncEngineDeadError
+    except:
+        return fn
+
+    @functools.wraps(fn)
+    async def _async_wrapper(self, *args, **kwargs):
+        try:
+            return await fn(self, *args, **kwargs)
+        except AsyncEngineDeadError:
+            logger.info("Detecting vLLM is not health, prepare to quit the process")
+            try:
+                self.stop()
+            except:
+                # ignore error when stop
+                pass
+            # Just kill the process and let xinference auto-recover the model
+            os._exit(1)
+
+    return _async_wrapper

xinference/model/rerank/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import gc
+import importlib
 import logging
 import os
 import threading
@@ -178,9 +179,27 @@ class RerankModel:
         return rerank_type
 
     def load(self):
+        flash_attn_installed = importlib.util.find_spec("flash_attn") is not None
+        if (
+            self._auto_detect_type(self._model_path) != "normal"
+            and flash_attn_installed
+        ):
+            logger.warning(
+                "flash_attn can only support fp16 and bf16, "
+                "will force set `use_fp16` to True"
+            )
+            self._use_fp16 = True
         if self._model_spec.type == "normal":
             try:
+                import sentence_transformers
                 from sentence_transformers.cross_encoder import CrossEncoder
+
+                if sentence_transformers.__version__ < "3.1.0":
+                    raise ValueError(
+                        "The sentence_transformers version must be greater than 3.1.0. "
+                        "Please upgrade your version via `pip install -U sentence_transformers` or refer to "
+                        "https://github.com/UKPLab/sentence-transformers"
+                    )
             except ImportError:
                 error_message = "Failed to import module 'sentence-transformers'"
                 installation_guide = [

xinference/model/rerank/model_spec.json

@@ -54,5 +54,13 @@
     "max_tokens": 1024,
     "model_id": "jinaai/jina-reranker-v2-base-multilingual",
     "model_revision": "298e48cada4a9318650d7fbd795f63827f884087"
+  },
+  {
+    "model_name": "minicpm-reranker",
+    "type": "normal",
+    "language": ["en", "zh"],
+    "max_tokens": 1024,
+    "model_id": "openbmb/MiniCPM-Reranker",
+    "model_revision": "5d2fd7345b6444c89d4c0fa59c92272888f3f2d0"
   }
 ]

xinference/model/rerank/model_spec_modelscope.json

@@ -49,5 +49,13 @@
     "max_tokens": 2048,
     "model_id": "mirror013/bge-reranker-v2-minicpm-layerwise",
     "model_hub": "modelscope"
+  },
+  {
+    "model_name": "minicpm-reranker",
+    "type": "normal",
+    "language": ["en", "zh"],
+    "max_tokens": 1024,
+    "model_id": "OpenBMB/MiniCPM-Reranker",
+    "model_hub": "modelscope"
   }
 ]

xinference/model/utils.py

@@ -23,12 +23,15 @@ import huggingface_hub
 import numpy as np
 import torch
 
-from ..constants import XINFERENCE_CACHE_DIR, XINFERENCE_ENV_MODEL_SRC
+from ..constants import (
+    XINFERENCE_CACHE_DIR,
+    XINFERENCE_DOWNLOAD_MAX_ATTEMPTS,
+    XINFERENCE_ENV_MODEL_SRC,
+)
 from ..device_utils import get_available_device, is_device_available
 from .core import CacheableModelSpec
 
 logger = logging.getLogger(__name__)
-MAX_ATTEMPTS = 3
 IS_NEW_HUGGINGFACE_HUB: bool = huggingface_hub.__version__ >= "0.23.0"
 
 
@@ -100,11 +103,11 @@ def retry_download(
     **kwargs,
 ):
     last_ex = None
-    for current_attempt in range(1, MAX_ATTEMPTS + 1):
+    for current_attempt in range(1, XINFERENCE_DOWNLOAD_MAX_ATTEMPTS + 1):
         try:
             return download_func(*args, **kwargs)
         except Exception as e:
-            remaining_attempts = MAX_ATTEMPTS - current_attempt
+            remaining_attempts = XINFERENCE_DOWNLOAD_MAX_ATTEMPTS - current_attempt
             last_ex = e
             logger.debug(
                 "Download failed: %s, download func: %s, download args: %s, kwargs: %s",
@@ -300,31 +303,6 @@ def cache(model_spec: CacheableModelSpec, model_description_type: type):
     return cache_dir
 
 
-def patch_trust_remote_code():
-    """sentence-transformers calls transformers without the trust_remote_code=True, some embedding
-    models will fail to load, e.g. jina-embeddings-v2-base-en
-
-    :return:
-    """
-    try:
-        from transformers.dynamic_module_utils import resolve_trust_remote_code
-    except ImportError:
-        logger.error("Patch transformers trust_remote_code failed.")
-    else:
-
-        def _patched_resolve_trust_remote_code(*args, **kwargs):
-            logger.info("Patched resolve_trust_remote_code: %s %s", args, kwargs)
-            return True
-
-        if (
-            resolve_trust_remote_code.__code__
-            != _patched_resolve_trust_remote_code.__code__
-        ):
-            resolve_trust_remote_code.__code__ = (
-                _patched_resolve_trust_remote_code.__code__
-            )
-
-
 def select_device(device):
     try:
         import torch  # noqa: F401

xinference/model/video/core.py

@@ -21,8 +21,6 @@ from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .diffusers import DiffUsersVideoModel
 
-MAX_ATTEMPTS = 3
-
 logger = logging.getLogger(__name__)
 
 MODEL_NAME_TO_REVISION: Dict[str, List[str]] = defaultdict(list)

xinference/web/ui/build/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.5061c4c3.css",
-    "main.js": "./static/js/main.29578905.js",
+    "main.js": "./static/js/main.e51a356d.js",
     "static/media/icon.webp": "./static/media/icon.4603d52c63041e5dfbfd.webp",
     "index.html": "./index.html",
     "main.5061c4c3.css.map": "./static/css/main.5061c4c3.css.map",
-    "main.29578905.js.map": "./static/js/main.29578905.js.map"
+    "main.e51a356d.js.map": "./static/js/main.e51a356d.js.map"
   },
   "entrypoints": [
     "static/css/main.5061c4c3.css",
-    "static/js/main.29578905.js"
+    "static/js/main.e51a356d.js"
   ]
 }

xinference/web/ui/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.29578905.js"></script><link href="./static/css/main.5061c4c3.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.e51a356d.js"></script><link href="./static/css/main.5061c4c3.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>