xinference 0.15.1__py3-none-any.whl → 0.15.3__py3-none-any.whl

xinference/model/llm/sglang/core.py

@@ -68,6 +68,8 @@ SGLANG_SUPPORTED_MODELS = [
     "llama-3.1",
     "mistral-v0.1",
     "mixtral-v0.1",
+    "qwen2.5",
+    "qwen2.5-coder",
 ]
 SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
@@ -85,6 +87,8 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "deepseek-v2.5",
     "deepseek-v2-chat",
     "deepseek-v2-chat-0628",
+    "qwen2.5-instruct",
+    "qwen2.5-coder-instruct",
 ]
 
 

xinference/model/llm/utils.py

@@ -159,14 +159,25 @@ class ChatModelMixin:
                         for image_url in image_urls:
                             fut = executor.submit(_decode_image, image_url)
                             image_futures.append(fut)
-                    images = [fut.result() for fut in image_futures]
+                    images.extend([fut.result() for fut in image_futures])
                     if len(image_futures) == 0:
                         ret += role + "\n" + text + intra_message_sep + "\n"
                     else:
+                        placeholders = "\n".join(
+                            f"Image-{i+1}: <image>\n"
+                            for i in range(
+                                len(images) - len(image_futures), len(images)
+                            )
+                        )
                         ret += (
-                            role + "\n" + f"<image>\n{text}" + intra_message_sep + "\n"
+                            role
+                            + "\n"
+                            + f"{placeholders}\n{text}"
+                            + intra_message_sep
+                            + "\n"
                         )
-
+            if len(images) == 1:
+                ret = ret.replace("Image-1: <image>\n", "<image>\n")
             return ret, images
         else:
             raise ValueError(f"Invalid model family: {model_family}")

xinference/model/llm/vllm/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import json
 import logging
 import multiprocessing
 import os
@@ -47,6 +48,7 @@ from ..utils import (
     ChatModelMixin,
     generate_completion_chunk,
 )
+from .utils import vllm_check
 
 logger = logging.getLogger(__name__)
 
@@ -65,6 +67,7 @@ class VLLMModelConfig(TypedDict, total=False):
     max_num_seqs: int
     quantization: Optional[str]
     max_model_len: Optional[int]
+    limit_mm_per_prompt: Optional[Dict[str, int]]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -90,9 +93,7 @@ try:
 except ImportError:
     VLLM_INSTALLED = False
 
-VLLM_SUPPORTED_VISION_MODEL_LIST: List[str] = [
-    "internvl2",
-]
+VLLM_SUPPORTED_VISION_MODEL_LIST: List[str] = []
 VLLM_SUPPORTED_MODELS = [
     "llama-2",
     "llama-3",
@@ -138,6 +139,11 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_MODELS.append("codeqwen1.5")
     VLLM_SUPPORTED_CHAT_MODELS.append("codeqwen1.5-chat")
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2-instruct")
+    VLLM_SUPPORTED_MODELS.append("qwen2.5")
+    VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-instruct")
+    VLLM_SUPPORTED_MODELS.append("qwen2.5-coder")
+    VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-coder-instruct")
+
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
     VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
@@ -166,6 +172,9 @@ if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -299,7 +308,12 @@ class VLLMModel(LLM):
         model_config.setdefault("gpu_memory_utilization", 0.90)
         model_config.setdefault("max_num_seqs", 256)
         model_config.setdefault("quantization", None)
-        model_config.setdefault("max_model_len", 4096)
+        model_config.setdefault("max_model_len", None)
+        model_config["limit_mm_per_prompt"] = (
+            json.loads(model_config.get("limit_mm_per_prompt"))  # type: ignore
+            if model_config.get("limit_mm_per_prompt")
+            else None
+        )
 
         return model_config
 
@@ -429,6 +443,7 @@ class VLLMModel(LLM):
             usage=usage,
         )
 
+    @vllm_check
     async def async_generate(
         self,
         prompt: Union[str, Dict[str, Any]],
@@ -660,6 +675,7 @@ class VLLMChatModel(VLLMModel, ChatModelMixin):
                     yield self._to_chat_completion_chunk(chunk)
             i += 1
 
+    @vllm_check
     async def async_chat(
         self,
         messages: List[Dict],
@@ -736,13 +752,13 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
                     )
         return generate_config
 
+    @vllm_check
     async def async_chat(
         self,
         messages: List[Dict],
         generate_config: Optional[Dict] = None,
         request_id: Optional[str] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
-        # only support single image, waiting vllm support multi images
         model_family = self.model_family.model_family or self.model_family.model_name
         prompt, images = self.get_specific_prompt(model_family, messages)
 
@@ -750,11 +766,16 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
             inputs = {
                 "prompt": prompt,
             }
-        else:
+        elif len(images) == 1:
             inputs = {
                 "prompt": prompt,
                 "multi_modal_data": {"image": images[-1]},  # type: ignore
             }
+        else:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images},  # type: ignore
+            }
         generate_config = self._sanitize_chat_config(generate_config)
 
         stream = generate_config.get("stream", None)

xinference/model/llm/vllm/utils.py

@@ -0,0 +1,42 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import functools
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def vllm_check(fn):
+    try:
+        from vllm.engine.async_llm_engine import AsyncEngineDeadError
+    except:
+        return fn
+
+    @functools.wraps(fn)
+    async def _async_wrapper(self, *args, **kwargs):
+        logger.info("vllm_check")
+        try:
+            return await fn(self, *args, **kwargs)
+        except AsyncEngineDeadError:
+            logger.info("Detecting vLLM is not health, prepare to quit the process")
+            try:
+                self.stop()
+            except:
+                # ignore error when stop
+                pass
+            # Just kill the process and let xinference auto-recover the model
+            os._exit(1)
+
+    return _async_wrapper

xinference/model/rerank/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import gc
+import importlib
 import logging
 import os
 import threading
@@ -178,9 +179,27 @@ class RerankModel:
         return rerank_type
 
     def load(self):
+        flash_attn_installed = importlib.util.find_spec("flash_attn") is not None
+        if (
+            self._auto_detect_type(self._model_path) != "normal"
+            and flash_attn_installed
+        ):
+            logger.warning(
+                "flash_attn can only support fp16 and bf16, "
+                "will force set `use_fp16` to True"
+            )
+            self._use_fp16 = True
         if self._model_spec.type == "normal":
             try:
+                import sentence_transformers
                 from sentence_transformers.cross_encoder import CrossEncoder
+
+                if sentence_transformers.__version__ < "3.1.0":
+                    raise ValueError(
+                        "The sentence_transformers version must be greater than 3.1.0. "
+                        "Please upgrade your version via `pip install -U sentence_transformers` or refer to "
+                        "https://github.com/UKPLab/sentence-transformers"
+                    )
             except ImportError:
                 error_message = "Failed to import module 'sentence-transformers'"
                 installation_guide = [

xinference/model/rerank/model_spec.json

@@ -54,5 +54,13 @@
     "max_tokens": 1024,
     "model_id": "jinaai/jina-reranker-v2-base-multilingual",
     "model_revision": "298e48cada4a9318650d7fbd795f63827f884087"
+  },
+  {
+    "model_name": "minicpm-reranker",
+    "type": "normal",
+    "language": ["en", "zh"],
+    "max_tokens": 1024,
+    "model_id": "openbmb/MiniCPM-Reranker",
+    "model_revision": "5d2fd7345b6444c89d4c0fa59c92272888f3f2d0"
   }
 ]

xinference/model/rerank/model_spec_modelscope.json

@@ -49,5 +49,13 @@
     "max_tokens": 2048,
     "model_id": "mirror013/bge-reranker-v2-minicpm-layerwise",
     "model_hub": "modelscope"
+  },
+  {
+    "model_name": "minicpm-reranker",
+    "type": "normal",
+    "language": ["en", "zh"],
+    "max_tokens": 1024,
+    "model_id": "OpenBMB/MiniCPM-Reranker",
+    "model_hub": "modelscope"
   }
 ]

xinference/model/utils.py

@@ -300,31 +300,6 @@ def cache(model_spec: CacheableModelSpec, model_description_type: type):
     return cache_dir
 
 
-def patch_trust_remote_code():
-    """sentence-transformers calls transformers without the trust_remote_code=True, some embedding
-    models will fail to load, e.g. jina-embeddings-v2-base-en
-
-    :return:
-    """
-    try:
-        from transformers.dynamic_module_utils import resolve_trust_remote_code
-    except ImportError:
-        logger.error("Patch transformers trust_remote_code failed.")
-    else:
-
-        def _patched_resolve_trust_remote_code(*args, **kwargs):
-            logger.info("Patched resolve_trust_remote_code: %s %s", args, kwargs)
-            return True
-
-        if (
-            resolve_trust_remote_code.__code__
-            != _patched_resolve_trust_remote_code.__code__
-        ):
-            resolve_trust_remote_code.__code__ = (
-                _patched_resolve_trust_remote_code.__code__
-            )
-
-
 def select_device(device):
     try:
         import torch  # noqa: F401

xinference/web/ui/build/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.5061c4c3.css",
-    "main.js": "./static/js/main.754740c0.js",
+    "main.js": "./static/js/main.e51a356d.js",
     "static/media/icon.webp": "./static/media/icon.4603d52c63041e5dfbfd.webp",
     "index.html": "./index.html",
     "main.5061c4c3.css.map": "./static/css/main.5061c4c3.css.map",
-    "main.754740c0.js.map": "./static/js/main.754740c0.js.map"
+    "main.e51a356d.js.map": "./static/js/main.e51a356d.js.map"
   },
   "entrypoints": [
     "static/css/main.5061c4c3.css",
-    "static/js/main.754740c0.js"
+    "static/js/main.e51a356d.js"
   ]
 }

xinference/web/ui/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.754740c0.js"></script><link href="./static/css/main.5061c4c3.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.e51a356d.js"></script><link href="./static/css/main.5061c4c3.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>