xinference 0.15.3__py3-none-any.whl → 0.15.4__py3-none-any.whl

xinference/model/llm/utils.py

@@ -29,6 +29,7 @@ from ...types import (
     ChatCompletion,
     ChatCompletionChoice,
     ChatCompletionChunk,
+    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -50,6 +51,7 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen1.5-moe-chat",
     "qwen2-instruct",
     "qwen2-moe-instruct",
+    "qwen2.5-instruct",
 ]
 
 GLM4_TOOL_CALL_FAMILY = [
@@ -57,6 +59,10 @@ GLM4_TOOL_CALL_FAMILY = [
     "glm4-chat-1m",
 ]
 
+LLAMA3_TOOL_CALL_FAMILY = [
+    "llama-3.1-instruct",
+]
+
 QWEN_TOOL_CALL_SYMBOLS = ["<tool_call>", "</tool_call>"]
 
 
@@ -113,7 +119,7 @@ class ChatModelMixin:
             return self._build_from_raw_template(messages, chat_template, **kwargs)
 
     @staticmethod
-    def get_specific_prompt(model_family: str, messages: List[Dict]):
+    def get_specific_prompt(model_family: str, messages: List[ChatCompletionMessage]):
         """
         Inspired by FastChat. Format chat history into a prompt according to the prompty style of
         different models.
@@ -129,7 +135,7 @@ class ChatModelMixin:
             ret = (
                 "<s>"
                 if system_prompt == ""
-                else "<s><|im_start|>system\n"
+                else "<s><|im_start|>system\n"  # type: ignore
                 + system_prompt
                 + intra_message_sep
                 + "\n"
@@ -333,8 +339,9 @@ class ChatModelMixin:
         for content in contents:
             content = content.strip()
             if content:
-                if content.startswith(QWEN_TOOL_CALL_SYMBOLS[0]):
-                    content = content[len(QWEN_TOOL_CALL_SYMBOLS[0]) :]
+                pos = content.find(QWEN_TOOL_CALL_SYMBOLS[0])
+                if pos != -1:
+                    content = content[pos + len(QWEN_TOOL_CALL_SYMBOLS[0]) :]
                 content = content.strip()
                 try:
                     res = json.loads(content)
@@ -353,6 +360,15 @@ class ChatModelMixin:
         text = c["choices"][0]["text"]
         return cls._handle_qwen_tool_result(text)
 
+    @classmethod
+    def _eval_llama3_chat_arguments(cls, c) -> List[Tuple]:
+        text = c["choices"][0]["text"]
+        try:
+            data = eval(text, {}, {})
+            return [(None, data["name"], data["parameters"])]
+        except Exception:
+            return [(text, None, None)]
+
     @classmethod
     def _eval_tool_arguments(cls, model_family, c):
         family = model_family.model_family or model_family.model_name
@@ -360,6 +376,8 @@ class ChatModelMixin:
             result = cls._eval_glm_chat_arguments(c)
         elif family in QWEN_TOOL_CALL_FAMILY:
             result = cls._eval_qwen_chat_arguments(c)
+        elif family in LLAMA3_TOOL_CALL_FAMILY:
+            result = cls._eval_llama3_chat_arguments(c)
         else:
             raise Exception(
                 f"Model {model_family.model_name} is not support tool calls."
@@ -376,16 +394,14 @@ class ChatModelMixin:
         for content, func, args in tool_result:
             if func:
                 tool_calls.append(
-                    [
-                        {
-                            "id": f"call_{_id}",
-                            "type": "function",
-                            "function": {
-                                "name": func,
-                                "arguments": json.dumps(args, ensure_ascii=False),
-                            },
-                        }
-                    ]
+                    {
+                        "id": f"call_{_id}",
+                        "type": "function",
+                        "function": {
+                            "name": func,
+                            "arguments": json.dumps(args, ensure_ascii=False),
+                        },
+                    }
                 )
             else:
                 failed_contents.append(content)
@@ -471,6 +487,34 @@ class ChatModelMixin:
             "usage": usage,
         }
 
+    def _transform_messages(
+        self,
+        messages: List[ChatCompletionMessage],
+    ):
+        transformed_messages = []
+        for msg in messages:
+            new_content = []
+            role = msg["role"]
+            content = msg["content"]
+            if isinstance(content, str):
+                new_content.append({"type": "text", "text": content})
+            elif isinstance(content, List):
+                for item in content:  # type: ignore
+                    if "text" in item:
+                        new_content.append({"type": "text", "text": item["text"]})
+                    elif "image_url" in item:
+                        new_content.append(
+                            {"type": "image", "image": item["image_url"]["url"]}
+                        )
+                    elif "video_url" in item:
+                        new_content.append(
+                            {"type": "video", "video": item["video_url"]["url"]}
+                        )
+            new_message = {"role": role, "content": new_content}
+            transformed_messages.append(new_message)
+
+        return transformed_messages
+
 
 def get_file_location(
     llm_family: LLMFamilyV1, spec: LLMSpecV1, quantization: str

xinference/model/llm/vllm/core.py

@@ -34,6 +34,7 @@ from typing import (
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
+    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -175,6 +176,9 @@ if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -309,11 +313,6 @@ class VLLMModel(LLM):
         model_config.setdefault("max_num_seqs", 256)
         model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
-        model_config["limit_mm_per_prompt"] = (
-            json.loads(model_config.get("limit_mm_per_prompt"))  # type: ignore
-            if model_config.get("limit_mm_per_prompt")
-            else None
-        )
 
         return model_config
 
@@ -733,6 +732,33 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
             return False
         return VLLM_INSTALLED
 
+    def _sanitize_model_config(
+        self, model_config: Optional[VLLMModelConfig]
+    ) -> VLLMModelConfig:
+        if model_config is None:
+            model_config = VLLMModelConfig()
+
+        cuda_count = self._get_cuda_count()
+
+        model_config.setdefault("tokenizer_mode", "auto")
+        model_config.setdefault("trust_remote_code", True)
+        model_config.setdefault("tensor_parallel_size", cuda_count)
+        model_config.setdefault("block_size", 16)
+        model_config.setdefault("swap_space", 4)
+        model_config.setdefault("gpu_memory_utilization", 0.90)
+        model_config.setdefault("max_num_seqs", 256)
+        model_config.setdefault("quantization", None)
+        model_config.setdefault("max_model_len", None)
+        model_config["limit_mm_per_prompt"] = (
+            json.loads(model_config.get("limit_mm_per_prompt"))  # type: ignore
+            if model_config.get("limit_mm_per_prompt")
+            else {
+                "image": 2,  # default 2 images all chat
+            }
+        )
+
+        return model_config
+
     def _sanitize_chat_config(
         self,
         generate_config: Optional[Dict] = None,
@@ -755,14 +781,32 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
     @vllm_check
     async def async_chat(
         self,
-        messages: List[Dict],
+        messages: List[ChatCompletionMessage],  # type: ignore
         generate_config: Optional[Dict] = None,
         request_id: Optional[str] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
+        messages = self._transform_messages(messages)
+        tools = generate_config.pop("tools", []) if generate_config else None
+
         model_family = self.model_family.model_family or self.model_family.model_name
-        prompt, images = self.get_specific_prompt(model_family, messages)
 
-        if len(images) == 0:
+        if "internvl2" not in model_family.lower():
+            from qwen_vl_utils import process_vision_info
+
+            full_context_kwargs = {}
+            if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            assert self.model_family.chat_template is not None
+            prompt = self.get_full_context(
+                messages, self.model_family.chat_template, **full_context_kwargs
+            )
+            images, video_inputs = process_vision_info(messages)
+            if video_inputs:
+                raise ValueError("Not support video input now.")
+        else:
+            prompt, images = self.get_specific_prompt(model_family, messages)
+
+        if not images:
             inputs = {
                 "prompt": prompt,
             }

xinference/model/llm/vllm/utils.py

@@ -26,7 +26,6 @@ def vllm_check(fn):
 
     @functools.wraps(fn)
     async def _async_wrapper(self, *args, **kwargs):
-        logger.info("vllm_check")
         try:
             return await fn(self, *args, **kwargs)
         except AsyncEngineDeadError:

xinference/model/utils.py

@@ -23,12 +23,15 @@ import huggingface_hub
 import numpy as np
 import torch
 
-from ..constants import XINFERENCE_CACHE_DIR, XINFERENCE_ENV_MODEL_SRC
+from ..constants import (
+    XINFERENCE_CACHE_DIR,
+    XINFERENCE_DOWNLOAD_MAX_ATTEMPTS,
+    XINFERENCE_ENV_MODEL_SRC,
+)
 from ..device_utils import get_available_device, is_device_available
 from .core import CacheableModelSpec
 
 logger = logging.getLogger(__name__)
-MAX_ATTEMPTS = 3
 IS_NEW_HUGGINGFACE_HUB: bool = huggingface_hub.__version__ >= "0.23.0"
 
 
@@ -100,11 +103,11 @@ def retry_download(
     **kwargs,
 ):
     last_ex = None
-    for current_attempt in range(1, MAX_ATTEMPTS + 1):
+    for current_attempt in range(1, XINFERENCE_DOWNLOAD_MAX_ATTEMPTS + 1):
         try:
             return download_func(*args, **kwargs)
         except Exception as e:
-            remaining_attempts = MAX_ATTEMPTS - current_attempt
+            remaining_attempts = XINFERENCE_DOWNLOAD_MAX_ATTEMPTS - current_attempt
             last_ex = e
             logger.debug(
                 "Download failed: %s, download func: %s, download args: %s, kwargs: %s",

xinference/model/video/core.py

@@ -21,8 +21,6 @@ from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .diffusers import DiffUsersVideoModel
 
-MAX_ATTEMPTS = 3
-
 logger = logging.getLogger(__name__)
 
 MODEL_NAME_TO_REVISION: Dict[str, List[str]] = defaultdict(list)

{xinference-0.15.3.dist-info → xinference-0.15.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xinference
-Version: 0.15.3
+Version: 0.15.4
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye
@@ -71,7 +71,7 @@ Requires-Dist: WeTextProcessing<1.0.4; extra == "all"
 Requires-Dist: librosa; extra == "all"
 Requires-Dist: xxhash; extra == "all"
 Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: ChatTTS>0.1; extra == "all"
+Requires-Dist: ChatTTS<0.2,>0.1; extra == "all"
 Requires-Dist: lightning>=2.0.0; extra == "all"
 Requires-Dist: hydra-core>=1.3.2; extra == "all"
 Requires-Dist: inflect; extra == "all"
@@ -104,7 +104,7 @@ Requires-Dist: WeTextProcessing<1.0.4; extra == "audio"
 Requires-Dist: librosa; extra == "audio"
 Requires-Dist: xxhash; extra == "audio"
 Requires-Dist: torchaudio; extra == "audio"
-Requires-Dist: ChatTTS>0.1; extra == "audio"
+Requires-Dist: ChatTTS<0.2,>0.1; extra == "audio"
 Requires-Dist: tiktoken; extra == "audio"
 Requires-Dist: torch>=2.0.0; extra == "audio"
 Requires-Dist: lightning>=2.0.0; extra == "audio"

{xinference-0.15.3.dist-info → xinference-0.15.4.dist-info}/RECORD

@@ -1,15 +1,15 @@
 xinference/__init__.py,sha256=muQ9V9y11BcIqlZhhc06oDf193H7bwDIa8e_wSoDKI8,986
 xinference/_compat.py,sha256=xFztCfyrq3O_4bssL_ygghYkfxicv_ZhiX2YDDWHf-k,3571
-xinference/_version.py,sha256=RGjl0KY7iZC63yTyDPQaCtunSUGRo4ApaNUopMSdDP8,498
+xinference/_version.py,sha256=rsb6h82zrpecXkwoQPZlv48UaKLjrNi7qg2qXrXoSZE,498
 xinference/conftest.py,sha256=56HYQjsAJcQrpZSmskniPqH9dLoW-i3Oud6NVTtc4io,9752
-xinference/constants.py,sha256=f8RxXrnnhoEYSwhiDSp8nKeUMF-KE4GyerMg-pa3Vv4,3582
+xinference/constants.py,sha256=QsYxf86vqJVbn5L2SXH5hdyCdiZy7hRD-Qx5Sse4DqE,3758
 xinference/device_utils.py,sha256=zswJiws3VyTIaNO8z-MOcsJH_UiPoePPiKK5zoNrjTA,3285
 xinference/fields.py,sha256=0UtBFaDNzn1n9MRjyTkNrolsIML-TpZfudWOejqjni8,5245
 xinference/isolation.py,sha256=uhkzVyL3fSYZSuFexkG6Jm-tRTC5I607uNg000BXAnE,1949
 xinference/types.py,sha256=LHTbNLf0zI-FLruxRuBt2KMpk2P4eKpYdFvh2qzNTGI,12458
 xinference/utils.py,sha256=VSOJMFd9H7kce98OtJZbcDjjpfzRpHAFs8WU0xXPBM8,717
 xinference/api/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/api/restful_api.py,sha256=5UR9RVbrxOJmBwwepxOrRmH49JGVzkp6Tku1os-zzLU,83540
+xinference/api/restful_api.py,sha256=Gp_1fGYLximhr9yTqxvBv9O84HO47-tnTwA5h7o8Ff4,84506
 xinference/api/oauth2/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/api/oauth2/auth_service.py,sha256=74JzB42fbbmBu4Q1dW3A9Fp_N7167KgRGB42Z0NHjAM,6119
 xinference/api/oauth2/types.py,sha256=K923sv_XySIUtM2Eozl9IG082IJcDOS5SFLrPZ5ELBg,996
@@ -18,20 +18,21 @@ xinference/client/__init__.py,sha256=Gc4HOzAy_1cic5kXlso7hahYgw89CKvZSJDicEU461k
 xinference/client/common.py,sha256=iciZRs5YjM2gYsXnwACPMaiBZp4_XpawWwfym0Iyu40,1617
 xinference/client/handlers.py,sha256=OKl_i5FA341wsQf_0onSOPbbW6V861WJrSP7ghtDc8c,527
 xinference/client/restful/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/client/restful/restful_client.py,sha256=rz3d5n1sTluZG6zj0B8jsM40LLNtUAlmPzDdBDrDvFY,50780
+xinference/client/restful/restful_client.py,sha256=eTZf9M0GG6ZaShWhpY7O-yG0BH3ceKZZ-d-DuVDg55g,51189
 xinference/core/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/core/cache_tracker.py,sha256=3ubjYCU5aZToSp2GEuzedECVrg-PR4kThTefrFUkb9g,6971
 xinference/core/chat_interface.py,sha256=tM4hQPZ0UVcmE4_-auXWkq2z0rWmZBwwXNwbbl5zvGQ,20666
 xinference/core/event.py,sha256=42F38H2WOl6aPxp2oxX6WNxHRRxbnvYRmbt4Ar7NP4U,1640
-xinference/core/image_interface.py,sha256=-elEvAYVga8KXbl9uc1A8oV0YWK0QbKDu5RPofkxxXs,11837
+xinference/core/image_interface.py,sha256=WsJjrcJG3itQJb-qiuZKWtK4_XLPPnxkEy9VCyZcQmw,13636
 xinference/core/metrics.py,sha256=ScmTG15Uq3h_ob72ybZSMWdnk8P4sUZFcm60f4ikSXc,2631
-xinference/core/model.py,sha256=_X0aPIcTtdy886tMxuoto_nwXqCrm3S8IMalv3Kq5QU,30354
+xinference/core/model.py,sha256=bwmiqRctnXbJlsHhS3O3JA5G0xRNNRd_HqlAGyRrzVo,32086
+xinference/core/progress_tracker.py,sha256=LIF6CLIlnEoSBkuDCraJktDOzZ31mQ4HOo6EVr3KpQM,6453
 xinference/core/resource.py,sha256=FQ0aRt3T4ZQo0P6CZZf5QUKHiCsr5llBvKb1f7wfnxg,1611
 xinference/core/scheduler.py,sha256=qONNFqAlnYDcmmzPO5jfU-r0aZ1Lhhpn1oSaA5CAGTE,15485
 xinference/core/status_guard.py,sha256=4an1KjUOhCStgRQUw1VSzXcycXUtvhxwiMREKKcl1UI,2828
-xinference/core/supervisor.py,sha256=bNMyGM-cqHwSqhYxHlR6oePEKqt9D4tcrBFMAb6-oV0,52510
+xinference/core/supervisor.py,sha256=Wkjhk1tfRuhyQmcVNrHZApWO09MDA5-Uu4u2p1GBj3I,52964
 xinference/core/utils.py,sha256=p3ptQMdzKu9WxdUJ2EdDTXvPDl53BGwiNuVWuhaE4EU,8536
-xinference/core/worker.py,sha256=IvcagHkXpMKjTvZl9svXko5hRuKN3czhbi5phGv-6No,46264
+xinference/core/worker.py,sha256=QhxVhpeKl-QYKA_77kUXTj5-rhodHAXlOhgtvqZiiRI,46329
 xinference/deploy/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/deploy/cmdline.py,sha256=YNXbPIT9zJIp5EQzl_rH5KwDDYqBd2CbaOVF8hA0lws,48120
 xinference/deploy/local.py,sha256=gcH6WfTxfhjvNkxxKZH3tcGtXV48BEPoaLWYztZHaeo,3954
@@ -42,16 +43,16 @@ xinference/deploy/test/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQ
 xinference/deploy/test/test_cmdline.py,sha256=m8xDzjtDuAJy0QkvYVJIZDuTB29cmYBV0d231JyRCPU,7714
 xinference/model/__init__.py,sha256=J7cdxnDxbr-4c8JREXWEIZ8MkB_VokXVaEOsI7ycjho,1101
 xinference/model/core.py,sha256=WQakLJgxrJMbTGn9AVaw-Tas7QL5M8cJsuYpGgH-io8,4706
-xinference/model/utils.py,sha256=j4qdq_R8dwDqwD9YLQkbFSHTmXKxWdZbr5MSvoL8aAw,11122
+xinference/model/utils.py,sha256=-axJ9I1IZ0li5Y2qzfFrsz8nMOGnoXXhmULu_e4aIKc,11196
 xinference/model/audio/__init__.py,sha256=G4n-MyzdarFVOndPRkEyZZrCwqFIG8yIsky6_5dife0,3433
-xinference/model/audio/chattts.py,sha256=rMH6-9M8boZdpUSgxaAge-LraE79nRs6mVc0nPLHd5A,4585
-xinference/model/audio/core.py,sha256=2QexrIh3hDoaNeWh5rOMas9q4zDCZTaazsLRdQ7D5Zw,6512
+xinference/model/audio/chattts.py,sha256=EXAfNATwblcilMjU3ff1dzTDJkjwQMXBo6zZ_517Jvo,4659
+xinference/model/audio/core.py,sha256=8rIyw0PLW2Py2-V7xsij4uQGGY39D_uq6uICuICobO8,6494
 xinference/model/audio/cosyvoice.py,sha256=Enur1Y4Xa-mpr7wwnoXWwhyh7PUAjrHZ8DV91tTrpjE,6426
 xinference/model/audio/custom.py,sha256=8GXBRmTtR-GY03-E91nlRGTIuabCRzlt20ecU6Un6Y8,4985
 xinference/model/audio/fish_speech.py,sha256=v2WVEV-BLWnbiDvqrx8WTGE_YNKmd9QoAF1LZBXWxn0,7310
 xinference/model/audio/funasr.py,sha256=65z7U7_F14CCP-jg6BpeY3_49FK7Y5OCRSzrhhsklCg,4075
-xinference/model/audio/model_spec.json,sha256=Ixo-15HVY2vu3_J5lElLL6texoJ41YwH-TBDB139NP8,4858
-xinference/model/audio/model_spec_modelscope.json,sha256=club_Pb1BdFPu5EOR5oVktsi2SiSrKYc7lHKsERjpds,1765
+xinference/model/audio/model_spec.json,sha256=rBfDYgiZNI0d1t01emx_UosEqap4JxD1OUJoocmlEMI,5120
+xinference/model/audio/model_spec_modelscope.json,sha256=Ul7_zy49N5zvio8-1WZEBFzZXS5_ueT49frWepMF8KY,2031
 xinference/model/audio/utils.py,sha256=pwo5cHh8nvhyBa9f-17QaVpXMSjmbpGbPYKwBBtEhGM,717
 xinference/model/audio/whisper.py,sha256=PQL7rebGC7WlIOItuDtjdEtSJtlhxFkolot-Fj-8uDU,7982
 xinference/model/embedding/__init__.py,sha256=1GmvQsbeeVUT-VRaRGetf8UT4RQgLWIzfp5kfX5jw-k,3567
@@ -67,22 +68,22 @@ xinference/model/flexible/launchers/__init__.py,sha256=X8w_2hKuQ9H3f90XYK7H_AQU4
 xinference/model/flexible/launchers/image_process_launcher.py,sha256=APbbHls0N9DpLFL6_qTexuc5o6bQAvdgJEAZWU4clyw,2510
 xinference/model/flexible/launchers/transformers_launcher.py,sha256=OZeeogDfopRUGhulP4PRJ4fZEJ2D9cfv7lcC2qJBoDE,2012
 xinference/model/image/__init__.py,sha256=80HBIbKh6lh-BgNaTo6k0TxxKjdG30bwHAdCiwVk6wk,3198
-xinference/model/image/core.py,sha256=ir1ns0qlUIlKnd0JS2cAJUppeEeczWYOnf6ecUCaLhM,8907
+xinference/model/image/core.py,sha256=qdqFMpPa2OSi0d5a4_iASEjL8s5vxxl1IRTJjmxfwO0,8959
 xinference/model/image/custom.py,sha256=5gjujQpJVTJ-pVB5LzBo4-bWKKOHzFlRaoRKJ_CuIUg,3769
 xinference/model/image/model_spec.json,sha256=JyXU-v4ysRT4yqwkmXgISY3uVWjeSiBLyH8fS7XO1_g,5368
 xinference/model/image/model_spec_modelscope.json,sha256=r3_m9XZo1QZgmASg5navOPs0ivlft5wVPF1SpbAVNBg,4266
-xinference/model/image/sdapi.py,sha256=XhSIfEQY8giC0KC04CoMBJea9dZSFO4Ci8fQlAlxk54,4685
+xinference/model/image/sdapi.py,sha256=Xgdtnvw4Xwj1Nc0cBoDo_ogH6E2mFJqLvX0jSxxgdnA,5936
 xinference/model/image/utils.py,sha256=gxg8jJ2nYaDknzCcSC53WCy1slbB5aWU14AbJbfm6Z4,906
 xinference/model/image/stable_diffusion/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/image/stable_diffusion/core.py,sha256=jObFn-DmB210OrzDj38H5GtC7yLXY8tJ37lyUlRELLo,19316
+xinference/model/image/stable_diffusion/core.py,sha256=NeCfkKjRdebgOOGMmg8DGVPR29D9vgX3V72qETgwPp0,24230
 xinference/model/llm/__init__.py,sha256=elINGzzDXmSissG32UMG7BfMqNgXwSM7USBaTorGWSA,12428
 xinference/model/llm/core.py,sha256=f4nKVPTAseivij6mbL2yXEKxrzllKm-_i2ttSqckTCg,8157
-xinference/model/llm/llm_family.json,sha256=qGr8F8T4deKsDGHN9LQxz7HM4CliPqsvh9guJf3yY2M,284592
+xinference/model/llm/llm_family.json,sha256=9peVrsgESrC-HPsIXS7wDcTsz_oNcGwrsw-Jh_yDSLU,285589
 xinference/model/llm/llm_family.py,sha256=eqeaHwLeS2TDB_ATf_h6YkH6OiyyF_4cSF_bOq3pTws,37432
 xinference/model/llm/llm_family_csghub.json,sha256=zMKWbihsxQNVB1u5iKJbZUkbOfQ4IPNq1KQ-8IDPQQA,8759
-xinference/model/llm/llm_family_modelscope.json,sha256=EanDUOC0GNWNDGOT0TQtk9TPjalpAfsUkNShM4qjZLs,214921
+xinference/model/llm/llm_family_modelscope.json,sha256=IcnRScrHkM4b6-rrpto4hCZ541rkUpdkaUa8wgSw8No,215985
 xinference/model/llm/memory.py,sha256=NEIMw6wWaF9S_bnBYq-EyuDhVbUEEeceQhwE1iwsrhI,10207
-xinference/model/llm/utils.py,sha256=boK0xMGbWFRX5qUQqPm1z1IfTZgBvFKOnWnqC-gcw7c,21909
+xinference/model/llm/utils.py,sha256=_FHJHZ9d1tYj4NwiG4TYftEp9L5vah6slUkqHKnn21U,23543
 xinference/model/llm/llama_cpp/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/llama_cpp/core.py,sha256=vjuTapwbn-ZjUX-8WA0nFyicE4UGUSehU_csSetvcZw,10928
 xinference/model/llm/lmdeploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -93,27 +94,27 @@ xinference/model/llm/sglang/__init__.py,sha256=-sjSIQ4K6w-TEzx49kVaWeWC443fnZqOD
 xinference/model/llm/sglang/core.py,sha256=ft4QlDw36gwoic8lyjtSx2ai6KTW84CPVbYr8grqGMI,16698
 xinference/model/llm/transformers/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/transformers/chatglm.py,sha256=pCJgoc0Ikny5BL85iHXl9M6zKgIzAHBsd81NAZ31yRI,17834
-xinference/model/llm/transformers/cogvlm2.py,sha256=wqWM6AhVQssgkUOcVX2N7RU7kjnAu55r6ZlUjh9iJro,15942
-xinference/model/llm/transformers/cogvlm2_video.py,sha256=dupPGQur8xGie5roA1ibpTIIZNoO-KMynvr7303pyl0,11809
+xinference/model/llm/transformers/cogvlm2.py,sha256=I5Ftm0VYjbTAv5ZARZCo32Ggpw58PJfHs5B_nX_BIlU,15972
+xinference/model/llm/transformers/cogvlm2_video.py,sha256=ZGkpC4x2uEtjwoMrLSODmAUYTjOeSNYxZi9VpQrpnhU,11857
 xinference/model/llm/transformers/compression.py,sha256=U0vMJ-JaBt4oC2LffgWg6HbPj1CeUi_YdwVbjDd0mRA,8112
-xinference/model/llm/transformers/core.py,sha256=-YmBzM5WbK-B3YIv7rWPFewDEx5xFDWyPO8YqFr8Sv8,31012
+xinference/model/llm/transformers/core.py,sha256=fyM7WXsVTEs38LAUPe-CJf45czInrNf1ifsWWexZ5nM,31128
 xinference/model/llm/transformers/deepseek_v2.py,sha256=HSddUBm5sKpHTNtPbTao9r3Yif-_xRAJrAtfCyELnhw,12975
-xinference/model/llm/transformers/deepseek_vl.py,sha256=eb-UH6g1Vr-jaZnGSkn_Ud4WYBzDuYR24CV_tAvo9iM,10397
-xinference/model/llm/transformers/glm4v.py,sha256=Uz7y2A_cSDeVVfe4D31h9LGqtUJ51plckt6jmnm_z7c,13841
-xinference/model/llm/transformers/intern_vl.py,sha256=3K0_2ng4zBgsnobzV7AfDEq7NzZu676JfNM54oE3AXQ,18222
+xinference/model/llm/transformers/deepseek_vl.py,sha256=pB6i6DW5oyfHdqTgKpi2DkIKVGlPLGIDR_Op0sB1uKA,10445
+xinference/model/llm/transformers/glm4v.py,sha256=goph2HhpV8gUm2t8-T1P-jTF2r_kPeH6QNe64lmlm0g,13871
+xinference/model/llm/transformers/intern_vl.py,sha256=0pbze1eo3HvNQ0nW-mVJcJuJ4GrEyBBqQAYIdXnAn6c,18270
 xinference/model/llm/transformers/internlm2.py,sha256=nRrmbH9bJl_wLcCH4zSy0EeKeP_ht-b8bVvbG2pMgV0,7228
-xinference/model/llm/transformers/minicpmv25.py,sha256=8fcmQo5VAst0vniV2-N6109Nq0sA56O2vWVxMvdZWxo,6766
-xinference/model/llm/transformers/minicpmv26.py,sha256=QRO5gnxuFMiJDNZ-v3os1A_4bn4fzrDvYjAOhdPQ4Lw,13392
-xinference/model/llm/transformers/omnilmm.py,sha256=MBsh-qaDnjtrtTRrAR7ArgHyupfpowwntuTuOj7xGkA,5124
-xinference/model/llm/transformers/qwen2_audio.py,sha256=tkLL523jdn1rVDfHV9RfLldDIISuMiukTJYt-h-dJ4o,5987
-xinference/model/llm/transformers/qwen2_vl.py,sha256=IMEdVJiHm3JccZg-vpSKCIElv8XtMrUPD3wT6yHel0A,8419
-xinference/model/llm/transformers/qwen_vl.py,sha256=JfMuiEqYuRIlDv5cIiRbLCd4DJQRgwCFoxc0JTJTGgs,14028
+xinference/model/llm/transformers/minicpmv25.py,sha256=mr80-OlSlK_opSuAO3cz_QlkqujLr6V-OsTP0ebwpE8,6814
+xinference/model/llm/transformers/minicpmv26.py,sha256=_e2C4vmyKIzKt7S7AvKgiqhDOhGiBXa6Xoiix4UaYtI,13440
+xinference/model/llm/transformers/omnilmm.py,sha256=2ZLW979ETqDDKo9CaTNwi9uLBZ2d6itHAYqjUA4jdro,5172
+xinference/model/llm/transformers/qwen2_audio.py,sha256=1XmlawVF-Xh2pgGoLDX7kOYIiF_bDUR3doSOnM59QbQ,6107
+xinference/model/llm/transformers/qwen2_vl.py,sha256=i8mypQwaPaaGQ0OIS55H8yuUX6gH87ubPuPQHHAD9fw,7304
+xinference/model/llm/transformers/qwen_vl.py,sha256=LG19qJW30bFiZOS-t9OM3JP6K1KCLj_Sv3nKSCLvyts,14058
 xinference/model/llm/transformers/tensorizer_utils.py,sha256=VXSYbPZtCbd8lVvsnjDLPZjfCMil67Pkywd_Ze4dTx4,11362
-xinference/model/llm/transformers/utils.py,sha256=qob4wDMN98LKzYdDcQe8rFVA5_mX4i5XeVgm3HSq9iI,28505
-xinference/model/llm/transformers/yi_vl.py,sha256=w4EpUHpmT9P1u5yEv1Pm3Ico92nqZZv3fO4NEKXteK4,8913
+xinference/model/llm/transformers/utils.py,sha256=kTaNK65igHoWRUe00FD-Bs7nBv_OYre0KXjbmstlleU,29228
+xinference/model/llm/transformers/yi_vl.py,sha256=iCdRLw-wizbU-qXXc8CT4DhC0Pt-uYg0vFwXEhAZjQg,8961
 xinference/model/llm/vllm/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/vllm/core.py,sha256=L9jAZ2mb3vq5f9ZkrQz9k2oc8mBb6kdIUmHYwofO2d4,28936
-xinference/model/llm/vllm/utils.py,sha256=JyztCDV7FT39QVraacg6T-JpmaSyfNwp9StRUBbvKJw,1347
+xinference/model/llm/vllm/core.py,sha256=FhwRaRY29imMS4Aldda7qBQg4tCtUqG8adr1zfUF7jw,30729
+xinference/model/llm/vllm/utils.py,sha256=LKOmwfFRrlSecawxT-uE39tC2RQbf1UIiSH9Uz90X6w,1313
 xinference/model/rerank/__init__.py,sha256=wRpf1bOMfmAsuEKEGczMTB5fWEvuqltlJbIbRb-x8Ko,3483
 xinference/model/rerank/core.py,sha256=ZOlbtK1x8CgcAAg0Y-5AF9ItYbhxuGtf7C_Sf0D9Kww,14122
 xinference/model/rerank/custom.py,sha256=wPKF3bHbGap9dHz9yYvXMXhozh4hRzS78RQijqvaRq8,3846
@@ -121,7 +122,7 @@ xinference/model/rerank/model_spec.json,sha256=xUuJgJ8Ad4l2v8gEHxAdF_xoaSkA8j8AX
 xinference/model/rerank/model_spec_modelscope.json,sha256=pf5hX4g0HdVjk2-ibHTl_mXHgQSSPYMTBOIwvnwMMkk,1616
 xinference/model/rerank/utils.py,sha256=MJAFL47G3r3zLVGXKoi0QLTgU3Xr4Ffv72Ipn--psew,713
 xinference/model/video/__init__.py,sha256=mRhOhzMxzcPFdA5j4niAxH_j9dXLtT9HmchuICrdET8,2160
-xinference/model/video/core.py,sha256=PMqyWhhBWO77VjpEvTC7EQrGmyLWxJ_-Mm1VRqb2dNY,6031
+xinference/model/video/core.py,sha256=QEdVbVBDQebSWxqkL483Q2Y9Y1GGc2an0gi2QBPUH9I,6013
 xinference/model/video/diffusers.py,sha256=kSEBRf0vtWyo0IrwoiEpr_ROu7SwDAVBZ4leqkcPycM,6244
 xinference/model/video/model_spec.json,sha256=yQcLSU3vRJys-ACdHGtTNdz2pX1O9QDQ5rGHQd9LdFY,817
 xinference/model/video/model_spec_modelscope.json,sha256=U8p6IqNLbY5Safxwpa6dCfnGbyvOC4FtYIf2ucr8TvM,815
@@ -15507,9 +15508,9 @@ xinference/web/ui/node_modules/yargs-parser/package.json,sha256=BSwbOzgetKXMK4u0
 xinference/web/ui/node_modules/yocto-queue/package.json,sha256=6U1XHQPGXJTqsiFvT953ORihUtXTblZy4fXBWP9qxC0,725
 xinference/web/ui/node_modules/yup/package.json,sha256=xRFSROB9NKxqSWHEVFvSTsPs9Ll074uo8OS1zEw0qhA,1206
 xinference/web/ui/node_modules/yup/node_modules/type-fest/package.json,sha256=JTv2zTTVgxQ2H82m1-6qEpdMv08lHjFx4Puf_MsbB_Q,1134
-xinference-0.15.3.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-0.15.3.dist-info/METADATA,sha256=NSmJnHU0w8BD5kyJ0JusOYOdX0sfwerPDyb1PPutadc,19126
-xinference-0.15.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-xinference-0.15.3.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-0.15.3.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-0.15.3.dist-info/RECORD,,
+xinference-0.15.4.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-0.15.4.dist-info/METADATA,sha256=PANLzbQqqFtpqMFBVCP9JcY30fhoC63e5dG3Y5hbbr4,19136
+xinference-0.15.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+xinference-0.15.4.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-0.15.4.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-0.15.4.dist-info/RECORD,,