xinference 1.2.1__py3-none-any.whl → 1.2.2__py3-none-any.whl

xinference/model/llm/utils.py

@@ -11,16 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import base64
 import functools
 import json
 import logging
 import os
+import re
 import time
 import typing
 import uuid
 from io import BytesIO
-from typing import AsyncGenerator, Dict, Iterator, List, Optional, Tuple, cast
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    cast,
+)
 
 import requests
 from PIL import Image
@@ -64,6 +76,18 @@ LLAMA3_TOOL_CALL_FAMILY = [
     "llama-3.1-instruct",
 ]
 
+DEEPSEEK_TOOL_CALL_FAMILY = [
+    "deepseek-r1-distill-qwen",
+    "deepseek-r1-distill-llama",
+]
+
+TOOL_CALL_FAMILY = (
+    QWEN_TOOL_CALL_FAMILY
+    + GLM4_TOOL_CALL_FAMILY
+    + LLAMA3_TOOL_CALL_FAMILY
+    + DEEPSEEK_TOOL_CALL_FAMILY
+)
+
 QWEN_TOOL_CALL_SYMBOLS = ["<tool_call>", "</tool_call>"]
 
 
@@ -308,6 +332,35 @@ class ChatModelMixin:
             else:
                 yield cls._to_chat_completion_chunk(chunk)
 
+    @classmethod
+    def _tools_to_messages_for_deepseek(
+        cls, messages: List[dict], tools: Iterable[dict]
+    ):
+        # deepseek integrates tool calls into messages
+        # we follow the chat template rule to integrate tools into messages
+        tool_call_message: Dict[str, Any] = {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [],
+        }
+
+        for tool in tools:
+            function_name = tool["function"]["name"]
+            parameters = tool["function"].get("parameters", {}).get("properties", {})
+            function_args_json = json.dumps(parameters)
+
+            tool_call_message["tool_calls"].append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": function_name,
+                        "arguments": function_args_json,
+                    },
+                }
+            )
+
+        messages.append(tool_call_message)
+
     @classmethod
     async def _async_to_chat_completion_chunks(
         cls,
@@ -401,6 +454,61 @@ class ChatModelMixin:
         except Exception:
             return [(text, None, None)]
 
+    @classmethod
+    def _eval_deepseek_chat_arguments(cls, c) -> List[Tuple]:
+        """
+        Parses tool calls from deepseek-r1 format and removes duplicates.
+
+        Returns:
+        List[Tuple[Optional[str], Optional[str], Optional[dict]]]
+        - (None, function_name, arguments) if successfully parsed.
+        - (content, None, None) if parsing failed (content is raw JSON text).
+
+        Example input:
+        <｜tool▁call｜>get_current_weather
+        ```json
+        {"location": "tokyo", "unit": "fahrenheit"}
+        ```
+
+        Output:
+        [
+            (None, "get_current_weather", {"location": "tokyo", "unit": "fahrenheit"})
+        ]
+        """
+
+        text = c["choices"][0]["text"]
+
+        pattern = r"<｜tool▁call｜>(\w+)\s*```json\s*(.*?)\s*```"
+        matches = re.findall(pattern, text, re.DOTALL)
+
+        if not matches:
+            return [(text, None, None)]
+
+        tool_calls = set()  # Used for deduplication
+        results = []
+
+        for function_name, args_json in matches:
+            try:
+                arguments = json.loads(args_json)
+                # Convert dictionary to frozenset for deduplication
+                arguments_hashable = frozenset(arguments.items())
+                tool_call_tuple = (None, function_name, arguments)
+            except json.JSONDecodeError:
+                tool_call_tuple = (
+                    args_json,
+                    None,
+                    None,
+                )  # If parsing fails, treat as raw content
+                arguments_hashable = None  # No need for hashing
+
+            # Avoid duplicate entries
+            dedup_key = (function_name, arguments_hashable)
+            if dedup_key not in tool_calls:
+                tool_calls.add(dedup_key)
+                results.append(tool_call_tuple)
+
+        return results
+
     @classmethod
     def _eval_tool_arguments(cls, model_family, c):
         family = model_family.model_family or model_family.model_name
@@ -410,6 +518,8 @@ class ChatModelMixin:
             result = cls._eval_qwen_chat_arguments(c)
         elif family in LLAMA3_TOOL_CALL_FAMILY:
             result = cls._eval_llama3_chat_arguments(c)
+        elif family in DEEPSEEK_TOOL_CALL_FAMILY:
+            result = cls._eval_deepseek_chat_arguments(c)
         else:
             raise Exception(
                 f"Model {model_family.model_name} is not support tool calls."

xinference/model/llm/vllm/core.py

@@ -44,6 +44,7 @@ from ....types import (
 from .. import LLM, LLMFamilyV1, LLMSpecV1
 from ..llm_family import CustomLLMFamilyV1
 from ..utils import (
+    DEEPSEEK_TOOL_CALL_FAMILY,
     QWEN_TOOL_CALL_FAMILY,
     QWEN_TOOL_CALL_SYMBOLS,
     ChatModelMixin,
@@ -185,6 +186,7 @@ if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.3-instruct")
+    VLLM_SUPPORTED_CHAT_MODELS.append("deepseek-r1-distill-llama")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
@@ -198,6 +200,12 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("QvQ-72B-Preview")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.7.0":
+    VLLM_SUPPORTED_CHAT_MODELS.append("internlm3-instruct")
+
+if VLLM_INSTALLED and vllm.__version__ >= "0.7.2":
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2.5-vl-instruct")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -807,8 +815,11 @@ class VLLMChatModel(VLLMModel, ChatModelMixin):
         tools = generate_config.pop("tools", []) if generate_config else None
         model_family = self.model_family.model_family or self.model_family.model_name
         full_context_kwargs = {}
-        if tools and model_family in QWEN_TOOL_CALL_FAMILY:
-            full_context_kwargs["tools"] = tools
+        if tools:
+            if model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            elif model_family in DEEPSEEK_TOOL_CALL_FAMILY:
+                self._tools_to_messages_for_deepseek(messages, tools)
         assert self.model_family.chat_template is not None
         full_prompt = self.get_full_context(
             messages, self.model_family.chat_template, **full_context_kwargs

{xinference-1.2.1.dist-info → xinference-1.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xinference
-Version: 1.2.1
+Version: 1.2.2
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye
@@ -98,7 +98,7 @@ Requires-Dist: tomli; extra == "all"
 Requires-Dist: vocos; extra == "all"
 Requires-Dist: jieba; extra == "all"
 Requires-Dist: soundfile; extra == "all"
-Requires-Dist: qwen-vl-utils; extra == "all"
+Requires-Dist: qwen-vl-utils!=0.0.9; extra == "all"
 Requires-Dist: datamodel-code-generator; extra == "all"
 Requires-Dist: jsonschema; extra == "all"
 Requires-Dist: verovio>=4.3.1; extra == "all"
@@ -154,6 +154,7 @@ Requires-Dist: fugashi; extra == "audio"
 Requires-Dist: g2p-en; extra == "audio"
 Requires-Dist: anyascii; extra == "audio"
 Requires-Dist: gruut[de,es,fr]; extra == "audio"
+Requires-Dist: kokoro>=0.7.9; extra == "audio"
 Requires-Dist: nemo-text-processing<1.1.0; sys_platform == "linux" and extra == "audio"
 Requires-Dist: WeTextProcessing<1.0.4; sys_platform == "linux" and extra == "audio"
 Provides-Extra: benchmark
@@ -211,7 +212,7 @@ Requires-Dist: mlx-lm>=0.21.1; extra == "mlx"
 Requires-Dist: mlx-vlm>=0.1.11; extra == "mlx"
 Requires-Dist: mlx-whisper; extra == "mlx"
 Requires-Dist: f5-tts-mlx; extra == "mlx"
-Requires-Dist: qwen-vl-utils; extra == "mlx"
+Requires-Dist: qwen-vl-utils!=0.0.9; extra == "mlx"
 Requires-Dist: tomli; extra == "mlx"
 Provides-Extra: rerank
 Requires-Dist: FlagEmbedding; extra == "rerank"
@@ -225,7 +226,6 @@ Requires-Dist: torch; extra == "transformers"
 Requires-Dist: accelerate>=0.28.0; extra == "transformers"
 Requires-Dist: sentencepiece; extra == "transformers"
 Requires-Dist: transformers-stream-generator; extra == "transformers"
-Requires-Dist: bitsandbytes; extra == "transformers"
 Requires-Dist: protobuf; extra == "transformers"
 Requires-Dist: einops; extra == "transformers"
 Requires-Dist: tiktoken; extra == "transformers"
@@ -236,11 +236,12 @@ Requires-Dist: torchvision; extra == "transformers"
 Requires-Dist: peft; extra == "transformers"
 Requires-Dist: eva-decord; extra == "transformers"
 Requires-Dist: jj-pytorchvideo; extra == "transformers"
-Requires-Dist: qwen-vl-utils; extra == "transformers"
+Requires-Dist: qwen-vl-utils!=0.0.9; extra == "transformers"
 Requires-Dist: datamodel-code-generator; extra == "transformers"
 Requires-Dist: jsonschema; extra == "transformers"
 Requires-Dist: auto-gptq; sys_platform != "darwin" and extra == "transformers"
 Requires-Dist: autoawq<0.2.6; sys_platform != "darwin" and extra == "transformers"
+Requires-Dist: bitsandbytes; sys_platform == "linux" and extra == "transformers"
 Provides-Extra: video
 Requires-Dist: diffusers>=0.32.0; extra == "video"
 Requires-Dist: imageio-ffmpeg; extra == "video"
@@ -296,14 +297,14 @@ potential of cutting-edge AI models.
 - Support speech recognition model: [#929](https://github.com/xorbitsai/inference/pull/929)
 - Metrics support: [#906](https://github.com/xorbitsai/inference/pull/906)
 ### New Models
+- Built-in support for [DeepSeek-R1-Distill-Qwen](https://github.com/deepseek-ai/DeepSeek-R1?tab=readme-ov-file#deepseek-r1-distill-models): [#2781](https://github.com/xorbitsai/inference/pull/2781)
+- Built-in support for [qwen2.5-vl](https://github.com/QwenLM/Qwen2.5-VL): [#2788](https://github.com/xorbitsai/inference/pull/2788)
+- Built-in support for [internlm3-instruct](https://github.com/InternLM/InternLM): [#2789](https://github.com/xorbitsai/inference/pull/2789)
 - Built-in support for [MeloTTS](https://github.com/myshell-ai/MeloTTS): [#2760](https://github.com/xorbitsai/inference/pull/2760)
 - Built-in support for [CogAgent](https://github.com/THUDM/CogAgent): [#2740](https://github.com/xorbitsai/inference/pull/2740)
 - Built-in support for [HunyuanVideo](https://github.com/Tencent/HunyuanVideo): [#2721](https://github.com/xorbitsai/inference/pull/2721)
 - Built-in support for [HunyuanDiT](https://github.com/Tencent/HunyuanDiT): [#2727](https://github.com/xorbitsai/inference/pull/2727)
 - Built-in support for [Macro-o1](https://github.com/AIDC-AI/Marco-o1): [#2749](https://github.com/xorbitsai/inference/pull/2749)
-- Built-in support for [Stable Diffusion 3.5](https://huggingface.co/collections/stabilityai/stable-diffusion-35-671785cca799084f71fa2838): [#2706](https://github.com/xorbitsai/inference/pull/2706)
-- Built-in support for [CosyVoice 2](https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B): [#2684](https://github.com/xorbitsai/inference/pull/2684)
-- Built-in support for [Fish Speech V1.5](https://huggingface.co/fishaudio/fish-speech-1.5): [#2672](https://github.com/xorbitsai/inference/pull/2672)
 ### Integrations
 - [Dify](https://docs.dify.ai/advanced/model-configuration/xinference): an LLMOps platform that enables developers (and even non-developers) to quickly build useful applications based on large language models, ensuring they are visual, operable, and improvable.
 - [FastGPT](https://github.com/labring/FastGPT): a knowledge-based platform built on the LLM, offers out-of-the-box data processing and model invocation capabilities, allows for workflow orchestration through Flow visualization.

{xinference-1.2.1.dist-info → xinference-1.2.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 xinference/__init__.py,sha256=nmTTrYbIpj964ZF6ojtgOM7E85JBOj1EyQbmYjbj1jw,915
 xinference/_compat.py,sha256=URSJQLXrcsTO9B_4x0wVDPijYQDhuVJmZ95npID560w,4197
-xinference/_version.py,sha256=cRjgwmUs9T7LZabp8_lbu3KxEJZrH641zBpN4hZkBEY,497
+xinference/_version.py,sha256=tyoMIoWH7JrsR3odtmfwgrhewV2IdEEE1IRf42ATx28,497
 xinference/conftest.py,sha256=vETDpRBVIlWbWi7OTwf7og89U25KyYGyI7yPIB3O8N8,9564
 xinference/constants.py,sha256=mEW4HDzjXtDXN61Mt6TtJrJ4ljbB6VUkh97e3oDbNx4,3905
 xinference/device_utils.py,sha256=zswJiws3VyTIaNO8z-MOcsJH_UiPoePPiKK5zoNrjTA,3285
@@ -9,14 +9,14 @@ xinference/isolation.py,sha256=gTU1em5fxg1m-7hxieWBMZvVkXZX4GZYmeT7XxXsYrU,2699
 xinference/types.py,sha256=p9IV2vmbtFfpo9P66IynkEgU2N6310uVMKBuq0rbtNc,13102
 xinference/utils.py,sha256=zYgf9bCvfbybRt3gEog6r5WJCpj0czZCf0qgRdYjkN8,720
 xinference/api/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/api/restful_api.py,sha256=NlWmA1pDu8oPybI9daU4pplQXkRD-NExTO2L_E4Y2zM,92876
+xinference/api/restful_api.py,sha256=XENr9892-WHZ1XKaCwijdTAAd1uX2D0oMKUF6Pw99GQ,92730
 xinference/api/oauth2/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/api/oauth2/auth_service.py,sha256=74JzB42fbbmBu4Q1dW3A9Fp_N7167KgRGB42Z0NHjAM,6119
 xinference/api/oauth2/types.py,sha256=K923sv_XySIUtM2Eozl9IG082IJcDOS5SFLrPZ5ELBg,996
 xinference/api/oauth2/utils.py,sha256=SIiiUj6VuTEsj3bZ2TYUyhx3cGlLSX3ZNWDOgUwRtXc,1410
 xinference/client/__init__.py,sha256=Gc4HOzAy_1cic5kXlso7hahYgw89CKvZSJDicEU461k,669
 xinference/client/common.py,sha256=iciZRs5YjM2gYsXnwACPMaiBZp4_XpawWwfym0Iyu40,1617
-xinference/client/handlers.py,sha256=OKl_i5FA341wsQf_0onSOPbbW6V861WJrSP7ghtDc8c,527
+xinference/client/handlers.py,sha256=HYmQjG0BPK5PKcWVEDJqQz5iLjS8yjOq5j-HP_CAi_k,630
 xinference/client/restful/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/client/restful/restful_client.py,sha256=JwzP7etUZBR0mmU7y3dUOEWN_D7ol_2hXN9KMAzKZaw,53601
 xinference/core/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
@@ -28,14 +28,14 @@ xinference/core/metrics.py,sha256=ScmTG15Uq3h_ob72ybZSMWdnk8P4sUZFcm60f4ikSXc,26
 xinference/core/model.py,sha256=8FaEJK4tTL7wvYivLEqWKkZOrAMODBSoxg9A4fYzUH0,43365
 xinference/core/progress_tracker.py,sha256=LIF6CLIlnEoSBkuDCraJktDOzZ31mQ4HOo6EVr3KpQM,6453
 xinference/core/resource.py,sha256=FQ0aRt3T4ZQo0P6CZZf5QUKHiCsr5llBvKb1f7wfnxg,1611
-xinference/core/scheduler.py,sha256=gdj3SyP_jelJ86vTRrgnFynhxz5JSwLRsQgx8PTtBi8,15671
+xinference/core/scheduler.py,sha256=WHWOlbzAPoURXNzjFNdfvhkgBYO8ufZtIu3JXfq7fmY,15576
 xinference/core/status_guard.py,sha256=4an1KjUOhCStgRQUw1VSzXcycXUtvhxwiMREKKcl1UI,2828
 xinference/core/supervisor.py,sha256=4beE9EZx5xWpSLZtZwT3CPYWT25zPFl9gRj2oK8Za3w,60208
 xinference/core/utils.py,sha256=RR3XDioh52Wy8vmAqhCZ6EQskr-HPDfUp0vCCAPIAvs,11302
 xinference/core/worker.py,sha256=FVy_zXOHovYLPQQr6DSpjSGWaLyH425gTUUH_vwKVfk,50666
 xinference/deploy/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/deploy/cmdline.py,sha256=gPwp9IngaXCypUEnPDS_22U8GntsKr7qHDST7duyAoI,48478
-xinference/deploy/local.py,sha256=gcH6WfTxfhjvNkxxKZH3tcGtXV48BEPoaLWYztZHaeo,3954
+xinference/deploy/local.py,sha256=sO3BcpEH9oCF87CxWVA4AXAYcfHGnrcop40ew5NOA2g,3979
 xinference/deploy/supervisor.py,sha256=68rB2Ey5KFeF6zto9YGbw3P8QLZmF_KSh1NwH_pNP4w,2986
 xinference/deploy/utils.py,sha256=jdL7i2WV6u_BZ8IiE1d3YktvCARcB3ntzMQ5rHGD5DM,6756
 xinference/deploy/worker.py,sha256=VQ71ClWpeGsyFgDmcOes2ub1cil10cBjhFLHYeuVwC4,2974
@@ -46,16 +46,17 @@ xinference/model/core.py,sha256=_NEH4wkjjJgRDdLHNVY_hL3V0kT67CvTay89uIzx1Ns,4736
 xinference/model/utils.py,sha256=_yJ5h4RUzt7Kjs2WdjSzbVM3FTWEkX0ycOnXANZ9KVg,11394
 xinference/model/audio/__init__.py,sha256=KasWsaNPeij6sGpHKqXaUc_bxUw1yYbD7-fwxkcoAVE,3731
 xinference/model/audio/chattts.py,sha256=ny3DZTCTt2MzdkLw994_QHZ_4qIEUZcNexNJkCejCyo,4998
-xinference/model/audio/core.py,sha256=jboohBIr8GmXH4ASoKJvU0lWJzsCqUszTsJdn-pByaI,7517
+xinference/model/audio/core.py,sha256=AAuss7fGwGktcB_aGRwuR4TJIM6-Rrm0YCU752QoxfQ,7710
 xinference/model/audio/cosyvoice.py,sha256=vw7OR2t7zNimQn3Q74iiL1-2en5o6gvdcZsDgkmYpy4,7796
 xinference/model/audio/custom.py,sha256=8GXBRmTtR-GY03-E91nlRGTIuabCRzlt20ecU6Un6Y8,4985
 xinference/model/audio/f5tts.py,sha256=RyfEYVvKhV7JygIv4F45C8wBr-u0mx9WpXj3gIOtL7o,6809
 xinference/model/audio/f5tts_mlx.py,sha256=SbYIo1C3EfZ8L30P7OTx0Dx7KuXIIpQKf8uZqR1e760,8527
 xinference/model/audio/fish_speech.py,sha256=U1NtEhQFtzVYZ0vpx10EqBnqUz-hmx1ZTAzD9OSPskQ,6767
 xinference/model/audio/funasr.py,sha256=65z7U7_F14CCP-jg6BpeY3_49FK7Y5OCRSzrhhsklCg,4075
+xinference/model/audio/kokoro.py,sha256=7n2l0MWRvTz6uI4rtf-22MlrB6-HivmG8rp-JKdJCgY,4063
 xinference/model/audio/melotts.py,sha256=lJdOsce2mMOTQIslv6xq1JTEO7vC6Q9ga2xjY-_E7uw,3493
-xinference/model/audio/model_spec.json,sha256=rGe6_DBmit4CUCFv1y8r9qHFgiFXJePzeaIWhEyyYaE,10239
-xinference/model/audio/model_spec_modelscope.json,sha256=q4LKNjZ1ZG0hI-eWhLj9FWNcAVqiaLtsc6_SCrLkIVk,2788
+xinference/model/audio/model_spec.json,sha256=VbeNoQIWW1qe9TOEgU_m9wlfaaDjE_iKAMMhLs-kCqM,10477
+xinference/model/audio/model_spec_modelscope.json,sha256=aFCqDC74wlisqhWzt42k0vE42b8eshfz2YXhJf95nNQ,3029
 xinference/model/audio/utils.py,sha256=fnnQfZlhH6DRw6beXPUIO28u6qu2GgCONrWDIsTCNkw,1591
 xinference/model/audio/whisper.py,sha256=NePQOYy2CnVqi7g6uY9hq5tlUIxZ995FOIOPsPZCfJ8,9058
 xinference/model/audio/whisper_mlx.py,sha256=zBuCd7GUlsN9FC_-J11gqIkOCicihfbqxoabiXTvH2Q,7237
@@ -87,19 +88,19 @@ xinference/model/image/stable_diffusion/core.py,sha256=SaeBYNdRdqarEwoyjHDz_SfRI
 xinference/model/image/stable_diffusion/mlx.py,sha256=GZsozzGB04NfHAdU9MI6gwWE1t_A-s_Ddn_ic8DlkKQ,7476
 xinference/model/llm/__init__.py,sha256=zKmkoNsUoQk1_jnLHNpno4UPKGl5O_vJK5R5R-TYF9w,14140
 xinference/model/llm/core.py,sha256=g-luuAjZizrPunhyFE9IRjn57l0g6FY_1xUwtlRegbs,8151
-xinference/model/llm/llm_family.json,sha256=Q67g29KCIgnRQVS92ortD_RY1RhYU_47WnwFMM8GDbY,328868
-xinference/model/llm/llm_family.py,sha256=HiWobT1SInGXWT78g673K5-FoDkDi1dSUU4pmfFAPPI,39050
+xinference/model/llm/llm_family.json,sha256=6GXYmnqKZAh2dwnaDyg1tzk583hRLy33KAI4B5rDQc8,339783
+xinference/model/llm/llm_family.py,sha256=tee7rQYayo46-lK3QhrZqoGKYQM5ijA5tb6xQTPGZa0,39125
 xinference/model/llm/llm_family_csghub.json,sha256=zMKWbihsxQNVB1u5iKJbZUkbOfQ4IPNq1KQ-8IDPQQA,8759
-xinference/model/llm/llm_family_modelscope.json,sha256=jEp2-wzMsJYTR4xXqt5eETghsxgYgEGLiDSzc12pUzY,258710
+xinference/model/llm/llm_family_modelscope.json,sha256=-JAca-vryl6svsL1Nk4gd3eq-03BFJ8KfdS3q0g7A-U,269784
 xinference/model/llm/llm_family_openmind_hub.json,sha256=jl9pfbe5DztoxgEwKBxDk1Wd7TziTiJ48_Ie_lJdYjA,67872
 xinference/model/llm/memory.py,sha256=GLNmXBI-AtMbuaJfEf50fnhN4rdbOZjLyT6L_Vjqa5g,10206
-xinference/model/llm/utils.py,sha256=nSI5vtOp3qfAB-RgYXNrMZH9f-X1JbNOyCBVFz21eMw,25407
+xinference/model/llm/utils.py,sha256=AUm_q51rmV8HgVMcRttd8B8TA-EXxU1oJVdkLYYGhes,28641
 xinference/model/llm/llama_cpp/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/llama_cpp/core.py,sha256=vjuTapwbn-ZjUX-8WA0nFyicE4UGUSehU_csSetvcZw,10928
+xinference/model/llm/llama_cpp/core.py,sha256=3GSGk42c8Oy_jTqRv4nLC482V2tUis3V0LlohQy_I1U,11312
 xinference/model/llm/lmdeploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xinference/model/llm/lmdeploy/core.py,sha256=WvSP3x6t-HBv6hKh1qWZatFAzlcZCyyKqvc3ua8yPTI,19835
 xinference/model/llm/mlx/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/mlx/core.py,sha256=rpgwZ7R2WDd-ayhi28UDglGxIutBuIXLyX7n7fHIEYw,22936
+xinference/model/llm/mlx/core.py,sha256=WQN2iURiWSL_MY5hR0GkCYa08qr5wtOFx522_c2vW30,23130
 xinference/model/llm/sglang/__init__.py,sha256=-sjSIQ4K6w-TEzx49kVaWeWC443fnZqODU91GCQ_JNo,581
 xinference/model/llm/sglang/core.py,sha256=Ab0i6Q3M-DqQi5bHMyfa9klPElGSk1ThEke4mdsBHXU,16747
 xinference/model/llm/transformers/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
@@ -108,7 +109,7 @@ xinference/model/llm/transformers/cogagent.py,sha256=JbIiqVW-S9MA3d4CN2DlI7-WDB_
 xinference/model/llm/transformers/cogvlm2.py,sha256=I5Ftm0VYjbTAv5ZARZCo32Ggpw58PJfHs5B_nX_BIlU,15972
 xinference/model/llm/transformers/cogvlm2_video.py,sha256=ZGkpC4x2uEtjwoMrLSODmAUYTjOeSNYxZi9VpQrpnhU,11857
 xinference/model/llm/transformers/compression.py,sha256=U0vMJ-JaBt4oC2LffgWg6HbPj1CeUi_YdwVbjDd0mRA,8112
-xinference/model/llm/transformers/core.py,sha256=t1tujy8-IfD83CjKkvxZ-jqZISDQVFJpSiDkeIL4LGM,28286
+xinference/model/llm/transformers/core.py,sha256=7VkOA04mpiuyA8-P62NSC74_WkKLFb_O51hIEtUHPBw,28493
 xinference/model/llm/transformers/deepseek_v2.py,sha256=-RKlI3mhja730md4evQ2vfIxBnZD5vWyrgmg_3eovms,4096
 xinference/model/llm/transformers/deepseek_vl.py,sha256=pB6i6DW5oyfHdqTgKpi2DkIKVGlPLGIDR_Op0sB1uKA,10445
 xinference/model/llm/transformers/glm4v.py,sha256=goph2HhpV8gUm2t8-T1P-jTF2r_kPeH6QNe64lmlm0g,13871
@@ -119,14 +120,14 @@ xinference/model/llm/transformers/minicpmv25.py,sha256=mr80-OlSlK_opSuAO3cz_Qlkq
 xinference/model/llm/transformers/minicpmv26.py,sha256=_e2C4vmyKIzKt7S7AvKgiqhDOhGiBXa6Xoiix4UaYtI,13440
 xinference/model/llm/transformers/omnilmm.py,sha256=2ZLW979ETqDDKo9CaTNwi9uLBZ2d6itHAYqjUA4jdro,5172
 xinference/model/llm/transformers/opt.py,sha256=dkZFNwtw_sUuVaz9He6LWfEojRGfOQFQ5atvC5OYPuY,2429
-xinference/model/llm/transformers/qwen2_audio.py,sha256=1XmlawVF-Xh2pgGoLDX7kOYIiF_bDUR3doSOnM59QbQ,6107
-xinference/model/llm/transformers/qwen2_vl.py,sha256=TKn7p0-bNzzv7ZVZ18mOD9NYgJ9q_y_CJMA3OWWvv2c,7768
+xinference/model/llm/transformers/qwen2_audio.py,sha256=RkKSUFTgyolVSuFoLj2GdzsFfU8goOIYtDFLoRLiZ2s,6259
+xinference/model/llm/transformers/qwen2_vl.py,sha256=HEtxPJyhfsvcULu8guLwasB0DaXRgOGxP1WrhgDUWec,8437
 xinference/model/llm/transformers/qwen_vl.py,sha256=LG19qJW30bFiZOS-t9OM3JP6K1KCLj_Sv3nKSCLvyts,14058
 xinference/model/llm/transformers/tensorizer_utils.py,sha256=VXSYbPZtCbd8lVvsnjDLPZjfCMil67Pkywd_Ze4dTx4,11362
-xinference/model/llm/transformers/utils.py,sha256=a4-X5P9_L--rgSx5jI8haYA6GSpKhMdOYE97VNh54yM,19389
+xinference/model/llm/transformers/utils.py,sha256=GHJsjBjEXpzZAFbcfKiMxMjFnOBYeZt-eXF5S4HQ8I8,19582
 xinference/model/llm/transformers/yi_vl.py,sha256=iCdRLw-wizbU-qXXc8CT4DhC0Pt-uYg0vFwXEhAZjQg,8961
 xinference/model/llm/vllm/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/vllm/core.py,sha256=hrHFC3Q3J-MLJnyqX9HcdiIGBfxfrn_27UPsjldLSHc,37233
+xinference/model/llm/vllm/core.py,sha256=p6s5qsz9564yOfkD6nG2dISenwrRAtYUaGOe5plHs04,37708
 xinference/model/llm/vllm/utils.py,sha256=LKOmwfFRrlSecawxT-uE39tC2RQbf1UIiSH9Uz90X6w,1313
 xinference/model/llm/vllm/xavier/__init__.py,sha256=CyLLkbImZouAk4lePIgKXT4WQoqyauIEwdqea5IOUVU,581
 xinference/model/llm/vllm/xavier/allocator.py,sha256=SJ2eCOxF6CWTBZIP39FRxeK6fxIE8pRshOPnSRc72d4,2691
@@ -15718,9 +15719,9 @@ xinference/web/ui/node_modules/yup/package.json,sha256=xRFSROB9NKxqSWHEVFvSTsPs9
 xinference/web/ui/node_modules/yup/node_modules/type-fest/package.json,sha256=JTv2zTTVgxQ2H82m1-6qEpdMv08lHjFx4Puf_MsbB_Q,1134
 xinference/web/ui/src/locales/en.json,sha256=MahpAAKmZPqtK5-M_kwdI9IUbBP-GcNqI0jSTVXHEE8,8169
 xinference/web/ui/src/locales/zh.json,sha256=9-Hu72a9FSB1ZCUMkKDzopBTh7Aer6b-3PB62cYxsOg,7933
-xinference-1.2.1.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-1.2.1.dist-info/METADATA,sha256=NUwKoNtMeFa3TkRJMqGYa7E-HLCZqOxBysi6gNHY-c4,23919
-xinference-1.2.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-xinference-1.2.1.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-1.2.1.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-1.2.1.dist-info/RECORD,,
+xinference-1.2.2.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-1.2.2.dist-info/METADATA,sha256=IKf5udmeYA8AAAPMHGwry4YSehJPGvxp30AKB57P80Y,23991
+xinference-1.2.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+xinference-1.2.2.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-1.2.2.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-1.2.2.dist-info/RECORD,,