xinference 0.11.2__py3-none-any.whl → 0.11.3__py3-none-any.whl

xinference/model/llm/pytorch/baichuan.py

@@ -73,7 +73,8 @@ class BaichuanPytorchChatModel(PytorchChatModel):
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False
-        if llm_family.model_name not in ["baichuan-chat", "baichuan-2-chat"]:
+        model_family = llm_family.model_family or llm_family.model_name
+        if model_family not in ["baichuan-chat", "baichuan-2-chat"]:
             return False
         if "chat" not in llm_family.model_ability:
             return False

xinference/model/llm/pytorch/cogvlm2.py

@@ -0,0 +1,257 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import base64
+import logging
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from io import BytesIO
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+import requests
+import torch
+from PIL import Image
+
+from ....model.utils import select_device
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionUsage,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from .core import PytorchChatModel, PytorchGenerateConfig
+
+logger = logging.getLogger(__name__)
+
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
+IMAGENET_STD = (0.229, 0.224, 0.225)
+
+
+class CogVLM2Model(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._torch_type = None
+        self._device = None
+        self._tokenizer = None
+        self._model = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        family = model_family.model_family or model_family.model_name
+        if "cogvlm" in family.lower():
+            return True
+        return False
+
+    def load(self, **kwargs):
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        from transformers.generation import GenerationConfig
+
+        device = self._pytorch_model_config.get("device", "auto")
+        self._device = select_device(device)
+        self._torch_type = (
+            torch.bfloat16
+            if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8
+            else torch.float16
+        )
+
+        self._tokenizer = AutoTokenizer.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+        )
+
+        self._model = AutoModelForCausalLM.from_pretrained(
+            self.model_path,
+            torch_dtype=self._torch_type,
+            trust_remote_code=True,
+            low_cpu_mem_usage=True,
+            device_map="auto",
+        ).eval()
+
+        # Specify hyperparameters for generation
+        self._model.generation_config = GenerationConfig.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+        )
+
+    def _message_content_to_cogvlm2(self, content):
+        def _load_image(_url):
+            if _url.startswith("data:"):
+                logging.info("Parse url by base64 decoder.")
+                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+                # e.g. f"data:image/jpeg;base64,{base64_image}"
+                _type, data = _url.split(";")
+                _, ext = _type.split("/")
+                data = data[len("base64,") :]
+                data = base64.b64decode(data.encode("utf-8"))
+                return Image.open(BytesIO(data)).convert("RGB")
+            else:
+                try:
+                    response = requests.get(_url)
+                except requests.exceptions.MissingSchema:
+                    return Image.open(_url).convert("RGB")
+                else:
+                    return Image.open(BytesIO(response.content)).convert("RGB")
+
+        if not isinstance(content, str):
+            texts = []
+            image_urls = []
+            for c in content:
+                c_type = c.get("type")
+                if c_type == "text":
+                    texts.append(c["text"])
+                elif c_type == "image_url":
+                    image_urls.append(c["image_url"]["url"])
+            image_futures = []
+            with ThreadPoolExecutor() as executor:
+                for image_url in image_urls:
+                    fut = executor.submit(_load_image, image_url)
+                    image_futures.append(fut)
+            images = [fut.result() for fut in image_futures]
+            text = " ".join(texts)
+            if len(images) == 0:
+                return text, None
+            elif len(images) == 1:
+                return text, images
+            else:
+                raise RuntimeError(
+                    "Only one image per message is supported by CogVLM2."
+                )
+        return content, None
+
+    def _history_content_to_cogvlm2(
+        self, system_prompt: str, chat_history: List[ChatCompletionMessage]
+    ):
+        def _image_to_piexl_values(image):
+            if image.startswith("data:"):
+                logging.info("Parse url by base64 decoder.")
+                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+                # e.g. f"data:image/jpeg;base64,{base64_image}"
+                _type, data = image.split(";")
+                _, ext = _type.split("/")
+                data = data[len("base64,") :]
+                data = base64.b64decode(data.encode("utf-8"))
+                return Image.open(BytesIO(data)).convert("RGB")
+            else:
+                try:
+                    response = requests.get(image)
+                except requests.exceptions.MissingSchema:
+                    return Image.open(image).convert("RGB")
+                else:
+                    return Image.open(BytesIO(response.content)).convert("RGB")
+
+        query = system_prompt
+        history: List[Tuple] = []
+        pixel_values = None
+        for i in range(0, len(chat_history), 2):
+            user = chat_history[i]["content"]
+            if isinstance(user, List):
+                for content in user:
+                    c_type = content.get("type")
+                    if c_type == "text":
+                        user = content["text"]
+                    elif c_type == "image_url" and not pixel_values:
+                        pixel_values = _image_to_piexl_values(
+                            content["image_url"]["url"]
+                        )
+            assistant = chat_history[i + 1]["content"]
+            query = query + f" USER: {user} ASSISTANT:"
+            history.append((query, assistant))
+            query = query + f" {assistant}"
+        return query, history, [pixel_values]
+
+    def chat(
+        self,
+        prompt: Union[str, List[Dict]],
+        system_prompt: Optional[str] = None,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        system_prompt = system_prompt if system_prompt else ""
+        if generate_config and generate_config.get("stream"):
+            raise Exception(
+                f"Chat with model {self.model_family.model_name} does not support stream."
+            )
+
+        sanitized_config = {
+            "pad_token_id": 128002,
+            "max_new_tokens": generate_config.get("max_tokens", 512)
+            if generate_config
+            else 512,
+        }
+
+        content, image = self._message_content_to_cogvlm2(prompt)
+
+        history = []
+        query = ""
+        history_image = None
+        if chat_history:
+            query, history, history_image = self._history_content_to_cogvlm2(
+                system_prompt, chat_history
+            )
+
+        if image and history_image:
+            history = []
+            query = system_prompt + f" USER: {content} ASSISTANT:"
+        else:
+            image = image if image else history_image
+            query = query + f" USER: {content} ASSISTANT:"
+
+        input_by_model = self._model.build_conversation_input_ids(
+            self._tokenizer,
+            query=query,
+            history=history,
+            images=image,
+            template_version="chat",
+        )
+
+        inputs = {
+            "input_ids": input_by_model["input_ids"].unsqueeze(0).to(self._device),
+            "token_type_ids": input_by_model["token_type_ids"]
+            .unsqueeze(0)
+            .to(self._device),
+            "attention_mask": input_by_model["attention_mask"]
+            .unsqueeze(0)
+            .to(self._device),
+            "images": [
+                [input_by_model["images"][0].to(self._device).to(self._torch_type)]
+            ]
+            if image is not None
+            else None,
+        }
+        with torch.no_grad():
+            outputs = self._model.generate(**inputs, **sanitized_config)
+            outputs = outputs[:, inputs["input_ids"].shape[1] :]
+            response = self._tokenizer.decode(outputs[0])
+            response = response.split("<|end_of_text|>")[0]
+
+        chunk = Completion(
+            id=str(uuid.uuid1()),
+            object="text_completion",
+            created=int(time.time()),
+            model=self.model_uid,
+            choices=[
+                CompletionChoice(
+                    index=0, text=response, finish_reason="stop", logprobs=None
+                )
+            ],
+            usage=CompletionUsage(
+                prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
+            ),
+        )
+        return self._to_chat_completion(chunk)

xinference/model/llm/pytorch/core.py

@@ -62,6 +62,7 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "deepseek-vl-chat",
     "internvl-chat",
     "mini-internvl-chat",
+    "cogvlm2",
 ]
 
 

xinference/model/llm/pytorch/intern_vl.py

@@ -21,9 +21,7 @@ from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 import requests
 import torch
-import torchvision.transforms as T
 from PIL import Image
-from torchvision.transforms.functional import InterpolationMode
 
 from ....model.utils import select_device
 from ....types import (
@@ -79,19 +77,14 @@ class InternVLChatModel(PytorchChatModel):
             "device_map": device,
         }
 
-        if "Int8" in self.model_spec.quantizations:
-            kwargs.update(
-                {
-                    "load_in_8bit": True,
-                    "device_map": device,
-                }
-            )
-        elif "mini" in self.model_family.model_name:
+        if "int8" in self.quantization.lower():
+            kwargs["load_in_8bit"] = True
+        elif 2 == self.model_spec.model_size_in_billions:
             kwargs.pop("device_map")
 
         self._model = AutoModel.from_pretrained(self.model_path, **kwargs).eval()
 
-        if "Int8" not in self.model_spec.quantizations:
+        if "int8" not in self.quantization.lower():
             self._model.cuda()
 
         # Specify hyperparameters for generation
@@ -205,47 +198,6 @@ class InternVLChatModel(PytorchChatModel):
             history.append(tuple(tmp))
         return history, pixel_values
 
-        def _load_image(_url):
-            if _url.startswith("data:"):
-                logging.info("Parse url by base64 decoder.")
-                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
-                # e.g. f"data:image/jpeg;base64,{base64_image}"
-                _type, data = _url.split(";")
-                _, ext = _type.split("/")
-                data = data[len("base64,") :]
-                data = base64.b64decode(data.encode("utf-8"))
-
-                return Image.open(BytesIO(data)).convert("RGB")
-            else:
-                try:
-                    response = requests.get(_url)
-                except requests.exceptions.MissingSchema:
-                    return Image.open(_url).convert("RGB")
-                else:
-                    return Image.open(BytesIO(response.content)).convert("RGB")
-
-        if not isinstance(content, str):
-            texts = []
-            image_urls = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "text":
-                    texts.append(c["text"])
-                elif c_type == "image_url":
-                    image_urls.append(c["image_url"]["url"])
-            image_futures = []
-            with ThreadPoolExecutor() as executor:
-                for image_url in image_urls:
-                    fut = executor.submit(_load_image, image_url)
-                    image_futures.append(fut)
-            images = [fut.result() for fut in image_futures]
-            text = " ".join(texts)
-            if len(images) == 0:
-                return text
-            else:
-                return text, images
-        return content
-
     def _find_closest_aspect_ratio(
         self, aspect_ratio, target_ratios, width, height, image_size
     ):
@@ -309,6 +261,9 @@ class InternVLChatModel(PytorchChatModel):
         return processed_images
 
     def _build_transform(self, input_size):
+        import torchvision.transforms as T
+        from torchvision.transforms.functional import InterpolationMode
+
         MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
         transform = T.Compose(
             [
@@ -338,7 +293,7 @@ class InternVLChatModel(PytorchChatModel):
         chat_history: Optional[List[ChatCompletionMessage]] = None,
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        if generate_config and generate_config.pop("stream"):
+        if generate_config and generate_config.get("stream"):
             raise Exception(
                 f"Chat with model {self.model_family.model_name} does not support stream."
             )

xinference/model/llm/vllm/core.py

@@ -114,6 +114,7 @@ VLLM_SUPPORTED_CHAT_MODELS = [
     "qwen-chat",
     "Yi-chat",
     "Yi-1.5-chat",
+    "Yi-1.5-chat-16k",
     "code-llama-instruct",
     "mistral-instruct-v0.1",
     "mistral-instruct-v0.2",
@@ -140,7 +141,6 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.3":
 if VLLM_INSTALLED and vllm.__version__ >= "0.4.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen1.5-moe-chat")
     VLLM_SUPPORTED_CHAT_MODELS.append("c4ai-command-r-v01")
-    VLLM_SUPPORTED_CHAT_MODELS.append("c4ai-command-r-v01-4bit")
 
 
 class VLLMModel(LLM):

{xinference-0.11.2.dist-info → xinference-0.11.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xinference
-Version: 0.11.2
+Version: 0.11.3
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye
@@ -177,12 +177,12 @@ potential of cutting-edge AI models.
 - Docker image: [#855](https://github.com/xorbitsai/inference/pull/855)
 - Support multimodal: [#829](https://github.com/xorbitsai/inference/pull/829)
 ### New Models
+- Built-in support for [CogVLM2](https://github.com/THUDM/CogVLM2): [#1551](https://github.com/xorbitsai/inference/pull/1551)
+- Built-in support for [InternVL-Chat-V1-5](https://github.com/OpenGVLab/InternVL): [#1536](https://github.com/xorbitsai/inference/pull/1536)
+- Built-in support for [Yi-1.5](https://github.com/01-ai/Yi-1.5): [#1489](https://github.com/xorbitsai/inference/pull/1489)
 - Built-in support for [Llama 3](https://github.com/meta-llama/llama3): [#1332](https://github.com/xorbitsai/inference/pull/1332)
 - Built-in support for [Qwen1.5 110B](https://huggingface.co/Qwen/Qwen1.5-110B-Chat): [#1388](https://github.com/xorbitsai/inference/pull/1388)
 - Built-in support for [Mixtral-8x22B-instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1): [#1340](https://github.com/xorbitsai/inference/pull/1340)
-- Built-in support for [Command-R](https://huggingface.co/CohereForAI/c4ai-command-r-v01): [#1310](https://github.com/xorbitsai/inference/pull/1310)
-- Built-in support for [Qwen1.5 MOE](https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat): [#1263](https://github.com/xorbitsai/inference/pull/1263)
-- Built-in support for [Qwen1.5 32B](https://huggingface.co/Qwen/Qwen1.5-32B-Chat): [#1249](https://github.com/xorbitsai/inference/pull/1249)
 ### Integrations
 - [Dify](https://docs.dify.ai/advanced/model-configuration/xinference): an LLMOps platform that enables developers (and even non-developers) to quickly build useful applications based on large language models, ensuring they are visual, operable, and improvable.
 - [FastGPT](https://github.com/labring/FastGPT): a knowledge-based platform built on the LLM, offers out-of-the-box data processing and model invocation capabilities, allows for workflow orchestration through Flow visualization.

{xinference-0.11.2.dist-info → xinference-0.11.3.dist-info}/RECORD

@@ -1,15 +1,15 @@
 xinference/__init__.py,sha256=0LgIveLP6CXxoIaSrxhlFyOh0lOqPgJBVcBe0tkWJjc,987
 xinference/_compat.py,sha256=SQAjZMGxtBIce45qtW7ob7RWzA0zhv2yB3AxT0rb0uU,1778
-xinference/_version.py,sha256=UcTD5E4mTdIp6njSmYr_Ony5Wmkd1GJnwA9b2LvniUg,498
+xinference/_version.py,sha256=3PbaSmy-dqQ13cZaaCK5ch6akdnc3CpE14f0GcuqBi8,498
 xinference/conftest.py,sha256=Qus4KWCeaKS7c5UgNCTpPNucD2bjV8P7u1_qRosgGno,9743
-xinference/constants.py,sha256=Bu_fOJUGAvvqF_6FY5OzOHl7fQ1Nomek3LY17xr9oz4,2882
+xinference/constants.py,sha256=SIUOiSDpDIcZeuOE8kpF4NgUpCYGrHFIFjv4ogi5SAE,3040
 xinference/device_utils.py,sha256=zswJiws3VyTIaNO8z-MOcsJH_UiPoePPiKK5zoNrjTA,3285
 xinference/fields.py,sha256=0UtBFaDNzn1n9MRjyTkNrolsIML-TpZfudWOejqjni8,5245
 xinference/isolation.py,sha256=NstVRcO3dG4umHExICXAHlzVKwH8ch8MBwKwE-KFkE0,1826
 xinference/types.py,sha256=BFKUGHb0jKkAA1dczSf8pPlFutRE7-JtRp6C3oVSJ7Q,13626
 xinference/utils.py,sha256=VSOJMFd9H7kce98OtJZbcDjjpfzRpHAFs8WU0xXPBM8,717
 xinference/api/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/api/restful_api.py,sha256=Wj1LWFTQiwH7FEheaG1pA4wpVOCtWHnbTdRHUa6Qcek,60530
+xinference/api/restful_api.py,sha256=iuyxeNxwJHjEH5eSLCjoi1bZRmJFsG0j7pFI2ytQWjw,60830
 xinference/api/oauth2/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/api/oauth2/auth_service.py,sha256=74JzB42fbbmBu4Q1dW3A9Fp_N7167KgRGB42Z0NHjAM,6119
 xinference/api/oauth2/types.py,sha256=K923sv_XySIUtM2Eozl9IG082IJcDOS5SFLrPZ5ELBg,996
@@ -19,18 +19,18 @@ xinference/client/common.py,sha256=iciZRs5YjM2gYsXnwACPMaiBZp4_XpawWwfym0Iyu40,1
 xinference/client/handlers.py,sha256=3gd9C7u4URbcVdR6Eyv8cpEZ175Ll4q_jGL07CnEIpg,648
 xinference/client/restful/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/client/restful/restful_client.py,sha256=zqfZHliUeiDHvKKo1SrXH3rAvyxp0o_TaMwuCjV2pjQ,43589
-xinference/core/__init__.py,sha256=Fe5tYCHDbYJ7PhxJhQ68VbfgKgOsAuslNPr4wPhFMJM,612
-xinference/core/cache_tracker.py,sha256=OMY_0HZkb-R5EhHHWppIGqPhC3v9Iwdjp9oema_6eFc,4798
-xinference/core/chat_interface.py,sha256=B-qG7RF7HOquhKaPAJSnHA3Dqov5QZQA2yO-txt1IPs,17380
+xinference/core/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
+xinference/core/cache_tracker.py,sha256=xlhHwWsSLIo743r5U5l69b_rti_7Way-kGYzsunzs9s,5846
+xinference/core/chat_interface.py,sha256=7SOm6Qi-iFh1otycHpn6CpISq2wTLlJzEUngJtOwMIk,19558
 xinference/core/event.py,sha256=dTXv-zg-sAqlY1rFLvyor9D8WEXZvnUH7NigegpziO8,1648
 xinference/core/image_interface.py,sha256=G2iK24auEN4MrLkPlu1CAA_gf-BQrGQTjazi_FYqIxE,8825
 xinference/core/metrics.py,sha256=ScmTG15Uq3h_ob72ybZSMWdnk8P4sUZFcm60f4ikSXc,2631
 xinference/core/model.py,sha256=KAD9a2fYY97MfByNRnmia2oZR-UK7B-zW8gWl_tUJgs,18121
 xinference/core/resource.py,sha256=FQ0aRt3T4ZQo0P6CZZf5QUKHiCsr5llBvKb1f7wfnxg,1611
 xinference/core/status_guard.py,sha256=fF5hisvfn6es9DV6Z6RRD6V_S_uLcb8lHM6PArGgb04,2820
-xinference/core/supervisor.py,sha256=bREFDEeO6rF9hyWOuf9mA6IHo6Ur2BDGgPmnuysaN9o,41969
+xinference/core/supervisor.py,sha256=4UoXuzrDNk5P1eteUjOlXwdm8W5JFgUTn8kwSmxlROw,42076
 xinference/core/utils.py,sha256=LqPrez5dGELRQDSwOD5EP8XHb-aUKAdyszS-QpNouuw,6401
-xinference/core/worker.py,sha256=oGsqHQ5yJ4Ry-tAL06x6jkLxkQKbj0LxWtvvOy-SRdM,32780
+xinference/core/worker.py,sha256=bf5FtsVWMxQmQNmK0hTsC27CLU6ByGC02eCyrnJg4og,32999
 xinference/deploy/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/deploy/cmdline.py,sha256=J1NSfnQ-iJ87OPb_kNOfONgMZNqCow1Klm91uLS4_Lk,44582
 xinference/deploy/local.py,sha256=vlAvhcl8utP1DjW4MJpBgD4JLHQV-1Xebmdd8j9M8IM,3946
@@ -65,11 +65,11 @@ xinference/model/image/model_spec_modelscope.json,sha256=KMI-2YOJoLUtN93LZiqqETW
 xinference/model/image/utils.py,sha256=gxg8jJ2nYaDknzCcSC53WCy1slbB5aWU14AbJbfm6Z4,906
 xinference/model/image/stable_diffusion/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/image/stable_diffusion/core.py,sha256=ib_ZeSg7hzynmRqSnhjtrVuhoLOgZPrR1ZH2LjBmH2E,6063
-xinference/model/llm/__init__.py,sha256=kl2z63rNpgkoFQcg536SUt4TKub2wuDB8c7weYKeeoU,9874
+xinference/model/llm/__init__.py,sha256=AR36qg9QKRb3HtGauEGeFveFpg9xgOY0vJvJVHcJxkw,9946
 xinference/model/llm/core.py,sha256=CZJrbW3HaV8vJj5mxSnBD8ssABALG_xyXyPS0hHVBPY,7982
-xinference/model/llm/llm_family.json,sha256=otMgqAExkRqPC0yV_si0yc6sNhPEIxQjDrSg31Er9F8,152586
+xinference/model/llm/llm_family.json,sha256=ej30g6j2_jYHTFMj5_hi2ERbs-d5KUdSW47lKU0tjMo,160208
 xinference/model/llm/llm_family.py,sha256=aQZPE2gj5YfII2mhgbaSs1MfllKnUXd8xbflKr_YdFs,37357
-xinference/model/llm/llm_family_modelscope.json,sha256=5cyM-y0HtG_fBfC289IU97iasfjZZgUefn5UmVmg7TY,91282
+xinference/model/llm/llm_family_modelscope.json,sha256=f72zwrQfBJzijQwTD1yIy6KPInuyddKE44Iy8arMo-A,98156
 xinference/model/llm/memory.py,sha256=PTD8m6TCZVU1zrwc9wepX9cUjCqAXBENj6X7tjua0to,10207
 xinference/model/llm/utils.py,sha256=0SindpTW6dUWn17E-Ne9scnSfPOovb53sIuc9zxIFfo,31653
 xinference/model/llm/ggml/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
@@ -79,13 +79,14 @@ xinference/model/llm/ggml/tools/__init__.py,sha256=6a6P2VPKE06xKxJ-dTqp4TRO2IEDW
 xinference/model/llm/ggml/tools/convert_ggml_to_gguf.py,sha256=92To8eoVQBkDZD52_aWBNda2K1Ob6YaHlcfS-8_aOuw,17991
 xinference/model/llm/ggml/tools/gguf.py,sha256=Hv2haR-UN7NdB1N8YId32hFoEPd-JX6_aUNWRJhyJZc,30277
 xinference/model/llm/pytorch/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/pytorch/baichuan.py,sha256=JBHldfdjUlSV44HB_td6402CwryDm61rlzw4D_flKHI,2820
+xinference/model/llm/pytorch/baichuan.py,sha256=TpCfi37Ou87VA8S5siGRUK5JWC5SjGbZYkZoTe9VSA8,2883
 xinference/model/llm/pytorch/chatglm.py,sha256=meFb4P1MXw63IhoDQwmFGczTXa3vEpv1L1Eo9XIifG8,9624
+xinference/model/llm/pytorch/cogvlm2.py,sha256=hRFsNQIgvNa7Dfhh08Z2K3QuKzXop_fkkXtbXwUaKWg,9472
 xinference/model/llm/pytorch/compression.py,sha256=U0vMJ-JaBt4oC2LffgWg6HbPj1CeUi_YdwVbjDd0mRA,8112
-xinference/model/llm/pytorch/core.py,sha256=VGJfZ92e3SftQXrOwxiYMUSF1YnWSUU8Z8WHr6yzpJ8,19984
+xinference/model/llm/pytorch/core.py,sha256=qyAd2UdWq1PFJgFBksMGDouFxbksdr7AmM2bW8TsWEU,19999
 xinference/model/llm/pytorch/deepseek_vl.py,sha256=T9DKP4cULvRaHSiU08lOWd_j6mt8b3ZIBByneZ0jY8U,11498
 xinference/model/llm/pytorch/falcon.py,sha256=POSP7vzRJaM5PjvX8dh60jNDXgnCwktwSmeZ7kypQU0,4499
-xinference/model/llm/pytorch/intern_vl.py,sha256=ADmZeWvkKz_lGBJQqksgEv0sKa8dk6hNI9fNSdgRZDs,14835
+xinference/model/llm/pytorch/intern_vl.py,sha256=_7IxqtTODTOTfR_4BDQ1hSkgE_qLjAhdyNt7fz_AD20,13044
 xinference/model/llm/pytorch/internlm2.py,sha256=vjspoc2VHbuD1JaUtjt0sOq9MwvRr2OD3_tKQhBVUPc,7244
 xinference/model/llm/pytorch/llama_2.py,sha256=HMhUmn4oYW2maeSMIr1yY7jlAOMD0OVAxnF0dnRWmio,3710
 xinference/model/llm/pytorch/omnilmm.py,sha256=4r6pipch1LU1FPA80sOCE7Z0k3TO_J8CIT7pmVmWKEM,5664
@@ -96,7 +97,7 @@ xinference/model/llm/pytorch/yi_vl.py,sha256=MljT7tpgFIhL6n5rdoS3hmq_u0rtHRE6cxX
 xinference/model/llm/sglang/__init__.py,sha256=-sjSIQ4K6w-TEzx49kVaWeWC443fnZqODU91GCQ_JNo,581
 xinference/model/llm/sglang/core.py,sha256=RGHy6t9n0c4zL6Uha8P7t-qPvisPyulFVHw-8Aq8CJ0,14046
 xinference/model/llm/vllm/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/vllm/core.py,sha256=_cM-NEnDtpGVHMgHxr-uxV0sA-xXUjDsL5i__cpHQEU,21639
+xinference/model/llm/vllm/core.py,sha256=a__R1e3V_PwEOtky5NxVIMKWLo4eVX8S4BnSVWM5uAs,21597
 xinference/model/rerank/__init__.py,sha256=BXIL1uu3ZpZHX9bODhW9lxKUXudZE7-OkXFmmM5rpMU,2817
 xinference/model/rerank/core.py,sha256=BLIIStjxUFghSFoxCimet88ghqGwmVaskOYdVRxKdpI,10572
 xinference/model/rerank/custom.py,sha256=NKk7jA7p4xkuwS5WoOs2SY2wdnoAVpyCjBTvv317bBw,3917
@@ -15423,9 +15424,9 @@ xinference/web/ui/node_modules/yargs-parser/package.json,sha256=BSwbOzgetKXMK4u0
 xinference/web/ui/node_modules/yocto-queue/package.json,sha256=6U1XHQPGXJTqsiFvT953ORihUtXTblZy4fXBWP9qxC0,725
 xinference/web/ui/node_modules/yup/package.json,sha256=xRFSROB9NKxqSWHEVFvSTsPs9Ll074uo8OS1zEw0qhA,1206
 xinference/web/ui/node_modules/yup/node_modules/type-fest/package.json,sha256=JTv2zTTVgxQ2H82m1-6qEpdMv08lHjFx4Puf_MsbB_Q,1134
-xinference-0.11.2.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-0.11.2.dist-info/METADATA,sha256=DQYeSyUTuB1mVksGNtxaXfcwqI0Vc8r_ucm3BcpVcXg,15542
-xinference-0.11.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-xinference-0.11.2.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-0.11.2.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-0.11.2.dist-info/RECORD,,
+xinference-0.11.3.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-0.11.3.dist-info/METADATA,sha256=Hp7uQ0rgqOkdwWD1T7U3KHMijVdeISct2OvSYNSq4EY,15495
+xinference-0.11.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+xinference-0.11.3.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-0.11.3.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-0.11.3.dist-info/RECORD,,