xinference 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

xinference/model/llm/transformers/glm_edge_v.py

@@ -0,0 +1,230 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from threading import Thread
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+import torch
+
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
+from ...utils import select_device
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import (
+    _decode_image_without_rgb,
+    generate_chat_completion,
+    generate_completion_chunk,
+)
+from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class GlmEdgeVModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._device = None
+        self._tokenizer = None
+        self._model = None
+        self._processor = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        family = model_family.model_family or model_family.model_name
+        if "glm-edge-v" in family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoImageProcessor, AutoModelForCausalLM, AutoTokenizer
+
+        device = self._pytorch_model_config.get("device", "auto")
+        self._device = select_device(device)
+
+        kwargs = {"device_map": self._device}
+        quantization = self.quantization
+
+        # referenced from PytorchModel.load
+        if quantization != "none":
+            if self._device == "cuda" and self._is_linux():
+                kwargs["device_map"] = "auto"
+                if quantization == "4-bit":
+                    kwargs["load_in_4bit"] = True
+                elif quantization == "8-bit":
+                    kwargs["load_in_8bit"] = True
+                else:
+                    raise ValueError(
+                        f"Quantization {quantization} is not supported in temporary"
+                    )
+            else:
+                if quantization != "8-bit":
+                    raise ValueError(
+                        f"Only 8-bit quantization is supported if it is not linux system or cuda device"
+                    )
+
+        processor = AutoImageProcessor.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._processor = processor
+
+        model = AutoModelForCausalLM.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+        )
+
+        self._model = model
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._tokenizer = tokenizer
+
+    @staticmethod
+    def _get_processed_msgs(
+        messages: List[Dict],
+    ) -> Tuple[List[Dict[str, Any]], List[Any]]:
+        res = []
+        img = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if isinstance(content, str):
+                res.append({"role": role, "content": content})
+            else:
+                texts = []
+                image_urls = []
+                for c in content:
+                    c_type = c.get("type")
+                    if c_type == "text":
+                        texts.append(c["text"])
+                    else:
+                        assert (
+                            c_type == "image_url"
+                        ), "Please follow the image input of the OpenAI API."
+                        image_urls.append(c["image_url"]["url"])
+                if len(image_urls) > 1:
+                    raise RuntimeError("Only one image per message is supported")
+                image_futures = []
+                with ThreadPoolExecutor() as executor:
+                    for image_url in image_urls:
+                        fut = executor.submit(_decode_image_without_rgb, image_url)
+                        image_futures.append(fut)
+                images = [fut.result() for fut in image_futures]
+                assert len(images) <= 1
+                text = " ".join(texts)
+                img.extend(images)
+                if images:
+                    res.append(
+                        {
+                            "role": role,
+                            "content": [
+                                {"type": "image"},
+                                {"type": "text", "text": text},
+                            ],
+                        }
+                    )
+                else:
+                    res.append({"role": role, "content": text})
+        return res, img
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        from transformers import TextIteratorStreamer
+
+        if not generate_config:
+            generate_config = {}
+
+        stream = generate_config.get("stream", False)
+        msgs, imgs = self._get_processed_msgs(messages)
+
+        inputs = self._tokenizer.apply_chat_template(
+            msgs,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_tensors="pt",
+            return_dict=True,
+        )  # chat mode
+        inputs = inputs.to(self._model.device)
+
+        generate_kwargs = {
+            **inputs,
+        }
+        if len(imgs) > 0:
+            generate_kwargs["pixel_values"] = torch.tensor(
+                self._processor(imgs[-1]).pixel_values
+            ).to(self._model.device)
+        stop_str = "<|endoftext|>"
+
+        if stream:
+            streamer = TextIteratorStreamer(
+                tokenizer=self._tokenizer,
+                timeout=60,
+                skip_prompt=True,
+                skip_special_tokens=True,
+            )
+            generate_kwargs = {
+                **generate_kwargs,
+                "streamer": streamer,
+            }
+            t = Thread(target=self._model.generate, kwargs=generate_kwargs)
+            t.start()
+
+            it = self.chat_stream(streamer, stop_str)
+            return self._to_chat_completion_chunks(it)
+        else:
+            with torch.no_grad():
+                outputs = self._model.generate(**generate_kwargs)
+                outputs = outputs[0][len(inputs["input_ids"][0]) :]
+                response = self._tokenizer.decode(outputs)
+                if response.endswith(stop_str):
+                    response = response[: -len(stop_str)]
+            return generate_chat_completion(self.model_uid, response)
+
+    def chat_stream(self, streamer, stop_str) -> Iterator[CompletionChunk]:
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            if not new_text.endswith(stop_str):
+                yield generate_completion_chunk(
+                    chunk_text=new_text,
+                    finish_reason=None,
+                    chunk_id=completion_id,
+                    model_uid=self.model_uid,
+                    prompt_tokens=-1,
+                    completion_tokens=-1,
+                    total_tokens=-1,
+                    has_choice=True,
+                    has_content=True,
+                )
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )

xinference/model/llm/utils.py

@@ -569,6 +569,25 @@ def _decode_image(_url):
             return Image.open(BytesIO(response.content)).convert("RGB")
 
 
+def _decode_image_without_rgb(_url):
+    if _url.startswith("data:"):
+        logging.info("Parse url by base64 decoder.")
+        # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+        # e.g. f"data:image/jpeg;base64,{base64_image}"
+        _type, data = _url.split(";")
+        _, ext = _type.split("/")
+        data = data[len("base64,") :]
+        data = base64.b64decode(data.encode("utf-8"))
+        return Image.open(BytesIO(data))
+    else:
+        try:
+            response = requests.get(_url)
+        except requests.exceptions.MissingSchema:
+            return Image.open(_url)
+        else:
+            return Image.open(BytesIO(response.content))
+
+
 @typing.no_type_check
 def generate_completion_chunk(
     chunk_text: Optional[str],

xinference/model/llm/vllm/core.py

@@ -69,6 +69,7 @@ class VLLMModelConfig(TypedDict, total=False):
     quantization: Optional[str]
     max_model_len: Optional[int]
     limit_mm_per_prompt: Optional[Dict[str, int]]
+    guided_decoding_backend: Optional[str]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -85,6 +86,14 @@ class VLLMGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     stream: bool  # non-sampling param, should not be passed to the engine.
     stream_options: Optional[Union[dict, None]]
+    response_format: Optional[dict]
+    guided_json: Optional[Union[str, dict]]
+    guided_regex: Optional[str]
+    guided_choice: Optional[List[str]]
+    guided_grammar: Optional[str]
+    guided_json_object: Optional[bool]
+    guided_decoding_backend: Optional[str]
+    guided_whitespace_pattern: Optional[str]
 
 
 try:
@@ -144,6 +153,7 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-instruct")
     VLLM_SUPPORTED_MODELS.append("qwen2.5-coder")
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-coder-instruct")
+    VLLM_SUPPORTED_CHAT_MODELS.append("QwQ-32B-Preview")
 
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
@@ -314,6 +324,7 @@ class VLLMModel(LLM):
         model_config.setdefault("max_num_seqs", 256)
         model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
+        model_config.setdefault("guided_decoding_backend", "outlines")
 
         return model_config
 
@@ -325,6 +336,22 @@ class VLLMModel(LLM):
             generate_config = {}
 
         sanitized = VLLMGenerateConfig()
+
+        response_format = generate_config.pop("response_format", None)
+        guided_decoding_backend = generate_config.get("guided_decoding_backend", None)
+        guided_json_object = None
+        guided_json = None
+
+        if response_format is not None:
+            if response_format.get("type") == "json_object":
+                guided_json_object = True
+            elif response_format.get("type") == "json_schema":
+                json_schema = response_format.get("json_schema")
+                assert json_schema is not None
+                guided_json = json_schema.get("json_schema")
+                if guided_decoding_backend is None:
+                    guided_decoding_backend = "outlines"
+
         sanitized.setdefault("lora_name", generate_config.get("lora_name", None))
         sanitized.setdefault("n", generate_config.get("n", 1))
         sanitized.setdefault("best_of", generate_config.get("best_of", None))
@@ -346,6 +373,28 @@ class VLLMModel(LLM):
         sanitized.setdefault(
             "stream_options", generate_config.get("stream_options", None)
         )
+        sanitized.setdefault(
+            "guided_json", generate_config.get("guided_json", guided_json)
+        )
+        sanitized.setdefault("guided_regex", generate_config.get("guided_regex", None))
+        sanitized.setdefault(
+            "guided_choice", generate_config.get("guided_choice", None)
+        )
+        sanitized.setdefault(
+            "guided_grammar", generate_config.get("guided_grammar", None)
+        )
+        sanitized.setdefault(
+            "guided_whitespace_pattern",
+            generate_config.get("guided_whitespace_pattern", None),
+        )
+        sanitized.setdefault(
+            "guided_json_object",
+            generate_config.get("guided_json_object", guided_json_object),
+        )
+        sanitized.setdefault(
+            "guided_decoding_backend",
+            generate_config.get("guided_decoding_backend", guided_decoding_backend),
+        )
 
         return sanitized
 
@@ -483,13 +532,46 @@ class VLLMModel(LLM):
             if isinstance(stream_options, dict)
             else False
         )
-        sampling_params = SamplingParams(**sanitized_generate_config)
+
+        if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
+            # guided decoding only available for vllm >= 0.6.3
+            from vllm.sampling_params import GuidedDecodingParams
+
+            guided_options = GuidedDecodingParams.from_optional(
+                json=sanitized_generate_config.pop("guided_json", None),
+                regex=sanitized_generate_config.pop("guided_regex", None),
+                choice=sanitized_generate_config.pop("guided_choice", None),
+                grammar=sanitized_generate_config.pop("guided_grammar", None),
+                json_object=sanitized_generate_config.pop("guided_json_object", None),
+                backend=sanitized_generate_config.pop("guided_decoding_backend", None),
+                whitespace_pattern=sanitized_generate_config.pop(
+                    "guided_whitespace_pattern", None
+                ),
+            )
+
+            sampling_params = SamplingParams(
+                guided_decoding=guided_options, **sanitized_generate_config
+            )
+        else:
+            # ignore generate configs
+            sanitized_generate_config.pop("guided_json", None)
+            sanitized_generate_config.pop("guided_regex", None)
+            sanitized_generate_config.pop("guided_choice", None)
+            sanitized_generate_config.pop("guided_grammar", None)
+            sanitized_generate_config.pop("guided_json_object", None)
+            sanitized_generate_config.pop("guided_decoding_backend", None)
+            sanitized_generate_config.pop("guided_whitespace_pattern", None)
+            sampling_params = SamplingParams(**sanitized_generate_config)
+
         if not request_id:
             request_id = str(uuid.uuid1())
 
         assert self._engine is not None
         results_generator = self._engine.generate(
-            prompt, sampling_params, request_id, lora_request=lora_request
+            prompt,
+            sampling_params,
+            request_id,
+            lora_request,
         )
 
         async def stream_results() -> AsyncGenerator[CompletionChunk, None]:

xinference/types.py

@@ -71,7 +71,8 @@ class EmbeddingUsage(TypedDict):
 class EmbeddingData(TypedDict):
     index: int
     object: str
-    embedding: List[float]
+    # support sparse embedding
+    embedding: Union[List[float], Dict[str, float]]
 
 
 class Embedding(TypedDict):

{xinference-1.0.0.dist-info → xinference-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xinference
-Version: 1.0.0
+Version: 1.0.1
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: xoscar>=0.3.0
+Requires-Dist: xoscar<0.4.2,>=0.3.0
 Requires-Dist: torch
 Requires-Dist: gradio
 Requires-Dist: pillow
@@ -34,7 +34,7 @@ Requires-Dist: huggingface-hub>=0.19.4
 Requires-Dist: typing-extensions
 Requires-Dist: modelscope>=1.10.0
 Requires-Dist: sse-starlette>=1.6.5
-Requires-Dist: openai>1
+Requires-Dist: openai>=1.40.0
 Requires-Dist: python-jose[cryptography]
 Requires-Dist: passlib[bcrypt]
 Requires-Dist: aioprometheus[starlette]>=23.12.0
@@ -80,7 +80,7 @@ Requires-Dist: conformer; extra == "all"
 Requires-Dist: gdown; extra == "all"
 Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: HyperPyYAML; extra == "all"
-Requires-Dist: onnxruntime==1.16.0; extra == "all"
+Requires-Dist: onnxruntime>=1.16.0; extra == "all"
 Requires-Dist: boto3<1.28.65,>=1.28.55; extra == "all"
 Requires-Dist: tensorizer~=2.9.0; extra == "all"
 Requires-Dist: eva-decord; extra == "all"
@@ -98,6 +98,7 @@ Requires-Dist: verovio>=4.3.1; extra == "all"
 Requires-Dist: auto-gptq; sys_platform != "darwin" and extra == "all"
 Requires-Dist: autoawq<0.2.6; sys_platform != "darwin" and extra == "all"
 Requires-Dist: mlx-lm; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "all"
+Requires-Dist: mlx-whisper; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "all"
 Requires-Dist: vllm>=0.2.6; sys_platform == "linux" and extra == "all"
 Requires-Dist: sglang>=0.2.7; sys_platform == "linux" and extra == "all"
 Provides-Extra: audio
@@ -119,7 +120,7 @@ Requires-Dist: diffusers>=0.30.0; extra == "audio"
 Requires-Dist: gdown; extra == "audio"
 Requires-Dist: pyarrow; extra == "audio"
 Requires-Dist: HyperPyYAML; extra == "audio"
-Requires-Dist: onnxruntime==1.16.0; extra == "audio"
+Requires-Dist: onnxruntime>=1.16.0; extra == "audio"
 Requires-Dist: loguru; extra == "audio"
 Requires-Dist: natsort; extra == "audio"
 Requires-Dist: loralib; extra == "audio"
@@ -143,7 +144,7 @@ Requires-Dist: sphinx-intl>=0.9.9; extra == "dev"
 Requires-Dist: jieba>=0.42.0; extra == "dev"
 Requires-Dist: flake8>=3.8.0; extra == "dev"
 Requires-Dist: black; extra == "dev"
-Requires-Dist: openai>1; extra == "dev"
+Requires-Dist: openai>=1.40.0; extra == "dev"
 Requires-Dist: langchain; extra == "dev"
 Requires-Dist: langchain-community; extra == "dev"
 Requires-Dist: orjson; extra == "dev"
@@ -177,6 +178,7 @@ Provides-Extra: llama_cpp
 Requires-Dist: llama-cpp-python!=0.2.58,>=0.2.25; extra == "llama-cpp"
 Provides-Extra: mlx
 Requires-Dist: mlx-lm; extra == "mlx"
+Requires-Dist: mlx-whisper; extra == "mlx"
 Provides-Extra: rerank
 Requires-Dist: FlagEmbedding; extra == "rerank"
 Provides-Extra: sglang

{xinference-1.0.0.dist-info → xinference-1.0.1.dist-info}/RECORD

@@ -1,15 +1,15 @@
 xinference/__init__.py,sha256=nmTTrYbIpj964ZF6ojtgOM7E85JBOj1EyQbmYjbj1jw,915
-xinference/_compat.py,sha256=xFztCfyrq3O_4bssL_ygghYkfxicv_ZhiX2YDDWHf-k,3571
-xinference/_version.py,sha256=fpV50OH7B6n2pfTzqkYhtyB658xgOLxDdVxTQumkplE,497
+xinference/_compat.py,sha256=vpf_M9Ou6d9qaq-hG5isJ-C8e8UdPZPqoWcPhabfNko,4135
+xinference/_version.py,sha256=ENOsRw9OHQL_a2m0T4MVfI2yL84k11_7guNq3F53rsg,497
 xinference/conftest.py,sha256=vETDpRBVIlWbWi7OTwf7og89U25KyYGyI7yPIB3O8N8,9564
 xinference/constants.py,sha256=mEW4HDzjXtDXN61Mt6TtJrJ4ljbB6VUkh97e3oDbNx4,3905
 xinference/device_utils.py,sha256=zswJiws3VyTIaNO8z-MOcsJH_UiPoePPiKK5zoNrjTA,3285
 xinference/fields.py,sha256=0UtBFaDNzn1n9MRjyTkNrolsIML-TpZfudWOejqjni8,5245
 xinference/isolation.py,sha256=uhkzVyL3fSYZSuFexkG6Jm-tRTC5I607uNg000BXAnE,1949
-xinference/types.py,sha256=LHTbNLf0zI-FLruxRuBt2KMpk2P4eKpYdFvh2qzNTGI,12458
+xinference/types.py,sha256=t9SIU06Y1Y_lmXMfQmYAHmP8K6vTnD5Ly32z4KqriZE,12514
 xinference/utils.py,sha256=zYgf9bCvfbybRt3gEog6r5WJCpj0czZCf0qgRdYjkN8,720
 xinference/api/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/api/restful_api.py,sha256=eWw6tQ9zVOYAB_P9Zr_0bO3_ng7wyyGFIlA7MnV7q_k,88589
+xinference/api/restful_api.py,sha256=4tEu4YiT5iQU9ZVPI5gpQzwe-pEXiM05o_7S7MnJHBg,92140
 xinference/api/oauth2/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/api/oauth2/auth_service.py,sha256=74JzB42fbbmBu4Q1dW3A9Fp_N7167KgRGB42Z0NHjAM,6119
 xinference/api/oauth2/types.py,sha256=K923sv_XySIUtM2Eozl9IG082IJcDOS5SFLrPZ5ELBg,996
@@ -18,14 +18,14 @@ xinference/client/__init__.py,sha256=Gc4HOzAy_1cic5kXlso7hahYgw89CKvZSJDicEU461k
 xinference/client/common.py,sha256=iciZRs5YjM2gYsXnwACPMaiBZp4_XpawWwfym0Iyu40,1617
 xinference/client/handlers.py,sha256=OKl_i5FA341wsQf_0onSOPbbW6V861WJrSP7ghtDc8c,527
 xinference/client/restful/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/client/restful/restful_client.py,sha256=nz4Gz9PpX1seB20xmO5Pc6vliqB44gqJOfcIvdVuVVQ,52276
+xinference/client/restful/restful_client.py,sha256=JwzP7etUZBR0mmU7y3dUOEWN_D7ol_2hXN9KMAzKZaw,53601
 xinference/core/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/core/cache_tracker.py,sha256=3ubjYCU5aZToSp2GEuzedECVrg-PR4kThTefrFUkb9g,6971
 xinference/core/chat_interface.py,sha256=Kiqs1XOXgYBlP7DOXLEXaFjbVuS0yC1-dXJyxrxiRNE,20785
 xinference/core/event.py,sha256=42F38H2WOl6aPxp2oxX6WNxHRRxbnvYRmbt4Ar7NP4U,1640
 xinference/core/image_interface.py,sha256=5Iuoiw3g2TvgOYi3gRIAGApve2nNzfMPduRrBHvd1NY,13755
 xinference/core/metrics.py,sha256=ScmTG15Uq3h_ob72ybZSMWdnk8P4sUZFcm60f4ikSXc,2631
-xinference/core/model.py,sha256=fVE-b7vLgMgFmY4yJpFVnV_4Pw1Bde-ykBERZJVAjsM,39873
+xinference/core/model.py,sha256=k5hMtDUhZZjytz1aniTq4F9Bl2Kx-7fPv8cd_yLbCUw,40646
 xinference/core/progress_tracker.py,sha256=LIF6CLIlnEoSBkuDCraJktDOzZ31mQ4HOo6EVr3KpQM,6453
 xinference/core/resource.py,sha256=FQ0aRt3T4ZQo0P6CZZf5QUKHiCsr5llBvKb1f7wfnxg,1611
 xinference/core/scheduler.py,sha256=gdj3SyP_jelJ86vTRrgnFynhxz5JSwLRsQgx8PTtBi8,15671
@@ -44,22 +44,23 @@ xinference/deploy/test/test_cmdline.py,sha256=m8xDzjtDuAJy0QkvYVJIZDuTB29cmYBV0d
 xinference/model/__init__.py,sha256=bCEwvKjoazBW8QZ5C5Hpfd5v_VTbWpXvo8IB9d4Qs70,1127
 xinference/model/core.py,sha256=_NEH4wkjjJgRDdLHNVY_hL3V0kT67CvTay89uIzx1Ns,4736
 xinference/model/utils.py,sha256=_yJ5h4RUzt7Kjs2WdjSzbVM3FTWEkX0ycOnXANZ9KVg,11394
-xinference/model/audio/__init__.py,sha256=G4n-MyzdarFVOndPRkEyZZrCwqFIG8yIsky6_5dife0,3433
+xinference/model/audio/__init__.py,sha256=KasWsaNPeij6sGpHKqXaUc_bxUw1yYbD7-fwxkcoAVE,3731
 xinference/model/audio/chattts.py,sha256=ny3DZTCTt2MzdkLw994_QHZ_4qIEUZcNexNJkCejCyo,4998
-xinference/model/audio/core.py,sha256=73KojfhE8QFEVkWoqC8FohioQg3TALz1hKCzuDOArgM,6554
+xinference/model/audio/core.py,sha256=bdStvxdYJC2R8YAs4W58P97MrcREq70Ta9JX1ROxynM,6894
 xinference/model/audio/cosyvoice.py,sha256=Enur1Y4Xa-mpr7wwnoXWwhyh7PUAjrHZ8DV91tTrpjE,6426
 xinference/model/audio/custom.py,sha256=8GXBRmTtR-GY03-E91nlRGTIuabCRzlt20ecU6Un6Y8,4985
-xinference/model/audio/fish_speech.py,sha256=v2WVEV-BLWnbiDvqrx8WTGE_YNKmd9QoAF1LZBXWxn0,7310
+xinference/model/audio/fish_speech.py,sha256=rdt73VK512WocFhL6WqcqoxOkMYtB00sMu40umt9RuQ,8856
 xinference/model/audio/funasr.py,sha256=65z7U7_F14CCP-jg6BpeY3_49FK7Y5OCRSzrhhsklCg,4075
-xinference/model/audio/model_spec.json,sha256=iw0kE-0j2JaLBwIc3KKra9NERbxKNHIhXuhbwnhGXbA,5120
+xinference/model/audio/model_spec.json,sha256=Z_Y7Fv6fxLGAX8em5pgGVR2ZxBxtCsy0ANsKrjy9vA0,7300
 xinference/model/audio/model_spec_modelscope.json,sha256=U82E5vZahi4si6kpCjdp2FAG2lCpQ7s7w_1t6lj2ysI,2038
 xinference/model/audio/utils.py,sha256=pwo5cHh8nvhyBa9f-17QaVpXMSjmbpGbPYKwBBtEhGM,717
 xinference/model/audio/whisper.py,sha256=PQL7rebGC7WlIOItuDtjdEtSJtlhxFkolot-Fj-8uDU,7982
+xinference/model/audio/whisper_mlx.py,sha256=zBuCd7GUlsN9FC_-J11gqIkOCicihfbqxoabiXTvH2Q,7237
 xinference/model/embedding/__init__.py,sha256=1GmvQsbeeVUT-VRaRGetf8UT4RQgLWIzfp5kfX5jw-k,3567
-xinference/model/embedding/core.py,sha256=PP9Hpv_jK9x3cB2oSa1Q7SnzzPn7TnfVyF6uzx216OU,18762
+xinference/model/embedding/core.py,sha256=kGMxfKQ5s0G2fJcW-zVxKThDFbOWpSQD8vkCgnhFnoU,29593
 xinference/model/embedding/custom.py,sha256=757KncqhsOWVypHZFtuhBP_xy-UTNaFdy0BuZRfuIV8,3848
-xinference/model/embedding/model_spec.json,sha256=dn1XZDiB-HED_IqZO3iYdfhpqA1EO3wHcylXd1O9WK8,7060
-xinference/model/embedding/model_spec_modelscope.json,sha256=1qyMQR-JrcSZ_WB5gEtNI9IEzE9orSM41rjSXS6rMs4,6210
+xinference/model/embedding/model_spec.json,sha256=cVYzB4VLYw2FoP7bfLntTuMVDEigHTGDSyBtcj8F6gc,7060
+xinference/model/embedding/model_spec_modelscope.json,sha256=kWPY9tILL69X_6iDuVoh6Y_VODLJ7EFsD0CGkbuQlGo,6210
 xinference/model/embedding/utils.py,sha256=t_y-7TrYenJKzX3p8e8KWXyC66u7Kd7lMvSzEOolnZw,790
 xinference/model/flexible/__init__.py,sha256=DUAfq9vaiXOfUJfP2WRfqDmGfMtKMqRE-wLETaJw4_w,1718
 xinference/model/flexible/core.py,sha256=3REGHL9uUTgwgEEr6qv5uNMq-j-7by4bAco7QNwwxx4,7231
@@ -81,32 +82,33 @@ xinference/model/image/scheduler/flux.py,sha256=GHlpPfU5UxsiQWNyvNe9SaVZceNg_2Ci
 xinference/model/image/stable_diffusion/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/image/stable_diffusion/core.py,sha256=M_sYFsY_q91l0D6O7AqQVd_h-RIgXcmaydyCvASyNsI,23055
 xinference/model/image/stable_diffusion/mlx.py,sha256=GZsozzGB04NfHAdU9MI6gwWE1t_A-s_Ddn_ic8DlkKQ,7476
-xinference/model/llm/__init__.py,sha256=9g9dFG2XuNDCTLE5vuJ6kCT-rqe9MfN56aEapyXaJ5M,13938
+xinference/model/llm/__init__.py,sha256=WqNnI4ePhe1XJ_gzFOk-wpM9fjBjUpDwGlAW9CuzlNM,14020
 xinference/model/llm/core.py,sha256=g-luuAjZizrPunhyFE9IRjn57l0g6FY_1xUwtlRegbs,8151
-xinference/model/llm/llm_family.json,sha256=H1jJvTot6DsoJ_hyTMqJbcHI95tkuEwwYeljQRmPW_8,296753
+xinference/model/llm/llm_family.json,sha256=mlqXYbBkjBeB3xWRpaYpScpoDM5DIT1ESxI8IE7lDoI,307998
 xinference/model/llm/llm_family.py,sha256=tI2wPefd7v-PWcVhUO2qy6iGob_ioeNCwAQQzal-2o4,39549
 xinference/model/llm/llm_family_csghub.json,sha256=zMKWbihsxQNVB1u5iKJbZUkbOfQ4IPNq1KQ-8IDPQQA,8759
-xinference/model/llm/llm_family_modelscope.json,sha256=iNnH9DRTVc-EmX0YS2790LKRjJ945aLbtK97uoTY6_k,227335
+xinference/model/llm/llm_family_modelscope.json,sha256=nySjsEQk6V6PK0IX4zpjUyTQLBVGYv77cxwdHlagvps,238793
 xinference/model/llm/llm_family_openmind_hub.json,sha256=jl9pfbe5DztoxgEwKBxDk1Wd7TziTiJ48_Ie_lJdYjA,67872
 xinference/model/llm/memory.py,sha256=NEIMw6wWaF9S_bnBYq-EyuDhVbUEEeceQhwE1iwsrhI,10207
-xinference/model/llm/utils.py,sha256=DUC6jPr1-kPNsgc4J5MXNSMVgDlPLfQiitLGfdJxVxM,23596
+xinference/model/llm/utils.py,sha256=uHLhiaxPNgrIOzGvVJJlwE2S7eQtFis4uvfOjFSVm4Q,24294
 xinference/model/llm/llama_cpp/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/llama_cpp/core.py,sha256=vjuTapwbn-ZjUX-8WA0nFyicE4UGUSehU_csSetvcZw,10928
 xinference/model/llm/lmdeploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xinference/model/llm/lmdeploy/core.py,sha256=WvSP3x6t-HBv6hKh1qWZatFAzlcZCyyKqvc3ua8yPTI,19835
 xinference/model/llm/mlx/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/mlx/core.py,sha256=IsSFqkAK27yTafcWODw958jSNyxniiTsn9QR1Z6IhWk,15367
+xinference/model/llm/mlx/core.py,sha256=ojemIUVXt4ApI5LgjJ2BBXF_gw49Cu4EtixurLnSH1s,15318
 xinference/model/llm/sglang/__init__.py,sha256=-sjSIQ4K6w-TEzx49kVaWeWC443fnZqODU91GCQ_JNo,581
-xinference/model/llm/sglang/core.py,sha256=ft4QlDw36gwoic8lyjtSx2ai6KTW84CPVbYr8grqGMI,16698
+xinference/model/llm/sglang/core.py,sha256=1xXOyn3oWYYl-AYvAGwsQkxsaEf0kpbdKQlfORgU7Dk,16721
 xinference/model/llm/transformers/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/transformers/chatglm.py,sha256=tr6nEfHAg_mFRDlqXLMvbSq6U47851jOpqTz_vkWNJM,22266
 xinference/model/llm/transformers/cogvlm2.py,sha256=I5Ftm0VYjbTAv5ZARZCo32Ggpw58PJfHs5B_nX_BIlU,15972
 xinference/model/llm/transformers/cogvlm2_video.py,sha256=ZGkpC4x2uEtjwoMrLSODmAUYTjOeSNYxZi9VpQrpnhU,11857
 xinference/model/llm/transformers/compression.py,sha256=U0vMJ-JaBt4oC2LffgWg6HbPj1CeUi_YdwVbjDd0mRA,8112
-xinference/model/llm/transformers/core.py,sha256=rgRrqykyd4fpKNvMfmjAIApWjqvYO3HYB1wJ7cmB9S0,28229
+xinference/model/llm/transformers/core.py,sha256=8-BKAXWhBAQOnk1tDHwNQzq5x74-y1P9kdxO2D3ZhjQ,28247
 xinference/model/llm/transformers/deepseek_v2.py,sha256=-RKlI3mhja730md4evQ2vfIxBnZD5vWyrgmg_3eovms,4096
 xinference/model/llm/transformers/deepseek_vl.py,sha256=pB6i6DW5oyfHdqTgKpi2DkIKVGlPLGIDR_Op0sB1uKA,10445
 xinference/model/llm/transformers/glm4v.py,sha256=goph2HhpV8gUm2t8-T1P-jTF2r_kPeH6QNe64lmlm0g,13871
+xinference/model/llm/transformers/glm_edge_v.py,sha256=sedHB4iRd37UC__1MeXs33NLMQQKFYBIFf3A71rMEZU,8159
 xinference/model/llm/transformers/intern_vl.py,sha256=0pbze1eo3HvNQ0nW-mVJcJuJ4GrEyBBqQAYIdXnAn6c,18270
 xinference/model/llm/transformers/internlm2.py,sha256=3mjRgqU0RgCFF0F46ieVH0NO2JCKGsQkmkoVlWJrh8E,3221
 xinference/model/llm/transformers/minicpmv25.py,sha256=mr80-OlSlK_opSuAO3cz_QlkqujLr6V-OsTP0ebwpE8,6814
@@ -120,7 +122,7 @@ xinference/model/llm/transformers/tensorizer_utils.py,sha256=VXSYbPZtCbd8lVvsnjD
 xinference/model/llm/transformers/utils.py,sha256=Ej9Tu2yVAotfXMFsl30QlYXLZTODU6Pv_UppsGGUiSw,19185
 xinference/model/llm/transformers/yi_vl.py,sha256=iCdRLw-wizbU-qXXc8CT4DhC0Pt-uYg0vFwXEhAZjQg,8961
 xinference/model/llm/vllm/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/vllm/core.py,sha256=gflboRHy4JvhDG6G2bjPgidgNFTU2dDepbTZBmeDGlY,31516
+xinference/model/llm/vllm/core.py,sha256=ZPg4oJql8mhHMs5CpHo83Fm6peeKvB3EBXwaFNQQ3Cw,35185
 xinference/model/llm/vllm/utils.py,sha256=LKOmwfFRrlSecawxT-uE39tC2RQbf1UIiSH9Uz90X6w,1313
 xinference/model/rerank/__init__.py,sha256=wRpf1bOMfmAsuEKEGczMTB5fWEvuqltlJbIbRb-x8Ko,3483
 xinference/model/rerank/core.py,sha256=e-QoFgVk-6LOQPM5zqbEj095J-1bkuyd9c5zRI5DlF8,14560
@@ -15522,9 +15524,9 @@ xinference/web/ui/node_modules/yargs-parser/package.json,sha256=BSwbOzgetKXMK4u0
 xinference/web/ui/node_modules/yocto-queue/package.json,sha256=6U1XHQPGXJTqsiFvT953ORihUtXTblZy4fXBWP9qxC0,725
 xinference/web/ui/node_modules/yup/package.json,sha256=xRFSROB9NKxqSWHEVFvSTsPs9Ll074uo8OS1zEw0qhA,1206
 xinference/web/ui/node_modules/yup/node_modules/type-fest/package.json,sha256=JTv2zTTVgxQ2H82m1-6qEpdMv08lHjFx4Puf_MsbB_Q,1134
-xinference-1.0.0.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-1.0.0.dist-info/METADATA,sha256=H-KYRsy1S1Q2Wk_zY6Ei7xiBmaGfcOr4TqaC1NfBsnM,21824
-xinference-1.0.0.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-xinference-1.0.0.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-1.0.0.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-1.0.0.dist-info/RECORD,,
+xinference-1.0.1.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-1.0.1.dist-info/METADATA,sha256=pLzWAyI3EYcEd7FbPEOnh6qtrxo0LqjCv2EWxREy8ZU,21992
+xinference-1.0.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+xinference-1.0.1.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-1.0.1.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-1.0.1.dist-info/RECORD,,

{xinference-1.0.0.dist-info → xinference-1.0.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any