xinference 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

xinference/model/audio/cosyvoice.py

@@ -60,7 +60,6 @@ class CosyVoiceModel:
             from cosyvoice.cli.cosyvoice import CosyVoice2 as CosyVoice
 
             self._is_cosyvoice2 = True
-            kwargs = {"use_flow_cache": self._kwargs.get("use_flow_cache", False)}
         else:
             from cosyvoice.cli.cosyvoice import CosyVoice
 

xinference/model/audio/kokoro.py

@@ -81,7 +81,7 @@ class KokoroModel:
         logger.info("Launching Kokoro model with language code: %s", lang_code)
         self._model = KPipeline(
             lang_code=lang_code,
-            model=KModel(config=config_path, model=model_path),
+            model=KModel(config=config_path, model=model_path).to(self._device),
             device=self._device,
         )
 

xinference/model/audio/kokoro_zh.py

@@ -0,0 +1,124 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+
+from ...device_utils import get_available_device, is_device_available
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV2
+
+logger = logging.getLogger(__name__)
+
+REPO_ID = "hexgrad/Kokoro-82M-v1.1-zh"
+
+
+class KokoroZHModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV2",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self.model_family = model_spec
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+        self._en_pipeline = None
+
+    def _en_callable(self, text):
+        """
+        Fixing the issue of English words being skipped in the Chinese model.
+        from https://hf-mirror.com/hexgrad/Kokoro-82M-v1.1-zh/blob/main/samples/make_zh.py
+        """
+        if text == "Kokoro":
+            return "kˈOkəɹO"
+        elif text == "Sol":
+            return "sˈOl"
+        return next(self._en_pipeline(text)).phonemes
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        if self._device is None:
+            self._device = get_available_device()
+        else:
+            if not is_device_available(self._device):
+                raise ValueError(f"Device {self._device} is not available!")
+
+        import os
+
+        from kokoro import KModel, KPipeline
+
+        self._en_pipeline = KPipeline(lang_code="a", repo_id=REPO_ID, model=False)
+
+        config_path = os.path.join(self._model_path, "config.json")
+        model_path = os.path.join(self._model_path, "kokoro-v1_1-zh.pth")
+        lang_code = self._kwargs.get("lang_code", "z")
+        logger.info("Launching Kokoro model with language code: %s", lang_code)
+
+        self._model = KPipeline(
+            lang_code=lang_code,
+            model=KModel(config=config_path, model=model_path).to(self._device),
+            repo_id=REPO_ID,
+            en_callable=self._en_callable,
+            device=self._device,
+        )
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        import soundfile
+
+        if stream:
+            raise Exception("Kokoro does not support stream mode.")
+        assert self._model is not None
+        if not voice:
+            voice = "zf_001"
+            logger.info("Auto select speaker: %s", voice)
+        elif voice.endswith(".pt"):
+            logger.info("Using custom voice pt: %s", voice)
+        else:
+            logger.info("Using voice: %s", voice)
+        logger.info("Speech kwargs: %s", kwargs)
+        generator = self._model(text=input, voice=voice, speed=speed, **kwargs)
+        results = list(generator)
+        audio = np.concatenate([r[2] for r in results])
+        # Save the generated audio
+        with BytesIO() as out:
+            with soundfile.SoundFile(
+                out,
+                "w",
+                24000,
+                1,
+                format=response_format.upper(),
+            ) as f:
+                f.write(audio)
+            return out.getvalue()

xinference/model/audio/model_spec.json

@@ -525,7 +525,8 @@
     "model_name": "ChatTTS",
     "model_family": "ChatTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -551,7 +552,8 @@
     "model_name": "CosyVoice-300M",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -570,7 +572,8 @@
     "model_name": "CosyVoice-300M-SFT",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -589,7 +592,8 @@
     "model_name": "CosyVoice-300M-Instruct",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -608,7 +612,9 @@
     "model_name": "CosyVoice2-0.5B",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -625,7 +631,8 @@
         "HyperPyYAML",
         "onnxruntime>=1.16.0",
         "pyworld>=0.3.4",
-        "WeTextProcessing<1.0.4",
+        "wetext==0.0.9",
+        "transformers==4.51.3",
         "#system_numpy#",
         "#system_torch#"
       ]
@@ -646,7 +653,9 @@
     "model_name": "FishSpeech-1.5",
     "model_family": "FishAudio",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -665,7 +674,9 @@
     "model_name": "F5-TTS",
     "model_family": "F5-TTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -684,7 +695,9 @@
     "model_name": "F5-TTS-MLX",
     "model_family": "F5-TTS-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -699,7 +712,8 @@
     "model_name": "MeloTTS-English",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -715,7 +729,8 @@
     "model_name": "MeloTTS-English-v2",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -731,7 +746,8 @@
     "model_name": "MeloTTS-English-v3",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -747,7 +763,8 @@
     "model_name": "MeloTTS-French",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "FR",
@@ -763,7 +780,8 @@
     "model_name": "MeloTTS-Japanese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "JP",
@@ -779,7 +797,8 @@
     "model_name": "MeloTTS-Spanish",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ES",
@@ -795,7 +814,8 @@
     "model_name": "MeloTTS-Chinese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ZH",
@@ -811,7 +831,8 @@
     "model_name": "MeloTTS-Korean",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "KR",
@@ -827,7 +848,8 @@
     "model_name": "Kokoro-82M",
     "model_family": "Kokoro",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -840,13 +862,34 @@
         "model_revision": "master"
       }
     }
+  },
+    {
+    "version": 2,
+    "model_name": "Kokoro-82M-v1.1-zh",
+    "model_family": "Kokoro-zh",
+    "model_ability": [
+      "text2audio",
+      "text2audio_zero_shot"
+    ],
+    "multilingual": false,
+    "model_src": {
+      "huggingface": {
+        "model_id": "hexgrad/Kokoro-82M-v1.1-zh",
+        "model_revision": "01e7505bd6a7a2ac4975463114c3a7650a9f7218"
+      },
+      "modelscope": {
+        "model_id": "AI-ModelScope/Kokoro-82M-v1.1-zh",
+        "model_revision": "master"
+      }
+    }
   },
   {
     "version": 2,
     "model_name": "Kokoro-82M-MLX",
     "model_family": "Kokoro-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -874,7 +917,8 @@
     "model_name": "MegaTTS3",
     "model_family": "MegaTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {

xinference/model/embedding/flag/core.py

@@ -58,6 +58,11 @@ class FlagEmbeddingModel(EmbeddingModel):
         self._return_sparse = return_sparse
 
     def load(self):
+        # add truncate_dim args hint
+        if self._kwargs and "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "Flag embedder does not support dimensions argument now."
+            )
         try:
             from FlagEmbedding import BGEM3FlagModel
         except ImportError:

xinference/model/embedding/llama_cpp/core.py

@@ -22,7 +22,7 @@ import queue
 import sys
 from typing import List, Optional, Union
 
-import orjson
+from packaging import version
 
 from ....types import Embedding
 from ..core import EmbeddingModel, EmbeddingModelFamilyV2, EmbeddingSpecV1
@@ -69,15 +69,29 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
         return sys.platform.startswith("linux")
 
     def load(self):
+        # add truncate_dim args hint
+        if "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "LlamaCpp embedder does not support dimensions argument now."
+            )
         try:
             from xllamacpp import (
                 CommonParams,
                 Server,
+                __version__,
                 estimate_gpu_layers,
                 get_device_info,
                 ggml_backend_dev_type,
                 llama_pooling_type,
             )
+
+            try:
+                if version.parse(__version__) < version.parse("0.2.0"):
+                    raise RuntimeError(
+                        "Please update xllamacpp to >= 0.2.0 by `pip install -U xllamacpp`"
+                    )
+            except version.InvalidVersion:
+                pass  # If the version parse failed, we just skip the version check.
         except ImportError:
             error_message = "Failed to import module 'xllamacpp'"
             installation_guide = ["Please make sure 'xllamacpp' is installed. "]
@@ -162,7 +176,8 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
                         )
                         logger.info("Estimate num gpu layers: %s", estimate)
                         if estimate.tensor_split:
-                            params.tensor_split = estimate.tensor_split
+                            for i in range(len(estimate.tensor_split)):
+                                params.tensor_split[i] = estimate.tensor_split[i]
                         else:
                             params.n_gpu_layers = estimate.layers
                 except Exception as e:
@@ -190,24 +205,12 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
             model_uid: Optional[str] = kwargs.pop("model_uid", None)
             if model_uid:
                 data["model"] = model_uid
-            prompt_json = orjson.dumps(data)
-
-            def _error_callback(err):
-                try:
-                    msg = orjson.loads(err)
-                    q.put(_Error(msg))
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
-            def _ok_callback(ok):
-                try:
-                    res = orjson.loads(ok)
-                    q.put(res)
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
             try:
-                self._llm.handle_embeddings(prompt_json, _error_callback, _ok_callback)
+                res = self._llm.handle_embeddings(data)
+                if res.get("code"):
+                    q.put(_Error(res))
+                else:
+                    q.put(res)
             except Exception as ex:
                 q.put(_Error(str(ex)))
             q.put(_Done)

xinference/model/embedding/sentence_transformers/core.py

@@ -71,6 +71,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 )
                 torch_dtype = torch.float32
 
+        dimensions = self._kwargs.get("dimensions")
+        assert dimensions is None or isinstance(dimensions, int), (
+            "The `dimensions` argument must be an integer, "
+            f"but got {type(dimensions)}: {dimensions}"
+        )
+
         if (
             "gte" in self.model_family.model_name.lower()
             and "qwen2" in self.model_family.model_name.lower()
@@ -82,6 +88,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 self._model_path,
                 device=self._device,
                 model_kwargs=model_kwargs,
+                truncate_dim=dimensions,
             )
         elif "qwen3" in self.model_family.model_name.lower():
             # qwen3 embedding
@@ -106,6 +113,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 tokenizer_kwargs=tokenizer_kwargs,
+                truncate_dim=dimensions,
             )
         else:
             model_kwargs = {"torch_dtype": torch_dtype} if torch_dtype else None
@@ -114,6 +122,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 trust_remote_code=True,
+                truncate_dim=dimensions,
             )
 
         if hasattr(self._model, "tokenizer"):
@@ -256,10 +265,10 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                     "clip" in self.model_family.model_name.lower()
                     or "jina-embeddings-v4" in self.model_family.model_name.lower()
                 ):
-                    if "input_ids" in features and hasattr(
-                        features["input_ids"], "numel"
-                    ):
-                        all_token_nums += features["input_ids"].numel()
+                    # support input_ids and text_input_ids
+                    for key in ["input_ids", "text_input_ids"]:
+                        if key in features and hasattr(features[key], "numel"):
+                            all_token_nums += features[key].numel()
                     if "pixel_values" in features and hasattr(
                         features["pixel_values"], "numel"
                     ):
@@ -270,6 +279,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 with torch.no_grad():
                     out_features = model.forward(features, **kwargs)
 
+                    from sentence_transformers.util import truncate_embeddings
+
+                    out_features["sentence_embedding"] = truncate_embeddings(
+                        out_features["sentence_embedding"], model.truncate_dim
+                    )
+
                     if output_value == "token_embeddings":
                         embeddings = []
                         for token_emb, attention in zip(

xinference/model/embedding/vllm/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import importlib.util
+import json
 import logging
 from typing import List, Union
 
@@ -25,7 +26,6 @@ SUPPORTED_MODELS_PREFIXES = ["bge", "gte", "text2vec", "m3e", "gte", "Qwen3"]
 
 
 class VLLMEmbeddingModel(EmbeddingModel):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._context_length = None
@@ -42,13 +42,31 @@ class VLLMEmbeddingModel(EmbeddingModel):
             ]
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+        if self.model_family.model_name in {
+            "Qwen3-Embedding-0.6B",
+            "Qwen3-Embedding-4B",
+            "Qwen3-Embedding-8B",
+        }:
+            if "hf_overrides" not in self._kwargs:
+                self._kwargs["hf_overrides"] = {
+                    "is_matryoshka": True,
+                }
+            elif isinstance(self._kwargs["hf_overrides"], dict):
+                self._kwargs["hf_overrides"].update(
+                    is_matryoshka=True,
+                )
+            elif isinstance(self._kwargs["hf_overrides"], str):
+                self._kwargs["hf_overrides"] = json.loads(self._kwargs["hf_overrides"])
+                self._kwargs["hf_overrides"].update(
+                    is_matryoshka=True,
+                )
 
         self._model = LLM(model=self._model_path, task="embed", **self._kwargs)
         self._tokenizer = self._model.get_tokenizer()
 
     @staticmethod
     def _get_detailed_instruct(task_description: str, query: str) -> str:
-        return f"Instruct: {task_description}\nQuery:{query}"
+        return f"Instruct: {task_description}\nQuery:{query}"  # noqa: E231
 
     @cache_clean
     def create_embedding(
@@ -56,14 +74,15 @@ class VLLMEmbeddingModel(EmbeddingModel):
         sentences: Union[str, List[str]],
         **kwargs,
     ):
+        from packaging.version import Version
+        from vllm import PoolingParams
+        from vllm import __version__ as vllm_version
+
         sentences = self._fix_langchain_openai_inputs(sentences)
         model_uid = kwargs.pop("model_uid", None)
 
         normalize_embedding = kwargs.get("normalize_embedding", True)
-        if not normalize_embedding:
-            raise ValueError(
-                "vllm embedding engine does not support setting `normalize_embedding=False`"
-            )
+        dimensions = kwargs.get("dimensions", None)
 
         assert self._model is not None
 
@@ -92,8 +111,21 @@ class VLLMEmbeddingModel(EmbeddingModel):
                 sentences = truncated_sentences[0]
             else:
                 sentences = truncated_sentences
-
-        outputs = self._model.embed(sentences, use_tqdm=False)
+        if Version(vllm_version) > Version("0.10.1"):
+            pool_params = PoolingParams(
+                dimensions=dimensions, normalize=normalize_embedding
+            )
+        else:
+            if not normalize_embedding:
+                raise ValueError(
+                    f"vLLM version {vllm_version} does not support "
+                    f"unnormalized embeddings. "
+                    f"Please upgrade to v0.10.1 or later."
+                )
+            pool_params = PoolingParams(dimensions=dimensions)
+        outputs = self._model.embed(
+            sentences, use_tqdm=False, pooling_params=pool_params
+        )
         embedding_list = []
         all_token_nums = 0
         for index, output in enumerate(outputs):

xinference/model/image/cache_manager.py

@@ -60,3 +60,59 @@ class ImageCacheManager(CacheManager):
             raise NotImplementedError
 
         return full_path
+
+    def cache_lightning(self, lightning_version: Optional[str] = None):
+        from ..utils import IS_NEW_HUGGINGFACE_HUB, retry_download, symlink_local_file
+        from .core import ImageModelFamilyV2
+
+        if not lightning_version:
+            return None
+
+        assert isinstance(self._model_family, ImageModelFamilyV2)
+        cache_dir = self.get_cache_dir()
+
+        if not self._model_family.lightning_model_file_name_template:
+            raise NotImplementedError(
+                f"{self._model_family.model_name} does not support lightning"
+            )
+        if lightning_version not in (self._model_family.lightning_versions or []):
+            raise ValueError(
+                f"Cannot support lightning version {lightning_version}, "
+                f"available lightning version: {self._model_family.lightning_versions}"
+            )
+
+        filename = self._model_family.lightning_model_file_name_template.format(lightning_version=lightning_version)  # type: ignore
+        full_path = os.path.join(cache_dir, filename)
+
+        if self._model_family.model_hub == "huggingface":
+            import huggingface_hub
+
+            use_symlinks = {}
+            if not IS_NEW_HUGGINGFACE_HUB:
+                use_symlinks = {"local_dir_use_symlinks": True, "local_dir": cache_dir}
+            download_file_path = retry_download(
+                huggingface_hub.hf_hub_download,
+                self._model_family.model_name,
+                None,
+                self._model_family.lightning_model_id,
+                filename=filename,
+                **use_symlinks,
+            )
+            if IS_NEW_HUGGINGFACE_HUB:
+                symlink_local_file(download_file_path, cache_dir, filename)
+        elif self._model_family.model_hub == "modelscope":
+            from modelscope.hub.file_download import model_file_download
+
+            download_file_path = retry_download(
+                model_file_download,
+                self._model_family.model_name,
+                None,
+                self._model_family.lightning_model_id,
+                filename,
+                revision=self._model_family.model_revision,
+            )
+            symlink_local_file(download_file_path, cache_dir, filename)
+        else:
+            raise NotImplementedError
+
+        return full_path