xinference 1.9.0__py3-none-any.whl → 1.9.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-16T21:34:08+0800",
+ "date": "2025-08-30T03:57:39+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "38e0401f83799f57d42ef948c57782466b8e4777",
- "version": "1.9.0"
+ "full-revisionid": "b2d793d0b4a0af632932eb63dbeb1bc91b5b3d74",
+ "version": "1.9.1"
 }
 '''  # END VERSION_JSON
 

xinference/core/model.py

@@ -882,10 +882,9 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         **kwargs,
     ):
         if hasattr(self._model, "text_to_image"):
-            # Directly delegate to model, let model decide how to handle (batching or not)
-            progressor = kwargs["progressor"] = await self._get_progressor(
-                kwargs.pop("request_id", None)
-            )
+            # Get progressor (don't pop request_id, let _call_wrapper handle cancellation)
+            request_id = kwargs.get("request_id")
+            progressor = kwargs["progressor"] = await self._get_progressor(request_id)  # type: ignore
             with progressor:
                 return await self._call_wrapper_json(
                     self._model.text_to_image,

xinference/core/worker.py

@@ -827,10 +827,13 @@ class WorkerActor(xo.StatelessActor):
         settings: Optional[VirtualEnvSettings],
         virtual_env_packages: Optional[List[str]],
     ):
-        if not settings or not settings.packages:
+        if (not settings or not settings.packages) and not virtual_env_packages:
             # no settings or no packages
             return
 
+        if settings is None:
+            settings = VirtualEnvSettings(packages=virtual_env_packages)
+
         if settings.inherit_pip_config:
             # inherit pip config
             pip_config = get_pip_config_args()

xinference/deploy/cmdline.py

@@ -1345,6 +1345,8 @@ def model_chat(
                     messages,
                     generate_config={"stream": stream, "max_tokens": max_tokens},
                 ):
+                    if not chunk["choices"]:
+                        continue
                     delta = chunk["choices"][0]["delta"]
                     if "content" not in delta:
                         continue

xinference/deploy/test/test_cmdline.py

@@ -87,7 +87,7 @@ def test_cmdline(setup, stream, model_uid):
         ],
     )
     assert result.exit_code == 0
-    assert model_uid in result.stdout
+    assert model_uid in result.output
 
     # model generate
     result = runner.invoke(

xinference/model/audio/cosyvoice.py

@@ -60,7 +60,6 @@ class CosyVoiceModel:
             from cosyvoice.cli.cosyvoice import CosyVoice2 as CosyVoice
 
             self._is_cosyvoice2 = True
-            kwargs = {"use_flow_cache": self._kwargs.get("use_flow_cache", False)}
         else:
             from cosyvoice.cli.cosyvoice import CosyVoice
 

xinference/model/audio/model_spec.json

@@ -525,7 +525,8 @@
     "model_name": "ChatTTS",
     "model_family": "ChatTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -551,7 +552,8 @@
     "model_name": "CosyVoice-300M",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -570,7 +572,8 @@
     "model_name": "CosyVoice-300M-SFT",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -589,7 +592,8 @@
     "model_name": "CosyVoice-300M-Instruct",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -608,7 +612,9 @@
     "model_name": "CosyVoice2-0.5B",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -625,7 +631,8 @@
         "HyperPyYAML",
         "onnxruntime>=1.16.0",
         "pyworld>=0.3.4",
-        "WeTextProcessing<1.0.4",
+        "wetext==0.0.9",
+        "transformers==4.51.3",
         "#system_numpy#",
         "#system_torch#"
       ]
@@ -646,7 +653,9 @@
     "model_name": "FishSpeech-1.5",
     "model_family": "FishAudio",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -665,7 +674,9 @@
     "model_name": "F5-TTS",
     "model_family": "F5-TTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -684,7 +695,9 @@
     "model_name": "F5-TTS-MLX",
     "model_family": "F5-TTS-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -699,7 +712,8 @@
     "model_name": "MeloTTS-English",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -715,7 +729,8 @@
     "model_name": "MeloTTS-English-v2",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -731,7 +746,8 @@
     "model_name": "MeloTTS-English-v3",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -747,7 +763,8 @@
     "model_name": "MeloTTS-French",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "FR",
@@ -763,7 +780,8 @@
     "model_name": "MeloTTS-Japanese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "JP",
@@ -779,7 +797,8 @@
     "model_name": "MeloTTS-Spanish",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ES",
@@ -795,7 +814,8 @@
     "model_name": "MeloTTS-Chinese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ZH",
@@ -811,7 +831,8 @@
     "model_name": "MeloTTS-Korean",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "KR",
@@ -827,7 +848,8 @@
     "model_name": "Kokoro-82M",
     "model_family": "Kokoro",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -846,7 +868,8 @@
     "model_name": "Kokoro-82M-MLX",
     "model_family": "Kokoro-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -874,7 +897,8 @@
     "model_name": "MegaTTS3",
     "model_family": "MegaTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {

xinference/model/embedding/flag/core.py

@@ -58,6 +58,11 @@ class FlagEmbeddingModel(EmbeddingModel):
         self._return_sparse = return_sparse
 
     def load(self):
+        # add truncate_dim args hint
+        if self._kwargs and "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "Flag embedder does not support dimensions argument now."
+            )
         try:
             from FlagEmbedding import BGEM3FlagModel
         except ImportError:

xinference/model/embedding/llama_cpp/core.py

@@ -22,7 +22,7 @@ import queue
 import sys
 from typing import List, Optional, Union
 
-import orjson
+from packaging import version
 
 from ....types import Embedding
 from ..core import EmbeddingModel, EmbeddingModelFamilyV2, EmbeddingSpecV1
@@ -69,15 +69,29 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
         return sys.platform.startswith("linux")
 
     def load(self):
+        # add truncate_dim args hint
+        if "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "LlamaCpp embedder does not support dimensions argument now."
+            )
         try:
             from xllamacpp import (
                 CommonParams,
                 Server,
+                __version__,
                 estimate_gpu_layers,
                 get_device_info,
                 ggml_backend_dev_type,
                 llama_pooling_type,
             )
+
+            try:
+                if version.parse(__version__) < version.parse("0.2.0"):
+                    raise RuntimeError(
+                        "Please update xllamacpp to >= 0.2.0 by `pip install -U xllamacpp`"
+                    )
+            except version.InvalidVersion:
+                pass  # If the version parse failed, we just skip the version check.
         except ImportError:
             error_message = "Failed to import module 'xllamacpp'"
             installation_guide = ["Please make sure 'xllamacpp' is installed. "]
@@ -162,7 +176,8 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
                         )
                         logger.info("Estimate num gpu layers: %s", estimate)
                         if estimate.tensor_split:
-                            params.tensor_split = estimate.tensor_split
+                            for i in range(len(estimate.tensor_split)):
+                                params.tensor_split[i] = estimate.tensor_split[i]
                         else:
                             params.n_gpu_layers = estimate.layers
                 except Exception as e:
@@ -190,24 +205,12 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
             model_uid: Optional[str] = kwargs.pop("model_uid", None)
             if model_uid:
                 data["model"] = model_uid
-            prompt_json = orjson.dumps(data)
-
-            def _error_callback(err):
-                try:
-                    msg = orjson.loads(err)
-                    q.put(_Error(msg))
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
-            def _ok_callback(ok):
-                try:
-                    res = orjson.loads(ok)
-                    q.put(res)
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
             try:
-                self._llm.handle_embeddings(prompt_json, _error_callback, _ok_callback)
+                res = self._llm.handle_embeddings(data)
+                if res.get("code"):
+                    q.put(_Error(res))
+                else:
+                    q.put(res)
             except Exception as ex:
                 q.put(_Error(str(ex)))
             q.put(_Done)

xinference/model/embedding/sentence_transformers/core.py

@@ -71,6 +71,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 )
                 torch_dtype = torch.float32
 
+        dimensions = self._kwargs.get("dimensions")
+        assert dimensions is None or isinstance(dimensions, int), (
+            "The `dimensions` argument must be an integer, "
+            f"but got {type(dimensions)}: {dimensions}"
+        )
+
         if (
             "gte" in self.model_family.model_name.lower()
             and "qwen2" in self.model_family.model_name.lower()
@@ -82,6 +88,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 self._model_path,
                 device=self._device,
                 model_kwargs=model_kwargs,
+                truncate_dim=dimensions,
             )
         elif "qwen3" in self.model_family.model_name.lower():
             # qwen3 embedding
@@ -106,6 +113,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 tokenizer_kwargs=tokenizer_kwargs,
+                truncate_dim=dimensions,
             )
         else:
             model_kwargs = {"torch_dtype": torch_dtype} if torch_dtype else None
@@ -114,6 +122,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 trust_remote_code=True,
+                truncate_dim=dimensions,
             )
 
         if hasattr(self._model, "tokenizer"):
@@ -270,6 +279,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 with torch.no_grad():
                     out_features = model.forward(features, **kwargs)
 
+                    from sentence_transformers.util import truncate_embeddings
+
+                    out_features["sentence_embedding"] = truncate_embeddings(
+                        out_features["sentence_embedding"], model.truncate_dim
+                    )
+
                     if output_value == "token_embeddings":
                         embeddings = []
                         for token_emb, attention in zip(

xinference/model/embedding/vllm/core.py

@@ -25,7 +25,6 @@ SUPPORTED_MODELS_PREFIXES = ["bge", "gte", "text2vec", "m3e", "gte", "Qwen3"]
 
 
 class VLLMEmbeddingModel(EmbeddingModel):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._context_length = None
@@ -42,6 +41,19 @@ class VLLMEmbeddingModel(EmbeddingModel):
             ]
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+        if self.model_family.model_name in {
+            "Qwen3-Embedding-0.6B",
+            "Qwen3-Embedding-4B",
+            "Qwen3-Embedding-8B",
+        }:
+            if "hf_overrides" not in self._kwargs:
+                self._kwargs["hf_overrides"] = {
+                    "is_matryoshka": True,
+                }
+            elif isinstance(self._kwargs["hf_overrides"], dict):
+                self._kwargs["hf_overrides"].update(
+                    is_matryoshka=True,
+                )
 
         self._model = LLM(model=self._model_path, task="embed", **self._kwargs)
         self._tokenizer = self._model.get_tokenizer()
@@ -56,14 +68,15 @@ class VLLMEmbeddingModel(EmbeddingModel):
         sentences: Union[str, List[str]],
         **kwargs,
     ):
+        from packaging.version import Version
+        from vllm import PoolingParams
+        from vllm import __version__ as vllm_version
+
         sentences = self._fix_langchain_openai_inputs(sentences)
         model_uid = kwargs.pop("model_uid", None)
 
         normalize_embedding = kwargs.get("normalize_embedding", True)
-        if not normalize_embedding:
-            raise ValueError(
-                "vllm embedding engine does not support setting `normalize_embedding=False`"
-            )
+        dimensions = kwargs.get("dimensions", None)
 
         assert self._model is not None
 
@@ -92,8 +105,21 @@ class VLLMEmbeddingModel(EmbeddingModel):
                 sentences = truncated_sentences[0]
             else:
                 sentences = truncated_sentences
-
-        outputs = self._model.embed(sentences, use_tqdm=False)
+        if Version(vllm_version) > Version("0.10.1"):
+            pool_params = PoolingParams(
+                dimensions=dimensions, normalize=normalize_embedding
+            )
+        else:
+            if not normalize_embedding:
+                raise ValueError(
+                    f"vLLM version {vllm_version} does not support "
+                    f"unnormalized embeddings. "
+                    f"Please upgrade to v0.10.1 or later."
+                )
+            pool_params = PoolingParams(dimensions=dimensions)
+        outputs = self._model.embed(
+            sentences, use_tqdm=False, pooling_params=pool_params
+        )
         embedding_list = []
         all_token_nums = 0
         for index, output in enumerate(outputs):

xinference/model/image/cache_manager.py

@@ -60,3 +60,59 @@ class ImageCacheManager(CacheManager):
             raise NotImplementedError
 
         return full_path
+
+    def cache_lightning(self, lightning_version: Optional[str] = None):
+        from ..utils import IS_NEW_HUGGINGFACE_HUB, retry_download, symlink_local_file
+        from .core import ImageModelFamilyV2
+
+        if not lightning_version:
+            return None
+
+        assert isinstance(self._model_family, ImageModelFamilyV2)
+        cache_dir = self.get_cache_dir()
+
+        if not self._model_family.lightning_model_file_name_template:
+            raise NotImplementedError(
+                f"{self._model_family.model_name} does not support lightning"
+            )
+        if lightning_version not in (self._model_family.lightning_versions or []):
+            raise ValueError(
+                f"Cannot support lightning version {lightning_version}, "
+                f"available lightning version: {self._model_family.lightning_versions}"
+            )
+
+        filename = self._model_family.lightning_model_file_name_template.format(lightning_version=lightning_version)  # type: ignore
+        full_path = os.path.join(cache_dir, filename)
+
+        if self._model_family.model_hub == "huggingface":
+            import huggingface_hub
+
+            use_symlinks = {}
+            if not IS_NEW_HUGGINGFACE_HUB:
+                use_symlinks = {"local_dir_use_symlinks": True, "local_dir": cache_dir}
+            download_file_path = retry_download(
+                huggingface_hub.hf_hub_download,
+                self._model_family.model_name,
+                None,
+                self._model_family.lightning_model_id,
+                filename=filename,
+                **use_symlinks,
+            )
+            if IS_NEW_HUGGINGFACE_HUB:
+                symlink_local_file(download_file_path, cache_dir, filename)
+        elif self._model_family.model_hub == "modelscope":
+            from modelscope.hub.file_download import model_file_download
+
+            download_file_path = retry_download(
+                model_file_download,
+                self._model_family.model_name,
+                None,
+                self._model_family.lightning_model_id,
+                filename,
+                revision=self._model_family.model_revision,
+            )
+            symlink_local_file(download_file_path, cache_dir, filename)
+        else:
+            raise NotImplementedError
+
+        return full_path

xinference/model/image/core.py

@@ -51,6 +51,10 @@ class ImageModelFamilyV2(CacheableModelSpec, ModelInstanceInfoMixin):
     gguf_model_id: Optional[str]
     gguf_quantizations: Optional[List[str]]
     gguf_model_file_name_template: Optional[str]
+    lightning_model_id: Optional[str]
+    lightning_versions: Optional[List[str]]
+    lightning_model_file_name_template: Optional[str]
+
     virtualenv: Optional[VirtualEnvSettings]
 
     class Config:
@@ -180,6 +184,8 @@ def create_image_model_instance(
     model_path: Optional[str] = None,
     gguf_quantization: Optional[str] = None,
     gguf_model_path: Optional[str] = None,
+    lightning_version: Optional[str] = None,
+    lightning_model_path: Optional[str] = None,
     **kwargs,
 ) -> Union[DiffusionModel, MLXDiffusionModel, GotOCR2Model]:
     from .cache_manager import ImageCacheManager
@@ -235,6 +241,8 @@ def create_image_model_instance(
         model_path = cache_manager.cache()
     if not gguf_model_path and gguf_quantization:
         gguf_model_path = cache_manager.cache_gguf(gguf_quantization)
+    if not lightning_model_path and lightning_version:
+        lightning_model_path = cache_manager.cache_lightning(lightning_version)
     if peft_model_config is not None:
         lora_model = peft_model_config.peft_model
         lora_load_kwargs = peft_model_config.image_lora_load_kwargs
@@ -262,6 +270,7 @@ def create_image_model_instance(
         lora_fuse_kwargs=lora_fuse_kwargs,
         model_spec=model_spec,
         gguf_model_path=gguf_model_path,
+        lightning_model_path=lightning_model_path,
         **kwargs,
     )
     return model