xinference 1.8.1rc1__py3-none-any.whl → 1.9.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-03T12:12:02+0800",
+ "date": "2025-08-30T03:57:39+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "2adec1c027044a920632ad7626f8f278eef83361",
- "version": "1.8.1.rc1"
+ "full-revisionid": "b2d793d0b4a0af632932eb63dbeb1bc91b5b3d74",
+ "version": "1.9.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -2249,8 +2249,9 @@ class RESTfulAPI(CancelMixin):
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
+            is_sglang = await model.is_sglang_backend()
             if not (
-                (is_vllm and model_family in QWEN_TOOL_CALL_FAMILY)
+                ((is_vllm or is_sglang) and model_family in QWEN_TOOL_CALL_FAMILY)
                 or (not is_vllm and model_family in GLM4_TOOL_CALL_FAMILY)
             ):
                 raise HTTPException(

xinference/core/model.py

@@ -365,6 +365,11 @@ class ModelActor(xo.StatelessActor, CancelMixin):
 
         return isinstance(self._model, VLLMModel)
 
+    def is_sglang_backend(self) -> bool:
+        from ..model.llm.sglang.core import SGLANGModel
+
+        return isinstance(self._model, SGLANGModel)
+
     async def load(self):
         try:
             # Change process title for model
@@ -877,10 +882,9 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         **kwargs,
     ):
         if hasattr(self._model, "text_to_image"):
-            # Directly delegate to model, let model decide how to handle (batching or not)
-            progressor = kwargs["progressor"] = await self._get_progressor(
-                kwargs.pop("request_id", None)
-            )
+            # Get progressor (don't pop request_id, let _call_wrapper handle cancellation)
+            request_id = kwargs.get("request_id")
+            progressor = kwargs["progressor"] = await self._get_progressor(request_id)  # type: ignore
             with progressor:
                 return await self._call_wrapper_json(
                     self._model.text_to_image,

xinference/core/supervisor.py

@@ -476,7 +476,7 @@ class SupervisorActor(xo.StatelessActor):
     async def _to_rerank_model_reg(
         self, model_spec: "RerankModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.cache_manager import CacheManager
+        from ..model.rerank.cache_manager import RerankCacheManager as CacheManager
 
         instance_cnt = await self.get_instance_count(model_spec.model_name)
         version_cnt = await self.get_model_version_count(model_spec.model_name)
@@ -712,9 +712,8 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.rerank import BUILTIN_RERANK_MODELS
             from ..model.rerank.custom import get_user_defined_reranks
 
-            for model_name, families in BUILTIN_RERANK_MODELS.items():
+            for model_name, family in BUILTIN_RERANK_MODELS.items():
                 if detailed:
-                    family = [x for x in families if x.model_hub == "huggingface"][0]
                     ret.append(await self._to_rerank_model_reg(family, is_builtin=True))
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})

xinference/core/worker.py

@@ -817,10 +817,7 @@ class WorkerActor(xo.StatelessActor):
             # we specify python_path explicitly
             # sometimes uv would find other versions.
             python_path = pathlib.Path(sys.executable)
-        kw = {}
-        if XINFERENCE_VIRTUAL_ENV_SKIP_INSTALLED:
-            kw["skip_installed"] = XINFERENCE_VIRTUAL_ENV_SKIP_INSTALLED
-        virtual_env_manager.create_env(python_path=python_path, **kw)
+        virtual_env_manager.create_env(python_path=python_path)
         return virtual_env_manager
 
     @classmethod
@@ -830,10 +827,13 @@ class WorkerActor(xo.StatelessActor):
         settings: Optional[VirtualEnvSettings],
         virtual_env_packages: Optional[List[str]],
     ):
-        if not settings or not settings.packages:
+        if (not settings or not settings.packages) and not virtual_env_packages:
             # no settings or no packages
             return
 
+        if settings is None:
+            settings = VirtualEnvSettings(packages=virtual_env_packages)
+
         if settings.inherit_pip_config:
             # inherit pip config
             pip_config = get_pip_config_args()
@@ -847,6 +847,8 @@ class WorkerActor(xo.StatelessActor):
             packages.extend(virtual_env_packages)
         conf.pop("packages", None)
         conf.pop("inherit_pip_config", None)
+        if XINFERENCE_VIRTUAL_ENV_SKIP_INSTALLED:
+            conf["skip_installed"] = XINFERENCE_VIRTUAL_ENV_SKIP_INSTALLED
 
         logger.info(
             "Installing packages %s in virtual env %s, with settings(%s)",

xinference/deploy/cmdline.py

@@ -1345,6 +1345,8 @@ def model_chat(
                     messages,
                     generate_config={"stream": stream, "max_tokens": max_tokens},
                 ):
+                    if not chunk["choices"]:
+                        continue
                     delta = chunk["choices"][0]["delta"]
                     if "content" not in delta:
                         continue

xinference/deploy/local.py

@@ -152,6 +152,11 @@ def main(
     logging_conf: Optional[Dict] = None,
     auth_config_file: Optional[str] = None,
 ):
+    # force to set spawn,
+    # cuda may be inited in xoscar virtualenv
+    # which will raise error after sub pool is created
+    multiprocessing.set_start_method("spawn")
+
     supervisor_address = f"{host}:{get_next_port()}"
     local_cluster = run_in_subprocess(
         supervisor_address, metrics_exporter_host, metrics_exporter_port, logging_conf

xinference/deploy/test/test_cmdline.py

@@ -87,7 +87,7 @@ def test_cmdline(setup, stream, model_uid):
         ],
     )
     assert result.exit_code == 0
-    assert model_uid in result.stdout
+    assert model_uid in result.output
 
     # model generate
     result = runner.invoke(

xinference/deploy/worker.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import logging
+import multiprocessing
 import os
 from typing import Any, Optional
 
@@ -81,6 +82,11 @@ def main(
     metrics_exporter_port: Optional[int] = None,
     logging_conf: Optional[dict] = None,
 ):
+    # force to set spawn,
+    # cuda may be inited in xoscar virtualenv
+    # which will raise error after sub pool is created
+    multiprocessing.set_start_method("spawn")
+
     loop = asyncio.get_event_loop()
     task = loop.create_task(
         _start_worker(

xinference/model/audio/cosyvoice.py

@@ -60,7 +60,6 @@ class CosyVoiceModel:
             from cosyvoice.cli.cosyvoice import CosyVoice2 as CosyVoice
 
             self._is_cosyvoice2 = True
-            kwargs = {"use_flow_cache": self._kwargs.get("use_flow_cache", False)}
         else:
             from cosyvoice.cli.cosyvoice import CosyVoice
 

xinference/model/audio/model_spec.json

@@ -525,7 +525,8 @@
     "model_name": "ChatTTS",
     "model_family": "ChatTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -551,7 +552,8 @@
     "model_name": "CosyVoice-300M",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -570,7 +572,8 @@
     "model_name": "CosyVoice-300M-SFT",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -589,7 +592,8 @@
     "model_name": "CosyVoice-300M-Instruct",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -608,7 +612,9 @@
     "model_name": "CosyVoice2-0.5B",
     "model_family": "CosyVoice",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "virtualenv": {
@@ -625,7 +631,8 @@
         "HyperPyYAML",
         "onnxruntime>=1.16.0",
         "pyworld>=0.3.4",
-        "WeTextProcessing<1.0.4",
+        "wetext==0.0.9",
+        "transformers==4.51.3",
         "#system_numpy#",
         "#system_torch#"
       ]
@@ -646,7 +653,9 @@
     "model_name": "FishSpeech-1.5",
     "model_family": "FishAudio",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -665,7 +674,9 @@
     "model_name": "F5-TTS",
     "model_family": "F5-TTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -684,7 +695,9 @@
     "model_name": "F5-TTS-MLX",
     "model_family": "F5-TTS-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning"
     ],
     "multilingual": true,
     "model_src": {
@@ -699,7 +712,8 @@
     "model_name": "MeloTTS-English",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -715,7 +729,8 @@
     "model_name": "MeloTTS-English-v2",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -731,7 +746,8 @@
     "model_name": "MeloTTS-English-v3",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "EN",
@@ -747,7 +763,8 @@
     "model_name": "MeloTTS-French",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "FR",
@@ -763,7 +780,8 @@
     "model_name": "MeloTTS-Japanese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "JP",
@@ -779,7 +797,8 @@
     "model_name": "MeloTTS-Spanish",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ES",
@@ -795,7 +814,8 @@
     "model_name": "MeloTTS-Chinese",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "ZH",
@@ -811,7 +831,8 @@
     "model_name": "MeloTTS-Korean",
     "model_family": "MeloTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": false,
     "language": "KR",
@@ -827,7 +848,8 @@
     "model_name": "Kokoro-82M",
     "model_family": "Kokoro",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -846,7 +868,8 @@
     "model_name": "Kokoro-82M-MLX",
     "model_family": "Kokoro-MLX",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {
@@ -874,7 +897,8 @@
     "model_name": "MegaTTS3",
     "model_family": "MegaTTS",
     "model_ability": [
-      "text2audio"
+      "text2audio",
+      "text2audio_zero_shot"
     ],
     "multilingual": true,
     "model_src": {

xinference/model/core.py

@@ -81,6 +81,9 @@ def create_model_instance(
         return create_rerank_model_instance(
             model_uid,
             model_name,
+            model_engine,
+            model_format,
+            quantization,
             download_hub,
             model_path,
             **kwargs,

xinference/model/embedding/flag/core.py

@@ -58,6 +58,11 @@ class FlagEmbeddingModel(EmbeddingModel):
         self._return_sparse = return_sparse
 
     def load(self):
+        # add truncate_dim args hint
+        if self._kwargs and "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "Flag embedder does not support dimensions argument now."
+            )
         try:
             from FlagEmbedding import BGEM3FlagModel
         except ImportError:

xinference/model/embedding/llama_cpp/core.py

@@ -22,7 +22,7 @@ import queue
 import sys
 from typing import List, Optional, Union
 
-import orjson
+from packaging import version
 
 from ....types import Embedding
 from ..core import EmbeddingModel, EmbeddingModelFamilyV2, EmbeddingSpecV1
@@ -69,15 +69,29 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
         return sys.platform.startswith("linux")
 
     def load(self):
+        # add truncate_dim args hint
+        if "dimensions" in self._kwargs:
+            raise NotImplementedError(
+                "LlamaCpp embedder does not support dimensions argument now."
+            )
         try:
             from xllamacpp import (
                 CommonParams,
                 Server,
+                __version__,
                 estimate_gpu_layers,
                 get_device_info,
                 ggml_backend_dev_type,
                 llama_pooling_type,
             )
+
+            try:
+                if version.parse(__version__) < version.parse("0.2.0"):
+                    raise RuntimeError(
+                        "Please update xllamacpp to >= 0.2.0 by `pip install -U xllamacpp`"
+                    )
+            except version.InvalidVersion:
+                pass  # If the version parse failed, we just skip the version check.
         except ImportError:
             error_message = "Failed to import module 'xllamacpp'"
             installation_guide = ["Please make sure 'xllamacpp' is installed. "]
@@ -162,7 +176,8 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
                         )
                         logger.info("Estimate num gpu layers: %s", estimate)
                         if estimate.tensor_split:
-                            params.tensor_split = estimate.tensor_split
+                            for i in range(len(estimate.tensor_split)):
+                                params.tensor_split[i] = estimate.tensor_split[i]
                         else:
                             params.n_gpu_layers = estimate.layers
                 except Exception as e:
@@ -190,24 +205,12 @@ class XllamaCppEmbeddingModel(EmbeddingModel):
             model_uid: Optional[str] = kwargs.pop("model_uid", None)
             if model_uid:
                 data["model"] = model_uid
-            prompt_json = orjson.dumps(data)
-
-            def _error_callback(err):
-                try:
-                    msg = orjson.loads(err)
-                    q.put(_Error(msg))
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
-            def _ok_callback(ok):
-                try:
-                    res = orjson.loads(ok)
-                    q.put(res)
-                except Exception as e:
-                    q.put(_Error(str(e)))
-
             try:
-                self._llm.handle_embeddings(prompt_json, _error_callback, _ok_callback)
+                res = self._llm.handle_embeddings(data)
+                if res.get("code"):
+                    q.put(_Error(res))
+                else:
+                    q.put(res)
             except Exception as ex:
                 q.put(_Error(str(ex)))
             q.put(_Done)

xinference/model/embedding/sentence_transformers/core.py

@@ -19,8 +19,8 @@ from typing import List, Optional, Union, no_type_check
 import numpy as np
 import torch
 
-from ....device_utils import is_device_available
 from ....types import Embedding, EmbeddingData, EmbeddingUsage
+from ...utils import is_flash_attn_available
 from ..core import EmbeddingModel, EmbeddingModelFamilyV2, EmbeddingSpecV1
 
 logger = logging.getLogger(__name__)
@@ -71,6 +71,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 )
                 torch_dtype = torch.float32
 
+        dimensions = self._kwargs.get("dimensions")
+        assert dimensions is None or isinstance(dimensions, int), (
+            "The `dimensions` argument must be an integer, "
+            f"but got {type(dimensions)}: {dimensions}"
+        )
+
         if (
             "gte" in self.model_family.model_name.lower()
             and "qwen2" in self.model_family.model_name.lower()
@@ -82,16 +88,16 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 self._model_path,
                 device=self._device,
                 model_kwargs=model_kwargs,
+                truncate_dim=dimensions,
             )
         elif "qwen3" in self.model_family.model_name.lower():
             # qwen3 embedding
-            flash_attn_installed = importlib.util.find_spec("flash_attn") is not None
             flash_attn_enabled = self._kwargs.get(
-                "enable_flash_attn", is_device_available("cuda")
+                "enable_flash_attn", is_flash_attn_available()
             )
             model_kwargs = {"device_map": "auto"}
             tokenizer_kwargs = {}
-            if flash_attn_installed and flash_attn_enabled:
+            if flash_attn_enabled:
                 model_kwargs["attn_implementation"] = "flash_attention_2"
                 model_kwargs["torch_dtype"] = "bfloat16"
                 tokenizer_kwargs["padding_side"] = "left"
@@ -107,6 +113,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 tokenizer_kwargs=tokenizer_kwargs,
+                truncate_dim=dimensions,
             )
         else:
             model_kwargs = {"torch_dtype": torch_dtype} if torch_dtype else None
@@ -115,6 +122,7 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 device=self._device,
                 model_kwargs=model_kwargs,
                 trust_remote_code=True,
+                truncate_dim=dimensions,
             )
 
         if hasattr(self._model, "tokenizer"):
@@ -271,6 +279,12 @@ class SentenceTransformerEmbeddingModel(EmbeddingModel):
                 with torch.no_grad():
                     out_features = model.forward(features, **kwargs)
 
+                    from sentence_transformers.util import truncate_embeddings
+
+                    out_features["sentence_embedding"] = truncate_embeddings(
+                        out_features["sentence_embedding"], model.truncate_dim
+                    )
+
                     if output_value == "token_embeddings":
                         embeddings = []
                         for token_emb, attention in zip(

xinference/model/embedding/vllm/core.py

@@ -17,6 +17,7 @@ import logging
 from typing import List, Union
 
 from ....types import Embedding, EmbeddingData, EmbeddingUsage
+from ...utils import cache_clean
 from ..core import EmbeddingModel, EmbeddingModelFamilyV2, EmbeddingSpecV1
 
 logger = logging.getLogger(__name__)
@@ -24,7 +25,6 @@ SUPPORTED_MODELS_PREFIXES = ["bge", "gte", "text2vec", "m3e", "gte", "Qwen3"]
 
 
 class VLLMEmbeddingModel(EmbeddingModel):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._context_length = None
@@ -41,28 +41,42 @@ class VLLMEmbeddingModel(EmbeddingModel):
             ]
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+        if self.model_family.model_name in {
+            "Qwen3-Embedding-0.6B",
+            "Qwen3-Embedding-4B",
+            "Qwen3-Embedding-8B",
+        }:
+            if "hf_overrides" not in self._kwargs:
+                self._kwargs["hf_overrides"] = {
+                    "is_matryoshka": True,
+                }
+            elif isinstance(self._kwargs["hf_overrides"], dict):
+                self._kwargs["hf_overrides"].update(
+                    is_matryoshka=True,
+                )
 
-        self._model = LLM(model=self._model_path, task="embed")
+        self._model = LLM(model=self._model_path, task="embed", **self._kwargs)
         self._tokenizer = self._model.get_tokenizer()
 
     @staticmethod
     def _get_detailed_instruct(task_description: str, query: str) -> str:
         return f"Instruct: {task_description}\nQuery:{query}"
 
+    @cache_clean
     def create_embedding(
         self,
         sentences: Union[str, List[str]],
         **kwargs,
     ):
+        from packaging.version import Version
+        from vllm import PoolingParams
+        from vllm import __version__ as vllm_version
+
         sentences = self._fix_langchain_openai_inputs(sentences)
         model_uid = kwargs.pop("model_uid", None)
 
         normalize_embedding = kwargs.get("normalize_embedding", True)
-        if not normalize_embedding:
-            raise ValueError(
-                "vllm embedding engine does not support "
-                "setting `normalize_embedding=False`"
-            )
+        dimensions = kwargs.get("dimensions", None)
 
         assert self._model is not None
 
@@ -91,8 +105,21 @@ class VLLMEmbeddingModel(EmbeddingModel):
                 sentences = truncated_sentences[0]
             else:
                 sentences = truncated_sentences
-
-        outputs = self._model.embed(sentences, use_tqdm=False)
+        if Version(vllm_version) > Version("0.10.1"):
+            pool_params = PoolingParams(
+                dimensions=dimensions, normalize=normalize_embedding
+            )
+        else:
+            if not normalize_embedding:
+                raise ValueError(
+                    f"vLLM version {vllm_version} does not support "
+                    f"unnormalized embeddings. "
+                    f"Please upgrade to v0.10.1 or later."
+                )
+            pool_params = PoolingParams(dimensions=dimensions)
+        outputs = self._model.embed(
+            sentences, use_tqdm=False, pooling_params=pool_params
+        )
         embedding_list = []
         all_token_nums = 0
         for index, output in enumerate(outputs):