xinference 0.15.1__py3-none-any.whl → 0.15.3__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-09-14T13:22:13+0800",
+ "date": "2024-09-30T20:17:26+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "961d355102007e3cd7963a353105b2422a31d4fd",
- "version": "0.15.1"
+ "full-revisionid": "00a9ee15279a60a6d75393c4720d8da5cbbf5796",
+ "version": "0.15.3"
 }
 '''  # END VERSION_JSON
 

xinference/core/model.py

@@ -769,7 +769,7 @@ class ModelActor(xo.StatelessActor):
         self,
         image: "PIL.Image",
         prompt: str,
-        negative_prompt: str,
+        negative_prompt: Optional[str] = None,
         n: int = 1,
         size: Optional[str] = None,
         response_format: str = "url",
@@ -777,12 +777,12 @@ class ModelActor(xo.StatelessActor):
         **kwargs,
     ):
         kwargs.pop("request_id", None)
+        kwargs["negative_prompt"] = negative_prompt
         if hasattr(self._model, "image_to_image"):
             return await self._call_wrapper_json(
                 self._model.image_to_image,
                 image,
                 prompt,
-                negative_prompt,
                 n,
                 size,
                 response_format,

xinference/model/audio/cosyvoice.py

@@ -122,10 +122,10 @@ class CosyVoiceModel:
                             last_pos = new_last_pos
 
         def _generator_block():
-            chunk = next(output)
-            assert isinstance(chunk, dict), "Expected data to be of type dict"
+            chunks = [o["tts_speech"] for o in output]
+            t = torch.cat(chunks, dim=1)
             with BytesIO() as out:
-                torchaudio.save(out, chunk["tts_speech"], 22050, format=response_format)
+                torchaudio.save(out, t, 22050, format=response_format)
                 return out.getvalue()
 
         return _generator_stream() if stream else _generator_block()

xinference/model/embedding/core.py

@@ -141,7 +141,15 @@ class EmbeddingModel:
 
     def load(self):
         try:
+            import sentence_transformers
             from sentence_transformers import SentenceTransformer
+
+            if sentence_transformers.__version__ < "3.1.0":
+                raise ValueError(
+                    "The sentence_transformers version must be greater than 3.1.0. "
+                    "Please upgrade your version via `pip install -U sentence_transformers` or refer to "
+                    "https://github.com/UKPLab/sentence-transformers"
+                )
         except ImportError:
             error_message = "Failed to import module 'SentenceTransformer'"
             installation_guide = [
@@ -173,9 +181,6 @@ class EmbeddingModel:
                 )
                 torch_dtype = torch.float32
 
-        from ..utils import patch_trust_remote_code
-
-        patch_trust_remote_code()
         if (
             "gte" in self._model_spec.model_name.lower()
             and "qwen2" in self._model_spec.model_name.lower()
@@ -191,7 +196,10 @@ class EmbeddingModel:
         else:
             model_kwargs = {"torch_dtype": torch_dtype} if torch_dtype else None
             self._model = SentenceTransformer(
-                self._model_path, device=self._device, model_kwargs=model_kwargs
+                self._model_path,
+                device=self._device,
+                model_kwargs=model_kwargs,
+                trust_remote_code=True,
             )
 
     def create_embedding(self, sentences: Union[str, List[str]], **kwargs):
@@ -213,6 +221,7 @@ class EmbeddingModel:
             convert_to_tensor: bool = False,
             device: str = None,
             normalize_embeddings: bool = False,
+            **kwargs,
         ):
             """
             Computes sentence embeddings
@@ -317,7 +326,7 @@ class EmbeddingModel:
                 all_token_nums += features["attention_mask"].sum().item()
 
                 with torch.no_grad():
-                    out_features = model.forward(features)
+                    out_features = model.forward(features, **kwargs)
 
                     if output_value == "token_embeddings":
                         embeddings = []

xinference/model/embedding/model_spec.json

@@ -238,5 +238,12 @@
     "language": ["zh", "en"],
     "model_id": "Alibaba-NLP/gte-Qwen2-7B-instruct",
     "model_revision": "e26182b2122f4435e8b3ebecbf363990f409b45b"
+  },
+  {
+    "model_name": "jina-embeddings-v3",
+    "dimensions": 1024,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "jinaai/jina-embeddings-v3"
   }
 ]

xinference/model/embedding/model_spec_modelscope.json

@@ -233,12 +233,20 @@
     "model_id": "AI-ModelScope/m3e-large",
     "model_hub": "modelscope"
   },
-    {
+  {
     "model_name": "gte-Qwen2",
     "dimensions": 4096,
     "max_tokens": 32000,
     "language": ["zh", "en"],
     "model_id": "iic/gte_Qwen2-7B-instruct",
     "model_hub": "modelscope"
+  },
+  {
+    "model_name": "jina-embeddings-v3",
+    "dimensions": 1024,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "jinaai/jina-embeddings-v3",
+    "model_hub": "modelscope"
   }
 ]

xinference/model/image/stable_diffusion/core.py

@@ -21,6 +21,7 @@ import re
 import sys
 import time
 import uuid
+import warnings
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from io import BytesIO
@@ -31,7 +32,7 @@ import torch
 from PIL import ImageOps
 
 from ....constants import XINFERENCE_IMAGE_DIR
-from ....device_utils import move_model_to_available_device
+from ....device_utils import get_available_device, move_model_to_available_device
 from ....types import Image, ImageList, LoRA
 from ..sdapi import SDAPIDiffusionModelMixin
 
@@ -60,6 +61,23 @@ SAMPLING_METHODS = [
 ]
 
 
+def model_accept_param(params: Union[str, List[str]], model: Any) -> bool:
+    params = [params] if isinstance(params, str) else params
+    # model is diffusers Pipeline
+    parameters = inspect.signature(model.__call__).parameters  # type: ignore
+    allow_params = False
+    for param in parameters.values():
+        if param.kind == inspect.Parameter.VAR_KEYWORD:
+            # the __call__ can accept **kwargs,
+            # we treat it as it can accept any parameters
+            allow_params = True
+            break
+    if not allow_params:
+        if all(param in parameters for param in params):
+            allow_params = True
+    return allow_params
+
+
 class DiffusionModel(SDAPIDiffusionModelMixin):
     def __init__(
         self,
@@ -175,6 +193,18 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
             self._model_path,
             **self._kwargs,
         )
+        if self._kwargs.get("deepcache", True):
+            # NOTE: DeepCache should be loaded first before cpu_offloading
+            try:
+                from DeepCache import DeepCacheSDHelper
+
+                helper = DeepCacheSDHelper(pipe=self._model)
+                helper.set_params(cache_interval=3, cache_branch_id=0)
+                helper.enable()
+            except ImportError:
+                logger.debug("deepcache is not installed")
+                pass
+
         if self._kwargs.get("cpu_offload", False):
             logger.debug("CPU offloading model")
             self._model.enable_model_cpu_offload()
@@ -187,7 +217,7 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
 
     @staticmethod
     def _get_scheduler(model: Any, sampler_name: str):
-        if not sampler_name:
+        if not sampler_name or sampler_name == "default":
             return
 
         assert model is not None
@@ -283,13 +313,14 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
         origin_size = kwargs.pop("origin_size", None)
         seed = kwargs.pop("seed", None)
         if seed is not None:
-            kwargs["generator"] = generator = torch.Generator(device=self._model.device)  # type: ignore
+            kwargs["generator"] = generator = torch.Generator(device=get_available_device())  # type: ignore
             if seed != -1:
                 kwargs["generator"] = generator.manual_seed(seed)
         sampler_name = kwargs.pop("sampler_name", None)
         assert callable(model)
         with self._reset_when_done(model, sampler_name):
             logger.debug("stable diffusion args: %s, model: %s", kwargs, model)
+            self._filter_kwargs(model, kwargs)
             images = model(**kwargs).images
 
         # revert padding if padded
@@ -328,11 +359,17 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
             raise ValueError(f"Unsupported response format: {response_format}")
 
     @classmethod
-    def _filter_kwargs(cls, kwargs: dict):
+    def _filter_kwargs(cls, model, kwargs: dict):
         for arg in ["negative_prompt", "num_inference_steps"]:
             if not kwargs.get(arg):
                 kwargs.pop(arg, None)
 
+        for key in list(kwargs):
+            allow_key = model_accept_param(key, model)
+            if not allow_key:
+                warnings.warn(f"{type(model)} cannot accept `{key}`, will ignore it")
+                kwargs.pop(key)
+
     def text_to_image(
         self,
         prompt: str,
@@ -346,7 +383,6 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
         width, height = map(int, re.split(r"[^\d]+", size))
         generate_kwargs = self._model_spec.default_generate_config.copy()  # type: ignore
         generate_kwargs.update({k: v for k, v in kwargs.items() if v is not None})
-        self._filter_kwargs(generate_kwargs)
         return self._call_model(
             prompt=prompt,
             height=height,
@@ -368,7 +404,6 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
         self,
         image: PIL.Image,
         prompt: Optional[Union[str, List[str]]] = None,
-        negative_prompt: Optional[Union[str, List[str]]] = None,
         n: int = 1,
         size: Optional[str] = None,
         response_format: str = "url",
@@ -404,19 +439,10 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
             kwargs["height"] = height
         else:
             # SD3 image2image cannot accept width and height
-            parameters = inspect.signature(model.__call__).parameters  # type: ignore
-            allow_width_height = False
-            for param in parameters.values():
-                if param.kind == inspect.Parameter.VAR_KEYWORD:
-                    allow_width_height = True
-                    break
-            if "width" in parameters or "height" in parameters:
-                allow_width_height = True
+            allow_width_height = model_accept_param(["width", "height"], model)
             if allow_width_height:
                 kwargs["width"], kwargs["height"] = image.size
 
-        kwargs["negative_prompt"] = negative_prompt
-        self._filter_kwargs(kwargs)
         return self._call_model(
             image=image,
             prompt=prompt,
@@ -431,7 +457,6 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
         image: PIL.Image,
         mask_image: PIL.Image,
         prompt: Optional[Union[str, List[str]]] = None,
-        negative_prompt: Optional[Union[str, List[str]]] = None,
         n: int = 1,
         size: str = "1024*1024",
         response_format: str = "url",
@@ -469,8 +494,6 @@ class DiffusionModel(SDAPIDiffusionModelMixin):
             # calculate actual image size after padding
             width, height = image.size
 
-        kwargs["negative_prompt"] = negative_prompt
-        self._filter_kwargs(kwargs)
         return self._call_model(
             image=image,
             mask_image=mask_image,

xinference/model/llm/__init__.py

@@ -121,7 +121,7 @@ def register_custom_model():
                 with codecs.open(
                     os.path.join(user_defined_llm_dir, f), encoding="utf-8"
                 ) as fd:
-                    user_defined_llm_family = CustomLLMFamilyV1.parse_obj(json.load(fd))
+                    user_defined_llm_family = CustomLLMFamilyV1.parse_raw(fd.read())
                     register_llm(user_defined_llm_family, persist=False)
             except Exception as e:
                 warnings.warn(f"{user_defined_llm_dir}/{f} has error, {e}")