xinference 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

xinference/model/image/model_spec_modelscope.json

@@ -4,21 +4,31 @@
     "model_family": "stable_diffusion",
     "model_hub": "modelscope",
     "model_id": "AI-ModelScope/stable-diffusion-3-medium-diffusers",
-    "model_revision": "master"
+    "model_revision": "master",
+    "abilities": [
+      "text2iamge",
+      "image2image"
+    ]
   },
   {
     "model_name": "sd-turbo",
     "model_family": "stable_diffusion",
     "model_hub": "modelscope",
     "model_id": "AI-ModelScope/sd-turbo",
-    "model_revision": "master"
+    "model_revision": "master",
+    "abilities": [
+      "text2iamge"
+    ]
   },
   {
     "model_name": "sdxl-turbo",
     "model_family": "stable_diffusion",
     "model_hub": "modelscope",
     "model_id": "AI-ModelScope/sdxl-turbo",
-    "model_revision": "master"
+    "model_revision": "master",
+    "abilities": [
+      "text2iamge"
+    ]
   },
   {
     "model_name": "stable-diffusion-v1.5",
@@ -26,6 +36,10 @@
     "model_hub": "modelscope",
     "model_id": "AI-ModelScope/stable-diffusion-v1-5",
     "model_revision": "master",
+    "abilities": [
+      "text2iamge",
+      "image2image"
+    ],
     "controlnet": [
       {
         "model_name":"canny",
@@ -77,6 +91,10 @@
     "model_hub": "modelscope",
     "model_id": "AI-ModelScope/stable-diffusion-xl-base-1.0",
     "model_revision": "master",
+    "abilities": [
+      "text2iamge",
+      "image2image"
+    ],
     "controlnet": [
       {
         "model_name":"canny",

xinference/model/image/stable_diffusion/core.py

@@ -35,22 +35,23 @@ class DiffusionModel:
     def __init__(
         self,
         model_uid: str,
-        model_path: str,
+        model_path: Optional[str] = None,
         device: Optional[str] = None,
         lora_model: Optional[List[LoRA]] = None,
         lora_load_kwargs: Optional[Dict] = None,
         lora_fuse_kwargs: Optional[Dict] = None,
-        ability: Optional[str] = None,
+        abilities: Optional[List[str]] = None,
         **kwargs,
     ):
         self._model_uid = model_uid
         self._model_path = model_path
         self._device = device
         self._model = None
+        self._i2i_model = None  # image to image model
         self._lora_model = lora_model
         self._lora_load_kwargs = lora_load_kwargs or {}
         self._lora_fuse_kwargs = lora_fuse_kwargs or {}
-        self._ability = ability
+        self._abilities = abilities
         self._kwargs = kwargs
 
     def _apply_lora(self):
@@ -69,12 +70,12 @@ class DiffusionModel:
     def load(self):
         import torch
 
-        if self._ability in [None, "text2image", "image2image"]:
+        if "text2image" in self._abilities or "image2image" in self._abilities:
             from diffusers import AutoPipelineForText2Image as AutoPipelineModel
-        elif self._ability == "inpainting":
+        elif "inpainting" in self._abilities:
             from diffusers import AutoPipelineForInpainting as AutoPipelineModel
         else:
-            raise ValueError(f"Unknown ability: {self._ability}")
+            raise ValueError(f"Unknown ability: {self._abilities}")
 
         controlnet = self._kwargs.get("controlnet")
         if controlnet is not None:
@@ -106,28 +107,17 @@ class DiffusionModel:
 
     def _call_model(
         self,
-        height: int,
-        width: int,
-        num_images_per_prompt: int,
         response_format: str,
+        model=None,
         **kwargs,
     ):
         logger.debug(
             "stable diffusion args: %s",
-            dict(
-                kwargs,
-                height=height,
-                width=width,
-                num_images_per_prompt=num_images_per_prompt,
-            ),
+            kwargs,
         )
-        assert callable(self._model)
-        images = self._model(
-            height=height,
-            width=width,
-            num_images_per_prompt=num_images_per_prompt,
-            **kwargs,
-        ).images
+        model = model if model is not None else self._model
+        assert callable(model)
+        images = model(**kwargs).images
         if response_format == "url":
             os.makedirs(XINFERENCE_IMAGE_DIR, exist_ok=True)
             image_list = []
@@ -145,7 +135,7 @@ class DiffusionModel:
                 return base64.b64encode(buffered.getvalue()).decode()
 
             with ThreadPoolExecutor() as executor:
-                results = list(map(partial(executor.submit, _gen_base64_image), images))
+                results = list(map(partial(executor.submit, _gen_base64_image), images))  # type: ignore
                 image_list = [Image(url=None, b64_json=s.result()) for s in results]
             return ImageList(created=int(time.time()), data=image_list)
         else:
@@ -177,19 +167,32 @@ class DiffusionModel:
         prompt: Optional[Union[str, List[str]]] = None,
         negative_prompt: Optional[Union[str, List[str]]] = None,
         n: int = 1,
-        size: str = "1024*1024",
+        size: Optional[str] = None,
         response_format: str = "url",
         **kwargs,
     ):
-        width, height = map(int, re.split(r"[^\d]+", size))
+        if "controlnet" in self._kwargs:
+            model = self._model
+        else:
+            if self._i2i_model is not None:
+                model = self._i2i_model
+            else:
+                from diffusers import AutoPipelineForImage2Image
+
+                self._i2i_model = model = AutoPipelineForImage2Image.from_pipe(
+                    self._model
+                )
+        if size:
+            width, height = map(int, re.split(r"[^\d]+", size))
+            kwargs["width"] = width
+            kwargs["height"] = height
         return self._call_model(
             image=image,
             prompt=prompt,
             negative_prompt=negative_prompt,
-            height=height,
-            width=width,
             num_images_per_prompt=n,
             response_format=response_format,
+            model=model,
             **kwargs,
         )
 

xinference/model/llm/core.py

@@ -194,6 +194,7 @@ def create_llm_model_instance(
     quantization: Optional[str] = None,
     peft_model_config: Optional[PeftModelConfig] = None,
     download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
+    model_path: Optional[str] = None,
     **kwargs,
 ) -> Tuple[LLM, LLMDescription]:
     from .llm_family import cache, check_engine_by_spec_parameters, match_llm
@@ -221,7 +222,8 @@ def create_llm_model_instance(
     )
     logger.debug(f"Launching {model_uid} with {llm_cls.__name__}")
 
-    save_path = cache(llm_family, llm_spec, quantization)
+    if not model_path:
+        model_path = cache(llm_family, llm_spec, quantization)
 
     peft_model = peft_model_config.peft_model if peft_model_config else None
     if peft_model is not None:
@@ -231,7 +233,7 @@ def create_llm_model_instance(
                 llm_family,
                 llm_spec,
                 quantization,
-                save_path,
+                model_path,
                 kwargs,
                 peft_model,
             )
@@ -241,11 +243,11 @@ def create_llm_model_instance(
                 f"Load this without lora."
             )
             model = llm_cls(
-                model_uid, llm_family, llm_spec, quantization, save_path, kwargs
+                model_uid, llm_family, llm_spec, quantization, model_path, kwargs
             )
     else:
         model = llm_cls(
-            model_uid, llm_family, llm_spec, quantization, save_path, kwargs
+            model_uid, llm_family, llm_spec, quantization, model_path, kwargs
         )
     return model, LLMDescription(
         subpool_addr, devices, llm_family, llm_spec, quantization

xinference/model/llm/ggml/llamacpp.py

@@ -155,11 +155,13 @@ class LlamaCppModel(LLM):
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
 
         # handle legacy cache.
-        model_path = os.path.join(
-            self.model_path,
-            self.model_spec.model_file_name_template.format(
-                quantization=self.quantization
-            ),
+        model_path = os.path.realpath(
+            os.path.join(
+                self.model_path,
+                self.model_spec.model_file_name_template.format(
+                    quantization=self.quantization
+                ),
+            )
         )
         legacy_model_file_path = os.path.join(self.model_path, "model.bin")
         if os.path.exists(legacy_model_file_path):

xinference/model/llm/llm_family.py

@@ -699,12 +699,12 @@ def _generate_model_file_names(
 def _merge_cached_files(
     cache_dir: str, input_file_names: List[str], output_file_name: str
 ):
-    with open(os.path.join(cache_dir, output_file_name), "wb") as output_file:
-        for file_name in input_file_names:
-            logger.info(f"Merging file {file_name} into {output_file_name} ...")
-
-            with open(os.path.join(cache_dir, file_name), "rb") as input_file:
-                shutil.copyfileobj(input_file, output_file)
+    # now llama.cpp can find the gguf parts automatically
+    # we only need to provide the first part
+    # thus we create the symlink to the first part
+    symlink_local_file(
+        os.path.join(cache_dir, input_file_names[0]), cache_dir, output_file_name
+    )
 
     logger.info(f"Merge complete.")
 

xinference/model/llm/mlx/core.py

@@ -101,6 +101,7 @@ class MLXModel(LLM):
 
     def _load_model(self, **kwargs):
         try:
+            import mlx.core as mx
             from mlx_lm import load
         except ImportError:
             error_message = "Failed to import module 'mlx_lm'"
@@ -122,6 +123,11 @@ class MLXModel(LLM):
             self._model_config,
         )
 
+        cache_limit_gb = kwargs.get("cache_limit_gb", None)
+        if cache_limit_gb:
+            logger.debug(f"Setting cache limit to {cache_limit_gb} GB")
+            mx.metal.set_cache_limit(cache_limit_gb * 1024 * 1024 * 1024)
+
         return load(
             self.model_path,
             tokenizer_config=tokenizer_config,
@@ -134,6 +140,7 @@ class MLXModel(LLM):
             "revision", self.model_spec.model_revision
         )
         kwargs["trust_remote_code"] = self._model_config.get("trust_remote_code")
+        kwargs["cache_limit_gb"] = self._model_config.pop("cache_limit_gb", None)
 
         self._model, self._tokenizer = self._load_model(**kwargs)
 

xinference/model/llm/pytorch/chatglm.py

@@ -430,7 +430,10 @@ class ChatglmPytorchChatModel(PytorchChatModel):
         outputs = self._model.generate(**kwargs)
         outputs = outputs[:, kwargs["input_ids"].shape[1] :]
         response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        return self._process_response(response, history, tools, end=True)
+        if tools:
+            return self._process_response(response, history, tools, end=True)
+        else:
+            return self._process_response(response, history, tools)
 
     def chat(
         self,

xinference/model/llm/pytorch/deepseek_vl.py

@@ -52,7 +52,8 @@ class DeepSeekVLChatModel(PytorchChatModel):
     def match(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if "deepseek" in model_family.model_name:
+        llm_family = model_family.model_family or model_family.model_name
+        if "deepseek-vl" in llm_family:
             return True
         return False
 

xinference/model/llm/pytorch/falcon.py

@@ -71,7 +71,8 @@ class FalconPytorchModel(PytorchModel):
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False
-        if "falcon" not in llm_family.model_name:
+        model_family = llm_family.model_family or llm_family.model_name
+        if "falcon" not in model_family:
             return False
         if "generate" not in llm_family.model_ability:
             return False

xinference/model/llm/pytorch/llama_2.py

@@ -55,7 +55,8 @@ class LlamaPytorchModel(PytorchModel):
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False
-        if "llama-2" not in llm_family.model_name:
+        model_family = llm_family.model_family or llm_family.model_name
+        if "llama-2" not in model_family:
             return False
         if "generate" not in llm_family.model_ability:
             return False
@@ -99,7 +100,8 @@ class LlamaPytorchChatModel(PytorchChatModel):
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False
-        if "llama-2" not in llm_family.model_name:
+        model_family = llm_family.model_family or llm_family.model_name
+        if "llama-2" not in model_family:
             return False
         if "chat" not in llm_family.model_ability:
             return False

xinference/model/llm/pytorch/omnilmm.py

@@ -44,7 +44,8 @@ class OmniLMMModel(PytorchChatModel):
     def match(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if "OmniLMM" in model_family.model_name:
+        llm_family = model_family.model_family or model_family.model_name
+        if "OmniLMM" in llm_family:
             return True
         return False
 

xinference/model/llm/pytorch/qwen_vl.py

@@ -52,7 +52,8 @@ class QwenVLChatModel(PytorchChatModel):
     def match(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if "qwen" in model_family.model_name and "vision" in model_family.model_ability:
+        llm_family = model_family.model_family or model_family.model_name
+        if "qwen" in llm_family and "vision" in model_family.model_ability:
             return True
         return False
 

xinference/model/llm/pytorch/vicuna.py

@@ -61,7 +61,8 @@ class VicunaPytorchChatModel(PytorchChatModel):
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False
-        if "vicuna" not in llm_family.model_name:
+        model_family = llm_family.model_family or llm_family.model_name
+        if "vicuna" not in model_family:
             return False
         if "chat" not in llm_family.model_ability:
             return False

xinference/model/llm/pytorch/yi_vl.py

@@ -51,7 +51,8 @@ class YiVLChatModel(PytorchChatModel):
     def match(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if "yi" in model_family.model_name:
+        llm_family = model_family.model_family or model_family.model_name
+        if "yi-vl" in llm_family:
             return True
         return False
 

xinference/model/llm/sglang/core.py

@@ -17,7 +17,6 @@ import time
 import uuid
 from typing import AsyncGenerator, Dict, List, Optional, TypedDict, Union
 
-from ....constants import XINFERENCE_ENABLE_SGLANG
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -63,15 +62,26 @@ try:
 except ImportError:
     SGLANG_INSTALLED = False
 
-SGLANG_SUPPORTED_MODELS = ["llama-2", "mistral-v0.1", "mixtral-v0.1"]
+SGLANG_SUPPORTED_MODELS = [
+    "llama-2",
+    "llama-3",
+    "llama-3.1",
+    "mistral-v0.1",
+    "mixtral-v0.1",
+]
 SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
+    "llama-3-instruct",
+    "llama-3.1-instruct",
     "qwen-chat",
     "qwen1.5-chat",
+    "qwen2-instruct",
+    "qwen2-moe-instruct",
     "mistral-instruct-v0.1",
     "mistral-instruct-v0.2",
     "mixtral-instruct-v0.1",
     "gemma-it",
+    "gemma-2-it",
 ]
 
 
@@ -168,8 +178,6 @@ class SGLANGModel(LLM):
     def match(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if not XINFERENCE_ENABLE_SGLANG:
-            return False
         if not cls._has_cuda_device():
             return False
         if not cls._is_linux():
@@ -332,8 +340,6 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
     def match(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if not XINFERENCE_ENABLE_SGLANG:
-            return False
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False
         if llm_spec.model_format == "pytorch":

xinference/model/llm/vllm/core.py

@@ -28,7 +28,6 @@ from typing import (
     Union,
 )
 
-from ....constants import XINFERENCE_DISABLE_VLLM
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -152,6 +151,7 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.4.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("c4ai-command-r-v01")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.5.3":
+    VLLM_SUPPORTED_CHAT_MODELS.append("gemma-2-it")
     VLLM_SUPPORTED_CHAT_MODELS.append("mistral-nemo-instruct")
     VLLM_SUPPORTED_CHAT_MODELS.append("mistral-large-instruct")
 
@@ -296,8 +296,6 @@ class VLLMModel(LLM):
     def match(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if XINFERENCE_DISABLE_VLLM:
-            return False
         if not cls._has_cuda_device():
             return False
         if not cls._is_linux():
@@ -522,8 +520,6 @@ class VLLMChatModel(VLLMModel, ChatModelMixin):
     def match(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
-        if XINFERENCE_DISABLE_VLLM:
-            return False
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False
         if llm_spec.model_format == "pytorch":

xinference/model/rerank/core.py

@@ -107,7 +107,7 @@ class RerankModel:
         self,
         model_spec: RerankModelSpec,
         model_uid: str,
-        model_path: str,
+        model_path: Optional[str] = None,
         device: Optional[str] = None,
         use_fp16: bool = False,
         model_config: Optional[Dict] = None,
@@ -290,6 +290,7 @@ def create_rerank_model_instance(
     model_uid: str,
     model_name: str,
     download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
+    model_path: Optional[str] = None,
     **kwargs,
 ) -> Tuple[RerankModel, RerankModelDescription]:
     from ..utils import download_from_modelscope
@@ -321,8 +322,8 @@ def create_rerank_model_instance(
                 f"Huggingface: {BUILTIN_RERANK_MODELS.keys()}"
                 f"ModelScope: {MODELSCOPE_RERANK_MODELS.keys()}"
             )
-
-    model_path = cache(model_spec)
+    if not model_path:
+        model_path = cache(model_spec)
     use_fp16 = kwargs.pop("use_fp16", False)
     model = RerankModel(
         model_spec, model_uid, model_path, use_fp16=use_fp16, model_config=kwargs

xinference/web/ui/build/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.4bafd904.css",
-    "main.js": "./static/js/main.2ef0cfaf.js",
+    "main.js": "./static/js/main.af906659.js",
     "static/media/icon.webp": "./static/media/icon.4603d52c63041e5dfbfd.webp",
     "index.html": "./index.html",
     "main.4bafd904.css.map": "./static/css/main.4bafd904.css.map",
-    "main.2ef0cfaf.js.map": "./static/js/main.2ef0cfaf.js.map"
+    "main.af906659.js.map": "./static/js/main.af906659.js.map"
   },
   "entrypoints": [
     "static/css/main.4bafd904.css",
-    "static/js/main.2ef0cfaf.js"
+    "static/js/main.af906659.js"
   ]
 }

xinference/web/ui/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.2ef0cfaf.js"></script><link href="./static/css/main.4bafd904.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.af906659.js"></script><link href="./static/css/main.4bafd904.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>