xinference 0.8.5__py3-none-any.whl → 0.9.0__py3-none-any.whl

xinference/device_utils.py

@@ -0,0 +1,100 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import torch
+from typing_extensions import Literal, Union
+
+DeviceType = Literal["cuda", "mps", "xpu", "cpu"]
+
+
+def is_xpu_available() -> bool:
+    return hasattr(torch, "xpu") and torch.xpu.is_available()
+
+
+def get_available_device() -> DeviceType:
+    if torch.cuda.is_available():
+        return "cuda"
+    elif torch.backends.mps.is_available():
+        return "mps"
+    elif is_xpu_available():
+        return "xpu"
+    return "cpu"
+
+
+def is_device_available(device: str) -> bool:
+    if device == "cuda":
+        return torch.cuda.is_available()
+    elif device == "mps":
+        return torch.backends.mps.is_available()
+    elif device == "xpu":
+        return is_xpu_available()
+    elif device == "cpu":
+        return True
+
+    return False
+
+
+def move_model_to_available_device(model):
+    device = get_available_device()
+
+    if device == "cpu":
+        return model
+
+    return model.to(device)
+
+
+def get_device_preferred_dtype(device: str) -> Union[torch.dtype, None]:
+    if device == "cpu":
+        return torch.float32
+    elif device == "cuda" or device == "mps":
+        return torch.float16
+    elif device == "xpu":
+        return torch.bfloat16
+
+    return None
+
+
+def is_hf_accelerate_supported(device: str) -> bool:
+    return device == "cuda" or device == "xpu"
+
+
+def empty_cache():
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+    if torch.backends.mps.is_available():
+        torch.mps.empty_cache()
+    if is_xpu_available():
+        torch.xpu.empty_cache()
+
+
+def gpu_count():
+    if torch.cuda.is_available():
+        cuda_visible_devices_env = os.getenv("CUDA_VISIBLE_DEVICES", None)
+
+        if cuda_visible_devices_env is None:
+            return torch.cuda.device_count()
+
+        cuda_visible_devices = (
+            cuda_visible_devices_env.split(",") if cuda_visible_devices_env else []
+        )
+
+        return min(torch.cuda.device_count(), len(cuda_visible_devices))
+    elif torch.backends.mps.is_available():
+        return 1
+    elif is_xpu_available():
+        return torch.xpu.device_count()
+    else:
+        return 0

xinference/model/audio/whisper.py

@@ -14,6 +14,12 @@
 import logging
 from typing import TYPE_CHECKING, Dict, Optional
 
+from xinference.device_utils import (
+    get_available_device,
+    get_device_preferred_dtype,
+    is_device_available,
+)
+
 if TYPE_CHECKING:
     from .core import AudioModelFamilyV1
 
@@ -37,11 +43,15 @@ class WhisperModel:
         self._kwargs = kwargs
 
     def load(self):
-        import torch
         from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
 
-        device = "cuda:0" if torch.cuda.is_available() else "cpu"
-        torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+        if self._device is None:
+            self._device = get_available_device()
+        else:
+            if not is_device_available(self._device):
+                raise ValueError(f"Device {self._device} is not available!")
+
+        torch_dtype = get_device_preferred_dtype(self._device)
 
         model = AutoModelForSpeechSeq2Seq.from_pretrained(
             self._model_path,
@@ -49,7 +59,7 @@ class WhisperModel:
             low_cpu_mem_usage=True,
             use_safetensors=True,
         )
-        model.to(device)
+        model.to(self._device)
 
         processor = AutoProcessor.from_pretrained(self._model_path)
 
@@ -63,7 +73,7 @@ class WhisperModel:
             batch_size=16,
             return_timestamps=False,
             torch_dtype=torch_dtype,
-            device=device,
+            device=self._device,
         )
 
     def _call_model(
@@ -99,9 +109,11 @@ class WhisperModel:
             )
         return self._call_model(
             audio=audio,
-            generate_kwargs={"language": language, "task": "transcribe"}
-            if language is not None
-            else {"task": "transcribe"},
+            generate_kwargs=(
+                {"language": language, "task": "transcribe"}
+                if language is not None
+                else {"task": "transcribe"}
+            ),
             response_format=response_format,
         )
 

xinference/model/image/core.py

@@ -56,6 +56,10 @@ class ImageModelDescription(ModelDescription):
         self._model_spec = model_spec
 
     def to_dict(self):
+        if self._model_spec.controlnet is not None:
+            controlnet = [cn.dict() for cn in self._model_spec.controlnet]
+        else:
+            controlnet = self._model_spec.controlnet
         return {
             "model_type": "image",
             "address": self.address,
@@ -63,7 +67,7 @@ class ImageModelDescription(ModelDescription):
             "model_name": self._model_spec.model_name,
             "model_family": self._model_spec.model_family,
             "model_revision": self._model_spec.model_revision,
-            "controlnet": self._model_spec.controlnet,
+            "controlnet": controlnet,
         }
 
     def to_version_info(self):

xinference/model/image/stable_diffusion/core.py

@@ -24,6 +24,7 @@ from io import BytesIO
 from typing import List, Optional, Union
 
 from ....constants import XINFERENCE_IMAGE_DIR
+from ....device_utils import move_model_to_available_device
 from ....types import Image, ImageList
 
 logger = logging.getLogger(__name__)
@@ -40,7 +41,7 @@ class DiffusionModel:
         self._kwargs = kwargs
 
     def load(self):
-        import torch
+        # import torch
         from diffusers import AutoPipelineForText2Image
 
         controlnet = self._kwargs.get("controlnet")
@@ -57,10 +58,7 @@ class DiffusionModel:
             # torch_dtype=torch.float16,
             # use_safetensors=True,
         )
-        if torch.cuda.is_available():
-            self._model = self._model.to("cuda")
-        elif torch.backends.mps.is_available():
-            self._model = self._model.to("mps")
+        self._model = move_model_to_available_device(self._model)
         # Recommended if your computer has < 64 GB of RAM
         self._model.enable_attention_slicing()
 

xinference/model/llm/llm_family.json

@@ -1514,10 +1514,33 @@
         ],
         "model_id": "Qwen/Qwen1.5-72B-Chat-AWQ"
       },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
+        ],
+        "model_id": "Qwen/Qwen1.5-0.5B-Chat-GGUF",
+        "model_file_name_template": "qwen1_5-0_5b-chat-{quantization}.gguf"
+      },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_8",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-1.8B-Chat-GGUF",
@@ -1527,6 +1550,13 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 4,
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-4B-Chat-GGUF",
@@ -1536,7 +1566,14 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 7,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-7B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-7b-chat-{quantization}.gguf"
@@ -1545,7 +1582,14 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 14,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "Qwen/Qwen1.5-14B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-14b-chat-{quantization}.gguf"
@@ -1554,7 +1598,8 @@
         "model_format": "ggufv2",
         "model_size_in_billions": 72,
         "quantizations": [
-          "q2_k"
+          "q2_k",
+          "q3_k_m"
         ],
         "model_id": "Qwen/Qwen1.5-72B-Chat-GGUF",
         "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf"
@@ -3708,5 +3753,50 @@
         "<|im_sep|>"
       ]
     }
+  },
+  {
+    "version": 1,
+    "context_length": 8192,
+    "model_name": "gemma-it",
+    "model_lang": [
+      "en"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "Gemma is a family of lightweight, state-of-the-art open models from Google, built from the same research and technology used to create the Gemini models.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 2,
+        "quantizations": [
+          "none",
+          "4-bit",
+          "8-bit"
+        ],
+        "model_id": "google/gemma-2b-it"
+      },
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none",
+          "4-bit",
+          "8-bit"
+        ],
+        "model_id": "google/gemma-7b-it"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "gemma",
+      "roles": [
+        "user",
+        "model"
+      ],
+      "stop": [
+        "<end_of_turn>",
+        "<start_of_turn>"
+      ]
+    }
   }
 ]

xinference/model/llm/llm_family_modelscope.json

@@ -29,7 +29,7 @@
           "Q4_K_M"
         ],
         "model_id": "Xorbits/Llama-2-13b-Chat-GGUF",
-        "model_file_name_template": "llama-2-7b-chat.{quantization}.gguf",
+        "model_file_name_template": "llama-2-13b-chat.{quantization}.gguf",
         "model_hub": "modelscope",
         "model_revision": "v0.0.1"
       },
@@ -1821,61 +1821,97 @@
         "model_format": "ggufv2",
         "model_size_in_billions": "0_5",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-0.5B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-0.5b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-0_5b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_8",
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-1.8B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-1_8b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-1_8b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 4,
         "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
           "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-4B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-4b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-4b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 7,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-7B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-7b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-7b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 14,
         "quantizations": [
-          "q5_k_m"
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0"
         ],
         "model_id": "qwen/Qwen1.5-14B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-14b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-14b-chat-{quantization}.gguf"
       },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": 72,
         "quantizations": [
-          "q2_k"
+          "q2_k",
+          "q3_k_m"
         ],
         "model_id": "qwen/Qwen1.5-72B-Chat-GGUF",
         "model_hub": "modelscope",
-        "model_file_name_template": "qwen1.5-72b-chat-{quantization}.gguf"
+        "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf"
       }
     ],
     "prompt_style": {

xinference/model/llm/pytorch/compression.py

@@ -25,6 +25,8 @@ from torch.nn import functional as F
 from tqdm import tqdm
 from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
 
+from ....device_utils import empty_cache
+
 
 @dataclasses.dataclass
 class CompressionConfig:
@@ -153,7 +155,7 @@ def load_compress_model(
             tmp_state_dict[name] = None
             tensor = None
             gc.collect()
-            torch.cuda.empty_cache()
+            empty_cache()
 
     for name in model.state_dict():
         if name not in linear_weights:

xinference/model/llm/pytorch/core.py

@@ -12,10 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import logging
 import os
 from typing import Iterable, Iterator, List, Optional, Union
 
+from ....device_utils import (
+    get_device_preferred_dtype,
+    gpu_count,
+    is_hf_accelerate_supported,
+)
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -115,23 +121,18 @@ class PytorchModel(LLM):
             )
         from .compression import load_compress_model
 
-        cuda_visible_devices_env = os.getenv("CUDA_VISIBLE_DEVICES", None)
-        cuda_visible_devices = (
-            cuda_visible_devices_env.split(",") if cuda_visible_devices_env else []
-        )
-
         quantization = self.quantization
-        num_gpus = len(cuda_visible_devices) if cuda_visible_devices_env != "-1" else 0
+        num_gpus = gpu_count()
         device = self._pytorch_model_config.get("device", "auto")
         self._pytorch_model_config["device"] = select_device(device)
         self._device = self._pytorch_model_config["device"]
 
-        if self._device == "cpu":
-            kwargs = {"torch_dtype": torch.float32}
-        elif self._device == "cuda":
-            kwargs = {"torch_dtype": torch.float16}
-        elif self._device == "mps":
-            kwargs = {"torch_dtype": torch.float16}
+        kwargs = {}
+
+        dtype = get_device_preferred_dtype(self._device)
+
+        if dtype is not None:
+            kwargs["torch_dtype"] = dtype
         else:
             raise ValueError(f"Device {self._device} is not supported in temporary")
 
@@ -142,9 +143,25 @@ class PytorchModel(LLM):
             "trust_remote_code"
         )
         model_format = self.model_spec.model_format
+
+        is_device_map_auto = False
+
+        # This is required for Intel GPU to actually work with accelerate device_map until
+        # https://github.com/intel/intel-extension-for-pytorch/issues/522
+        # is resolved
+        max_memory_env = os.getenv("ACCELERATE_MAX_MEMORY", None)
+
+        if max_memory_env is not None:
+            max_memory_raw = json.loads(max_memory_env)
+            max_memory = {
+                int(k) if k.isdigit() else k: max_memory_raw[k] for k in max_memory_raw
+            }
+            kwargs["max_memory"] = max_memory
+
         if quantization != "none" and model_format == "pytorch":
             if self._device == "cuda" and self._is_linux():
                 kwargs["device_map"] = "auto"
+                is_device_map_auto = True
                 if quantization == "4-bit":
                     kwargs["load_in_4bit"] = True
                     kwargs["bnb_4bit_compute_dtype"] = torch.float16
@@ -178,11 +195,13 @@ class PytorchModel(LLM):
                     logger.debug(f"Model Memory: {self._model.get_memory_footprint()}")
                     return
 
-        if num_gpus > 0 and self._device == "cuda":
+        if num_gpus > 0 and is_hf_accelerate_supported(self._device):
             kwargs.update({"device_map": "auto"})
+            is_device_map_auto = True
+
         self._model, self._tokenizer = self._load_model(**kwargs)
 
-        if self._device == "mps":
+        if not is_device_map_auto:
             self._model.to(self._device)
         logger.debug(f"Model Memory: {self._model.get_memory_footprint()}")
 

xinference/model/llm/pytorch/qwen_vl.py

@@ -95,9 +95,11 @@ class QwenVLChatModel(PytorchChatModel):
         if not isinstance(content, str):
             # TODO(codingl2k1): Optimize _ensure_url
             content = [
-                {"image": _ensure_url(c["image_url"]["url"]), "type": "image"}
-                if c.get("type") == "image_url"
-                else c
+                (
+                    {"image": _ensure_url(c["image_url"]["url"]), "type": "image"}
+                    if c.get("type") == "image_url"
+                    else c
+                )
                 for c in content
             ]
             content = sorted(content, key=operator.itemgetter("type"))

xinference/model/llm/pytorch/spec_decoding_utils.py

@@ -17,6 +17,8 @@ import time
 import uuid
 from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
 
+from ....device_utils import empty_cache
+
 try:
     import torch
     from torch.nn import functional as F
@@ -526,4 +528,4 @@ def speculative_generate_stream(
     del kv_cache
     del draft_kv_cache
     gc.collect()
-    torch.cuda.empty_cache()
+    empty_cache()