xinference 0.8.5__py3-none-any.whl → 0.9.0__py3-none-any.whl

xinference/__init__.py

@@ -18,6 +18,12 @@ from . import _version
 __version__ = _version.get_versions()["version"]
 
 
+try:
+    import intel_extension_for_pytorch  # noqa: F401
+except:
+    pass
+
+
 def _install():
     from xoscar.backends.router import Router
 

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-02-06T13:35:33+0800",
+ "date": "2024-02-22T15:40:53+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "e903e05145efa8ad05d61e7e7cac55627f5ace51",
- "version": "0.8.5"
+ "full-revisionid": "c653c975847f9f6a81382033a9c8f5bd81bf70f2",
+ "version": "0.9.0"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -51,6 +51,7 @@ from uvicorn import Config, Server
 from xoscar.utils import get_next_port
 
 from .._compat import BaseModel, Field
+from .._version import get_versions
 from ..constants import XINFERENCE_DEFAULT_ENDPOINT_PORT
 from ..core.event import Event, EventCollectorActor, EventType
 from ..core.supervisor import SupervisorActor
@@ -221,6 +222,9 @@ class RESTfulAPI:
         self._router.add_api_route(
             "/v1/cluster/info", self.get_cluster_device_info, methods=["GET"]
         )
+        self._router.add_api_route(
+            "/v1/cluster/version", self.get_cluster_version, methods=["GET"]
+        )
         self._router.add_api_route(
             "/v1/cluster/devices", self._get_devices_count, methods=["GET"]
         )
@@ -231,9 +235,11 @@ class RESTfulAPI:
             "/v1/ui/{model_uid}",
             self.build_gradio_interface,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/token", self.login_for_access_token, methods=["POST"]
@@ -246,142 +252,176 @@ class RESTfulAPI:
             "/v1/models/instances",
             self.get_instance_info,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_type}/{model_name}/versions",
             self.get_model_versions,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models",
             self.list_models,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         self._router.add_api_route(
             "/v1/models/{model_uid}",
             self.describe_model,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_uid}/events",
             self.get_model_events,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/instance",
             self.launch_model_by_version,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models",
             self.launch_model,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/experimental/speculative_llms",
             self.launch_speculative_llm,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_uid}",
             self.terminate_model,
             methods=["DELETE"],
-            dependencies=[Security(self._auth_service, scopes=["models:stop"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:stop"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/completions",
             self.create_completion,
             methods=["POST"],
             response_model=Completion,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/embeddings",
             self.create_embedding,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/rerank",
             self.rerank,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/audio/transcriptions",
             self.create_transcriptions,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/audio/translations",
             self.create_translations,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/images/generations",
             self.create_images,
             methods=["POST"],
             response_model=ImageList,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/images/variations",
             self.create_variations,
             methods=["POST"],
             response_model=ImageList,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/chat/completions",
             self.create_chat_completion,
             methods=["POST"],
             response_model=ChatCompletion,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         # for custom models
@@ -389,33 +429,41 @@ class RESTfulAPI:
             "/v1/model_registrations/{model_type}",
             self.register_model,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:register"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:register"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}/{model_name}",
             self.unregister_model,
             methods=["DELETE"],
-            dependencies=[Security(self._auth_service, scopes=["models:unregister"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:unregister"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}",
             self.list_model_registrations,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}/{model_name}",
             self.get_model_registrations,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         # Clear the global Registry for the MetricsMiddleware, or
@@ -1094,10 +1142,12 @@ class RESTfulAPI:
         if body.logit_bias is not None:
             raise HTTPException(status_code=501, detail="Not implemented")
 
+        messages = body.messages and list(body.messages) or None
+
         if (
-            not body.messages
-            or body.messages[-1].get("role") not in ["user", "system", "tool"]
-            or not body.messages[-1].get("content")
+            not messages
+            or messages[-1].get("role") not in ["user", "system", "tool"]
+            or not messages[-1].get("content")
         ):
             raise HTTPException(
                 status_code=400, detail="Invalid input. Please specify the prompt."
@@ -1105,7 +1155,7 @@ class RESTfulAPI:
 
         system_messages = []
         non_system_messages = []
-        for msg in body.messages:
+        for msg in messages:
             assert (
                 msg.get("content") != SPECIAL_TOOL_PROMPT
             ), f"Invalid message content {SPECIAL_TOOL_PROMPT}"
@@ -1118,13 +1168,13 @@ class RESTfulAPI:
             raise HTTPException(
                 status_code=400, detail="Multiple system messages are not supported."
             )
-        if len(system_messages) == 1 and body.messages[0]["role"] != "system":
+        if len(system_messages) == 1 and messages[0]["role"] != "system":
             raise HTTPException(
                 status_code=400, detail="System message should be the first one."
             )
         assert non_system_messages
 
-        has_tool_message = body.messages[-1].get("role") == "tool"
+        has_tool_message = messages[-1].get("role") == "tool"
         if has_tool_message:
             prompt = SPECIAL_TOOL_PROMPT
             system_prompt = system_messages[0]["content"] if system_messages else None
@@ -1298,9 +1348,21 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def get_cluster_device_info(self) -> JSONResponse:
+    async def get_cluster_device_info(
+        self, detailed: bool = Query(False)
+    ) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).get_cluster_device_info(
+                detailed=detailed
+            )
+            return JSONResponse(content=data)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def get_cluster_version(self) -> JSONResponse:
         try:
-            data = await (await self._get_supervisor_ref()).get_cluster_device_info()
+            data = get_versions()
             return JSONResponse(content=data)
         except Exception as e:
             logger.error(e, exc_info=True)

xinference/core/model.py

@@ -44,6 +44,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+from ..device_utils import empty_cache
 from .utils import json_dumps, log_async
 
 try:
@@ -130,7 +131,7 @@ class ModelActor(xo.StatelessActor):
             try:
                 import gc
 
-                import torch
+                import torch  # noqa: F401
             except ImportError:
                 error_message = "Failed to import module 'torch'"
                 installation_guide = [
@@ -141,7 +142,7 @@ class ModelActor(xo.StatelessActor):
 
             del self._model
             gc.collect()
-            torch.cuda.empty_cache()
+            empty_cache()
 
     def __init__(
         self,

xinference/core/resource.py

@@ -22,8 +22,9 @@ from .utils import get_nvidia_gpu_info
 
 @dataclass
 class ResourceStatus:
-    available: float
+    usage: float
     total: float
+    memory_used: float
     memory_available: float
     memory_total: float
 
@@ -39,8 +40,9 @@ def gather_node_info() -> Dict[str, Union[ResourceStatus, GPUStatus]]:
     node_resource = dict()
     mem_info = psutil.virtual_memory()
     node_resource["cpu"] = ResourceStatus(
-        available=psutil.cpu_percent() / 100.0,
+        usage=psutil.cpu_percent() / 100.0,
         total=psutil.cpu_count(),
+        memory_used=mem_info.used,
         memory_available=mem_info.available,
         memory_total=mem_info.total,
     )

xinference/core/supervisor.py

@@ -15,6 +15,7 @@
 import asyncio
 import itertools
 import time
+import typing
 from dataclasses import dataclass
 from logging import getLogger
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
@@ -179,12 +180,26 @@ class SupervisorActor(xo.StatelessActor):
             model_version_infos, self.address
         )
 
-    async def get_cluster_device_info(self) -> List:
+    @typing.no_type_check
+    async def get_cluster_device_info(self, detailed: bool = False) -> List:
+        import psutil
+
         supervisor_device_info = {
             "ip_address": self.address.split(":")[0],
             "gpu_count": 0,
             "gpu_vram_total": 0,
         }
+        if detailed:
+            supervisor_device_info["gpu_vram_total"] = 0
+            supervisor_device_info["gpu_vram_available"] = 0
+            supervisor_device_info["cpu_available"] = psutil.cpu_count() * (
+                1 - psutil.cpu_percent() / 100.0
+            )
+            supervisor_device_info["cpu_count"] = psutil.cpu_count()
+            mem_info = psutil.virtual_memory()
+            supervisor_device_info["mem_used"] = mem_info.used
+            supervisor_device_info["mem_available"] = mem_info.available
+            supervisor_device_info["mem_total"] = mem_info.total
         res = [{"node_type": "Supervisor", **supervisor_device_info}]
         for worker_addr, worker_status in self._worker_status.items():
             vram_total: float = sum(
@@ -193,14 +208,24 @@ class SupervisorActor(xo.StatelessActor):
             total = (
                 vram_total if vram_total == 0 else f"{int(vram_total / 1024 / 1024)}MiB"
             )
-            res.append(
-                {
-                    "node_type": "Worker",
-                    "ip_address": worker_addr.split(":")[0],
-                    "gpu_count": len(worker_status.status) - 1,
-                    "gpu_vram_total": total,
-                }
-            )
+            info = {
+                "node_type": "Worker",
+                "ip_address": worker_addr.split(":")[0],
+                "gpu_count": len(worker_status.status) - 1,
+                "gpu_vram_total": total,
+            }
+            if detailed:
+                cpu_info = worker_status.status["cpu"]
+                info["cpu_available"] = cpu_info.total * (1 - cpu_info.usage)
+                info["cpu_count"] = cpu_info.total
+                info["mem_used"] = cpu_info.memory_used
+                info["mem_available"] = cpu_info.memory_available
+                info["mem_total"] = cpu_info.memory_total
+                info["gpu_vram_total"] = vram_total
+                info["gpu_vram_available"] = sum(
+                    [v.mem_free for k, v in worker_status.status.items() if k != "cpu"]
+                )
+            res.append(info)
         return res
 
     @staticmethod
@@ -227,11 +252,11 @@ class SupervisorActor(xo.StatelessActor):
         }
 
     async def get_devices_count(self) -> int:
-        from ..utils import cuda_count
+        from ..device_utils import gpu_count
 
         if self.is_local_deployment():
-            return cuda_count()
-        # distributed deployment, choose a worker and return its cuda_count.
+            return gpu_count()
+        # distributed deployment, choose a worker and return its device_count.
         # Assume that each worker has the same count of cards.
         worker_ref = await self._choose_worker()
         return await worker_ref.get_devices_count()

xinference/core/worker.py

@@ -30,8 +30,8 @@ from xoscar import MainActorPoolType
 from ..constants import XINFERENCE_CACHE_DIR
 from ..core import ModelActor
 from ..core.status_guard import LaunchStatus
+from ..device_utils import gpu_count
 from ..model.core import ModelDescription, create_model_instance
-from ..utils import cuda_count
 from .event import Event, EventCollectorActor, EventType
 from .metrics import launch_metrics_export_server, record_metrics
 from .resource import gather_node_info
@@ -54,13 +54,13 @@ class WorkerActor(xo.StatelessActor):
         self,
         supervisor_address: str,
         main_pool: MainActorPoolType,
-        cuda_devices: List[int],
+        gpu_devices: List[int],
         metrics_exporter_host: Optional[str] = None,
         metrics_exporter_port: Optional[int] = None,
     ):
         super().__init__()
         # static attrs.
-        self._total_cuda_devices = cuda_devices
+        self._total_gpu_devices = gpu_devices
         self._supervisor_address = supervisor_address
         self._supervisor_ref = None
         self._main_pool = main_pool
@@ -244,9 +244,9 @@ class WorkerActor(xo.StatelessActor):
 
     @staticmethod
     def get_devices_count():
-        from ..utils import cuda_count
+        from ..device_utils import gpu_count
 
-        return cuda_count()
+        return gpu_count()
 
     @log_sync(logger=logger)
     def get_model_count(self) -> int:
@@ -263,7 +263,7 @@ class WorkerActor(xo.StatelessActor):
         we assume that embedding model only takes 1 GPU slot.
         """
         candidates = []
-        for _dev in self._total_cuda_devices:
+        for _dev in self._total_gpu_devices:
             if _dev not in self._gpu_to_model_uid:
                 candidates.append(_dev)
             else:
@@ -291,11 +291,11 @@ class WorkerActor(xo.StatelessActor):
         return device
 
     def allocate_devices(self, model_uid: str, n_gpu: int) -> List[int]:
-        if n_gpu > len(self._total_cuda_devices) - len(self._gpu_to_model_uid):
+        if n_gpu > len(self._total_gpu_devices) - len(self._gpu_to_model_uid):
             raise RuntimeError("No available slot found for the model")
 
         devices: List[int] = [
-            dev for dev in self._total_cuda_devices if dev not in self._gpu_to_model_uid
+            dev for dev in self._total_gpu_devices if dev not in self._gpu_to_model_uid
         ][:n_gpu]
         for dev in devices:
             self._gpu_to_model_uid[int(dev)] = model_uid
@@ -324,7 +324,7 @@ class WorkerActor(xo.StatelessActor):
     ) -> Tuple[str, List[str]]:
         env = {}
         devices = []
-        if isinstance(n_gpu, int) or (n_gpu == "auto" and cuda_count() > 0):
+        if isinstance(n_gpu, int) or (n_gpu == "auto" and gpu_count() > 0):
             # Currently, n_gpu=auto means using 1 GPU
             gpu_cnt = n_gpu if isinstance(n_gpu, int) else 1
             devices = (
@@ -396,10 +396,10 @@ class WorkerActor(xo.StatelessActor):
         n_gpu: Optional[Union[int, str]] = "auto",
     ):
         if n_gpu is not None:
-            if isinstance(n_gpu, int) and (n_gpu <= 0 or n_gpu > cuda_count()):
+            if isinstance(n_gpu, int) and (n_gpu <= 0 or n_gpu > gpu_count()):
                 raise ValueError(
                     f"The parameter `n_gpu` must be greater than 0 and "
-                    f"not greater than the number of GPUs: {cuda_count()} on the machine."
+                    f"not greater than the number of GPUs: {gpu_count()} on the machine."
                 )
             if isinstance(n_gpu, str) and n_gpu != "auto":
                 raise ValueError("Currently `n_gpu` only supports `auto`.")
@@ -504,10 +504,10 @@ class WorkerActor(xo.StatelessActor):
         launch_args.pop("kwargs")
         launch_args.update(kwargs)
         if n_gpu is not None:
-            if isinstance(n_gpu, int) and (n_gpu <= 0 or n_gpu > cuda_count()):
+            if isinstance(n_gpu, int) and (n_gpu <= 0 or n_gpu > gpu_count()):
                 raise ValueError(
                     f"The parameter `n_gpu` must be greater than 0 and "
-                    f"not greater than the number of GPUs: {cuda_count()} on the machine."
+                    f"not greater than the number of GPUs: {gpu_count()} on the machine."
                 )
             if isinstance(n_gpu, str) and n_gpu != "auto":
                 raise ValueError("Currently `n_gpu` only supports `auto`.")

xinference/deploy/worker.py

@@ -21,7 +21,7 @@ import xoscar as xo
 from xoscar import MainActorPoolType
 
 from ..core.worker import WorkerActor
-from ..utils import cuda_count
+from ..device_utils import gpu_count
 
 logger = logging.getLogger(__name__)
 
@@ -33,12 +33,12 @@ async def start_worker_components(
     metrics_exporter_host: Optional[str],
     metrics_exporter_port: Optional[int],
 ):
-    cuda_device_indices = []
-    cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
-    if cuda_visible_devices:
-        cuda_device_indices.extend([int(i) for i in cuda_visible_devices.split(",")])
+    gpu_device_indices = []
+    cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
+    if cuda_visible_devices is not None and cuda_visible_devices != "-1":
+        gpu_device_indices.extend([int(i) for i in cuda_visible_devices.split(",")])
     else:
-        cuda_device_indices = list(range(cuda_count()))
+        gpu_device_indices = list(range(gpu_count()))
 
     await xo.create_actor(
         WorkerActor,
@@ -46,7 +46,7 @@ async def start_worker_components(
         uid=WorkerActor.uid(),
         supervisor_address=supervisor_address,
         main_pool=main_pool,
-        cuda_devices=cuda_device_indices,
+        gpu_devices=gpu_device_indices,
         metrics_exporter_host=metrics_exporter_host,
         metrics_exporter_port=metrics_exporter_port,
     )