xinference 0.12.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-06-07T15:04:33+0800",
+ "date": "2024-06-14T17:17:50+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "55c5636f2b6022842d1827eae373c8e5f162a1a3",
- "version": "0.12.0"
+ "full-revisionid": "34a57df449f0890415c424802d3596f3c8758412",
+ "version": "0.12.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -522,11 +522,31 @@ class RESTfulAPI:
             ),
         )
         self._router.add_api_route(
-            "/v1/cached/list_cached_models",
+            "/v1/cache/models",
             self.list_cached_models,
             methods=["GET"],
             dependencies=(
-                [Security(self._auth_service, scopes=["models:list"])]
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models/files",
+            self.list_model_files,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models",
+            self.confirm_and_remove_model,
+            methods=["DELETE"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:delete"])]
                 if self.is_authenticated()
                 else None
             ),
@@ -1401,9 +1421,11 @@ class RESTfulAPI:
         model_family = desc.get("model_family", "")
         function_call_models = [
             "chatglm3",
+            "glm4-chat",
             "gorilla-openfunctions-v1",
             "qwen-chat",
             "qwen1.5-chat",
+            "qwen2-instruct",
         ]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen-chat" == model_family
@@ -1426,7 +1448,11 @@ class RESTfulAPI:
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-            if not is_vllm or model_family not in ["qwen-chat", "qwen1.5-chat"]:
+            if not is_vllm or model_family not in [
+                "qwen-chat",
+                "qwen1.5-chat",
+                "qwen2-instruct",
+            ]:
                 raise HTTPException(
                     status_code=400,
                     detail="Streaming support for tool calls is available only when using vLLM backend and Qwen models.",
@@ -1555,10 +1581,17 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def list_cached_models(self) -> JSONResponse:
+    async def list_cached_models(
+        self, model_name: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
         try:
-            data = await (await self._get_supervisor_ref()).list_cached_models()
-            return JSONResponse(content=data)
+            data = await (await self._get_supervisor_ref()).list_cached_models(
+                model_name, worker_ip
+            )
+            resp = {
+                "list": data,
+            }
+            return JSONResponse(content=resp)
         except ValueError as re:
             logger.error(re, exc_info=True)
             raise HTTPException(status_code=400, detail=str(re))
@@ -1623,6 +1656,41 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def list_model_files(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).list_deletable_models(
+                model_version, worker_ip
+            )
+            response = {
+                "model_version": model_version,
+                "worker_ip": worker_ip,
+                "paths": data,
+            }
+            return JSONResponse(content=response)
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def confirm_and_remove_model(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            res = await (await self._get_supervisor_ref()).confirm_and_remove_model(
+                model_version=model_version, worker_ip=worker_ip
+            )
+            return JSONResponse(content={"result": res})
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
 
 def run(
     supervisor_address: str,

xinference/client/restful/restful_client.py

@@ -1145,13 +1145,17 @@ class Client:
         response_data = response.json()
         return response_data
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, model_name: Optional[str] = None, worker_ip: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         """
         Get a list of cached models.
-
         Parameters
         ----------
-        None
+        model_name: Optional[str]
+            The name of model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
 
         Returns
         -------
@@ -1164,16 +1168,81 @@ class Client:
             Raised when the request fails, including the reason for the failure.
         """
 
-        url = f"{self.base_url}/v1/cached/list_cached_models"
-        response = requests.get(url, headers=self._headers)
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_name": model_name,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list cached model, detail: {_get_error_string(response)}"
             )
 
+        response_data = response.json()
+        response_data = response_data.get("list")
+        return response_data
+
+    def list_deletable_models(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Get the cached models with the model path cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        Dict[str, Dict[str,str]]]
+            Dictionary with keys "model_name" and values model_file_location.
+        """
+        url = f"{self.base_url}/v1/cache/models/files"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to get paths by model name, detail: {_get_error_string(response)}"
+            )
+
         response_data = response.json()
         return response_data
 
+    def confirm_and_remove_model(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> bool:
+        """
+        Remove the cached models with the model name cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        str
+            The response of the server.
+        """
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.delete(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to remove cached models, detail: {_get_error_string(response)}"
+            )
+
+        response_data = response.json()
+        return response_data.get("result", False)
+
     def get_model_registration(
         self, model_type: str, model_name: str
     ) -> Dict[str, Any]:

xinference/constants.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
 XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"
+XINFERENCE_ENV_CSG_TOKEN = "XINFERENCE_CSG_TOKEN"
 XINFERENCE_ENV_HOME_PATH = "XINFERENCE_HOME"
 XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD = (
     "XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD"

xinference/core/cache_tracker.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 from logging import getLogger
 from typing import Any, Dict, List, Optional
 
@@ -102,33 +101,54 @@ class CacheTrackerActor(xo.Actor):
     def get_model_version_count(self, model_name: str) -> int:
         return len(self.get_model_versions(model_name))
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, worker_ip: str, model_name: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         cached_models = []
-        for model_name, model_versions in self._model_name_to_version_info.items():
-            for version_info in model_versions:
-                cache_status = version_info.get("cache_status", None)
-                if cache_status == True:
-                    ret = version_info.copy()
-                    ret["model_name"] = model_name
+        for name, versions in self._model_name_to_version_info.items():
+            # only return assigned cached model if model_name is not none
+            # else return all cached model
+            if model_name and model_name != name:
+                continue
+            for version_info in versions:
+                cache_status = version_info.get("cache_status", False)
+                # search cached model
+                if cache_status:
+                    res = version_info.copy()
+                    res["model_name"] = name
+                    paths = res.get("model_file_location", {})
+                    # only return assigned worker's device path
+                    if worker_ip in paths.keys():
+                        res["model_file_location"] = paths[worker_ip]
+                        cached_models.append(res)
+        return cached_models
 
-                    re_dict = version_info.get("model_file_location", None)
-                    if re_dict is not None and isinstance(re_dict, dict):
-                        if re_dict:
-                            actor_ip_address, path = next(iter(re_dict.items()))
-                        else:
-                            raise ValueError("The dictionary is empty.")
-                    else:
-                        raise ValueError("re_dict must be a non-empty dictionary.")
+    def list_deletable_models(self, model_version: str, worker_ip: str) -> str:
+        model_file_location = ""
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # search assign model version
+                if model_version == version_info.get("model_version", None):
+                    # check if exist
+                    if version_info.get("cache_status", False):
+                        paths = version_info.get("model_file_location", {})
+                        # only return assigned worker's device path
+                        if worker_ip in paths.keys():
+                            model_file_location = paths[worker_ip]
+        return model_file_location
 
-                    ret["actor_ip_address"] = actor_ip_address
-                    ret["path"] = path
-                    if os.path.isdir(path):
-                        files = os.listdir(path)
-                        resolved_file = os.path.realpath(os.path.join(path, files[0]))
-                        if resolved_file:
-                            ret["real_path"] = os.path.dirname(resolved_file)
-                    else:
-                        ret["real_path"] = os.path.realpath(path)
-                    cached_models.append(ret)
-        cached_models = sorted(cached_models, key=lambda x: x["model_name"])
-        return cached_models
+    def confirm_and_remove_model(self, model_version: str, worker_ip: str):
+        # find remove path
+        rm_path = self.list_deletable_models(model_version, worker_ip)
+        # search _model_name_to_version_info if exist this path, and delete
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # check if exist
+                if version_info.get("cache_status", False):
+                    paths = version_info.get("model_file_location", {})
+                    # only delete assigned worker's device path
+                    if worker_ip in paths.keys() and rm_path == paths[worker_ip]:
+                        del paths[worker_ip]
+                        # if path is empty, update cache status
+                        if not paths:
+                            version_info["cache_status"] = False

xinference/core/model.py

@@ -264,12 +264,13 @@ class ModelActor(xo.StatelessActor):
         return isinstance(self._model, VLLMModel)
 
     def allow_batching(self) -> bool:
-        from ..model.llm.pytorch.core import PytorchChatModel
+        from ..model.llm.pytorch.core import PytorchChatModel, PytorchModel
 
         return (
             XINFERENCE_TRANSFORMERS_ENABLE_BATCHING
-            and isinstance(self._model, PytorchChatModel)
-            and self._model.__class__.__name__ == PytorchChatModel.__name__
+            and isinstance(self._model, PytorchModel)
+            and self._model.__class__.__name__
+            in (PytorchChatModel.__name__, PytorchModel.__name__)
         )
 
     async def load(self):
@@ -393,18 +394,24 @@ class ModelActor(xo.StatelessActor):
     @request_limit
     @xo.generator
     async def generate(self, prompt: str, *args, **kwargs):
-        if hasattr(self._model, "generate"):
-            return await self._call_wrapper(
-                self._model.generate, prompt, *args, **kwargs
-            )
-        if hasattr(self._model, "async_generate"):
-            return await self._call_wrapper(
-                self._model.async_generate, prompt, *args, **kwargs
+        if self.allow_batching():
+            return await self.handle_batching_request(
+                prompt, "generate", *args, **kwargs
             )
-        raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
+        else:
+            if hasattr(self._model, "generate"):
+                return await self._call_wrapper(
+                    self._model.generate, prompt, *args, **kwargs
+                )
+            if hasattr(self._model, "async_generate"):
+                return await self._call_wrapper(
+                    self._model.async_generate, prompt, *args, **kwargs
+                )
+            raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
 
+    @staticmethod
     async def _queue_consumer(
-        self, queue: Queue, timeout: Optional[float] = None
+        queue: Queue, timeout: Optional[float] = None
     ) -> AsyncIterator[Any]:
         from .scheduler import (
             XINFERENCE_STREAMING_ABORT_FLAG,
@@ -429,9 +436,38 @@ class ModelActor(xo.StatelessActor):
                 yield res
 
     @staticmethod
-    def get_stream_from_args(*args) -> bool:
-        assert args[2] is None or isinstance(args[2], dict)
-        return False if args[2] is None else args[2].get("stream", False)
+    def _get_stream_from_args(ability: str, *args) -> bool:
+        if ability == "chat":
+            assert args[2] is None or isinstance(args[2], dict)
+            return False if args[2] is None else args[2].get("stream", False)
+        else:
+            assert args[0] is None or isinstance(args[0], dict)
+            return False if args[0] is None else args[0].get("stream", False)
+
+    async def handle_batching_request(self, prompt: str, ability: str, *args, **kwargs):
+        stream = self._get_stream_from_args(ability, *args)
+        assert self._scheduler_ref is not None
+        if stream:
+            assert self._scheduler_ref is not None
+            queue: Queue[Any] = Queue()
+            ret = self._queue_consumer(queue)
+            await self._scheduler_ref.add_request(prompt, queue, *args, **kwargs)
+            gen = self._to_json_async_gen(ret)
+            self._current_generator = weakref.ref(gen)
+            return gen
+        else:
+            from .scheduler import XINFERENCE_NON_STREAMING_ABORT_FLAG
+
+            assert self._loop is not None
+            future = ConcurrentFuture()
+            await self._scheduler_ref.add_request(prompt, future, *args, **kwargs)
+            fut = asyncio.wrap_future(future, loop=self._loop)
+            result = await fut
+            if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
+                raise RuntimeError(
+                    f"This request has been cancelled by another `abort_request` request."
+                )
+            return await asyncio.to_thread(json_dumps, result)
 
     @log_async(logger=logger)
     @request_limit
@@ -441,33 +477,9 @@ class ModelActor(xo.StatelessActor):
         response = None
         try:
             if self.allow_batching():
-                stream = self.get_stream_from_args(*args)
-                assert self._scheduler_ref is not None
-                if stream:
-                    assert self._scheduler_ref is not None
-                    queue: Queue[Any] = Queue()
-                    ret = self._queue_consumer(queue)
-                    await self._scheduler_ref.add_request(
-                        prompt, queue, *args, **kwargs
-                    )
-                    gen = self._to_json_async_gen(ret)
-                    self._current_generator = weakref.ref(gen)
-                    return gen
-                else:
-                    from .scheduler import XINFERENCE_NON_STREAMING_ABORT_FLAG
-
-                    assert self._loop is not None
-                    future = ConcurrentFuture()
-                    await self._scheduler_ref.add_request(
-                        prompt, future, *args, **kwargs
-                    )
-                    fut = asyncio.wrap_future(future, loop=self._loop)
-                    result = await fut
-                    if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
-                        raise RuntimeError(
-                            f"This request has been cancelled by another `abort_request` request."
-                        )
-                    return await asyncio.to_thread(json_dumps, result)
+                return await self.handle_batching_request(
+                    prompt, "chat", *args, **kwargs
+                )
             else:
                 if hasattr(self._model, "chat"):
                     response = await self._call_wrapper(

xinference/core/scheduler.py

@@ -15,6 +15,7 @@
 import asyncio
 import functools
 import logging
+import uuid
 from collections import deque
 from enum import Enum
 from typing import List, Optional, Set
@@ -50,9 +51,9 @@ class InferenceRequest:
         self._new_tokens = []
         # kv_cache used in decode phase
         self._kv_cache = None
-        # use passed args from `chat` interface
+        # use passed args from upstream interface
         self._inference_args = args
-        # use passed kwargs from `chat` interface, basically not used for now
+        # use passed kwargs from upstream interface, basically not used for now
         self._inference_kwargs = kwargs
         # should this request be stopped
         self._stopped = False
@@ -63,6 +64,8 @@ class InferenceRequest:
         self._aborted = False
         # sanitized generate config
         self._sanitized_generate_config = None
+        # Chunk id for results. In stream mode, all the chunk ids should be same.
+        self._stream_chunk_id = str(uuid.uuid4())
         # Use in stream mode
         self.last_output_length = 0
         # inference results,
@@ -81,19 +84,26 @@ class InferenceRequest:
         self._check_args()
 
     def _check_args(self):
-        assert len(self._inference_args) == 3
-        # system prompt
-        assert self._inference_args[0] is None or isinstance(
-            self._inference_args[0], str
-        )
-        # chat history
-        assert self._inference_args[1] is None or isinstance(
-            self._inference_args[1], list
-        )
-        # generate config
-        assert self._inference_args[2] is None or isinstance(
-            self._inference_args[2], dict
-        )
+        # chat
+        if len(self._inference_args) == 3:
+            # system prompt
+            assert self._inference_args[0] is None or isinstance(
+                self._inference_args[0], str
+            )
+            # chat history
+            assert self._inference_args[1] is None or isinstance(
+                self._inference_args[1], list
+            )
+            # generate config
+            assert self._inference_args[2] is None or isinstance(
+                self._inference_args[2], dict
+            )
+        else:  # generate
+            assert len(self._inference_args) == 1
+            # generate config
+            assert self._inference_args[0] is None or isinstance(
+                self._inference_args[0], dict
+            )
 
     @property
     def prompt(self):
@@ -148,7 +158,11 @@ class InferenceRequest:
 
     @property
     def generate_config(self):
-        return self._inference_args[2]
+        return (
+            self._inference_args[2]
+            if len(self._inference_args) == 3
+            else self._inference_args[0]
+        )
 
     @property
     def sanitized_generate_config(self):
@@ -174,6 +188,10 @@ class InferenceRequest:
     def finish_reason(self, value: Optional[str]):
         self._finish_reason = value
 
+    @property
+    def chunk_id(self):
+        return self._stream_chunk_id
+
     @property
     def stream(self) -> bool:
         return (

xinference/core/supervisor.py

@@ -982,32 +982,31 @@ class SupervisorActor(xo.StatelessActor):
         )
 
     @log_async(logger=logger)
-    async def list_cached_models(self) -> List[Dict[str, Any]]:
+    async def list_cached_models(
+        self, model_name: Optional[str] = None, worker_ip: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        # search assigned worker and return
+        if target_ip_worker_ref:
+            cached_models = await target_ip_worker_ref.list_cached_models(model_name)
+            cached_models = sorted(cached_models, key=lambda x: x["model_name"])
+            return cached_models
+
+        # search all worker
         cached_models = []
         for worker in self._worker_address_to_worker.values():
-            ret = await worker.list_cached_models()
-            for model_version in ret:
-                model_name = model_version.get("model_name", None)
-                model_format = model_version.get("model_format", None)
-                model_size_in_billions = model_version.get(
-                    "model_size_in_billions", None
-                )
-                quantizations = model_version.get("quantization", None)
-                actor_ip_address = model_version.get("actor_ip_address", None)
-                path = model_version.get("path", None)
-                real_path = model_version.get("real_path", None)
-
-                cache_entry = {
-                    "model_name": model_name,
-                    "model_format": model_format,
-                    "model_size_in_billions": model_size_in_billions,
-                    "quantizations": quantizations,
-                    "path": path,
-                    "Actor IP Address": actor_ip_address,
-                    "real_path": real_path,
-                }
-
-                cached_models.append(cache_entry)
+            res = await worker.list_cached_models(model_name)
+            cached_models.extend(res)
+        cached_models = sorted(cached_models, key=lambda x: x["model_name"])
         return cached_models
 
     @log_async(logger=logger)
@@ -1083,6 +1082,56 @@ class SupervisorActor(xo.StatelessActor):
             worker_status.update_time = time.time()
             worker_status.status = status
 
+    async def list_deletable_models(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> List[str]:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        ret = []
+        if target_ip_worker_ref:
+            ret = await target_ip_worker_ref.list_deletable_models(
+                model_version=model_version,
+            )
+            return ret
+
+        for worker in self._worker_address_to_worker.values():
+            path = await worker.list_deletable_models(model_version=model_version)
+            ret.extend(path)
+        return ret
+
+    async def confirm_and_remove_model(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> bool:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        if target_ip_worker_ref:
+            ret = await target_ip_worker_ref.confirm_and_remove_model(
+                model_version=model_version,
+            )
+            return ret
+        ret = True
+        for worker in self._worker_address_to_worker.values():
+            ret = ret and await worker.confirm_and_remove_model(
+                model_version=model_version,
+            )
+        return ret
+
     @staticmethod
     def record_metrics(name, op, kwargs):
         record_metrics(name, op, kwargs)