xinference 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-06-07T15:04:33+0800",
+ "date": "2024-06-21T15:34:17+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "55c5636f2b6022842d1827eae373c8e5f162a1a3",
- "version": "0.12.0"
+ "full-revisionid": "5cef7c3d4bb0c5208d262fc3ffb7d7083724de1c",
+ "version": "0.12.2"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -109,6 +109,7 @@ class RerankRequest(BaseModel):
     documents: List[str]
     top_n: Optional[int] = None
     return_documents: Optional[bool] = False
+    return_len: Optional[bool] = False
     max_chunks_per_doc: Optional[int] = None
 
 
@@ -522,11 +523,31 @@ class RESTfulAPI:
             ),
         )
         self._router.add_api_route(
-            "/v1/cached/list_cached_models",
+            "/v1/cache/models",
             self.list_cached_models,
             methods=["GET"],
             dependencies=(
-                [Security(self._auth_service, scopes=["models:list"])]
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models/files",
+            self.list_model_files,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models",
+            self.confirm_and_remove_model,
+            methods=["DELETE"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:delete"])]
                 if self.is_authenticated()
                 else None
             ),
@@ -961,7 +982,8 @@ class RESTfulAPI:
         return JSONResponse(content=self._supervisor_address)
 
     async def create_completion(self, request: Request) -> Response:
-        body = CreateCompletionRequest.parse_obj(await request.json())
+        raw_body = await request.json()
+        body = CreateCompletionRequest.parse_obj(raw_body)
         exclude = {
             "prompt",
             "model",
@@ -971,6 +993,7 @@ class RESTfulAPI:
             "logit_bias_type",
             "user",
         }
+        raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
         # TODO: Decide if this default value override is necessary #1061
@@ -1000,7 +1023,9 @@ class RESTfulAPI:
                 iterator = None
                 try:
                     try:
-                        iterator = await model.generate(body.prompt, kwargs)
+                        iterator = await model.generate(
+                            body.prompt, kwargs, raw_params=raw_kwargs
+                        )
                     except RuntimeError as re:
                         self.handle_request_limit_error(re)
                     async for item in iterator:
@@ -1020,7 +1045,7 @@ class RESTfulAPI:
             return EventSourceResponse(stream_results())
         else:
             try:
-                data = await model.generate(body.prompt, kwargs)
+                data = await model.generate(body.prompt, kwargs, raw_params=raw_kwargs)
                 return Response(data, media_type="application/json")
             except Exception as e:
                 logger.error(e, exc_info=True)
@@ -1092,6 +1117,7 @@ class RESTfulAPI:
                 top_n=body.top_n,
                 max_chunks_per_doc=body.max_chunks_per_doc,
                 return_documents=body.return_documents,
+                return_len=body.return_len,
                 **kwargs,
             )
             return Response(scores, media_type="application/json")
@@ -1321,7 +1347,8 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_chat_completion(self, request: Request) -> Response:
-        body = CreateChatCompletion.parse_obj(await request.json())
+        raw_body = await request.json()
+        body = CreateChatCompletion.parse_obj(raw_body)
         exclude = {
             "prompt",
             "model",
@@ -1331,6 +1358,7 @@ class RESTfulAPI:
             "logit_bias_type",
             "user",
         }
+        raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
         # TODO: Decide if this default value override is necessary #1061
@@ -1401,9 +1429,13 @@ class RESTfulAPI:
         model_family = desc.get("model_family", "")
         function_call_models = [
             "chatglm3",
+            "glm4-chat",
             "gorilla-openfunctions-v1",
             "qwen-chat",
             "qwen1.5-chat",
+            "qwen1.5-moe-chat",
+            "qwen2-instruct",
+            "qwen2-moe-instruct",
         ]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen-chat" == model_family
@@ -1426,7 +1458,13 @@ class RESTfulAPI:
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-            if not is_vllm or model_family not in ["qwen-chat", "qwen1.5-chat"]:
+            if not is_vllm or model_family not in [
+                "qwen-chat",
+                "qwen1.5-chat",
+                "qwen1.5-moe-chat",
+                "qwen2-instruct",
+                "qwen2-moe-instruct",
+            ]:
                 raise HTTPException(
                     status_code=400,
                     detail="Streaming support for tool calls is available only when using vLLM backend and Qwen models.",
@@ -1439,10 +1477,16 @@ class RESTfulAPI:
                 try:
                     try:
                         if is_qwen:
-                            iterator = await model.chat(prompt, chat_history, kwargs)
+                            iterator = await model.chat(
+                                prompt, chat_history, kwargs, raw_params=raw_kwargs
+                            )
                         else:
                             iterator = await model.chat(
-                                prompt, system_prompt, chat_history, kwargs
+                                prompt,
+                                system_prompt,
+                                chat_history,
+                                kwargs,
+                                raw_params=raw_kwargs,
                             )
                     except RuntimeError as re:
                         await self._report_error_event(model_uid, str(re))
@@ -1472,9 +1516,17 @@ class RESTfulAPI:
         else:
             try:
                 if is_qwen:
-                    data = await model.chat(prompt, chat_history, kwargs)
+                    data = await model.chat(
+                        prompt, chat_history, kwargs, raw_params=raw_kwargs
+                    )
                 else:
-                    data = await model.chat(prompt, system_prompt, chat_history, kwargs)
+                    data = await model.chat(
+                        prompt,
+                        system_prompt,
+                        chat_history,
+                        kwargs,
+                        raw_params=raw_kwargs,
+                    )
                 return Response(content=data, media_type="application/json")
             except Exception as e:
                 logger.error(e, exc_info=True)
@@ -1555,10 +1607,17 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def list_cached_models(self) -> JSONResponse:
+    async def list_cached_models(
+        self, model_name: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
         try:
-            data = await (await self._get_supervisor_ref()).list_cached_models()
-            return JSONResponse(content=data)
+            data = await (await self._get_supervisor_ref()).list_cached_models(
+                model_name, worker_ip
+            )
+            resp = {
+                "list": data,
+            }
+            return JSONResponse(content=resp)
         except ValueError as re:
             logger.error(re, exc_info=True)
             raise HTTPException(status_code=400, detail=str(re))
@@ -1623,6 +1682,41 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def list_model_files(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).list_deletable_models(
+                model_version, worker_ip
+            )
+            response = {
+                "model_version": model_version,
+                "worker_ip": worker_ip,
+                "paths": data,
+            }
+            return JSONResponse(content=response)
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def confirm_and_remove_model(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            res = await (await self._get_supervisor_ref()).confirm_and_remove_model(
+                model_version=model_version, worker_ip=worker_ip
+            )
+            return JSONResponse(content={"result": res})
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
 
 def run(
     supervisor_address: str,

xinference/client/restful/restful_client.py

@@ -135,6 +135,7 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
         top_n: Optional[int] = None,
         max_chunks_per_doc: Optional[int] = None,
         return_documents: Optional[bool] = None,
+        return_len: Optional[bool] = None,
         **kwargs,
     ):
         """
@@ -152,6 +153,8 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
             The maximum number of chunks derived from a document
         return_documents: bool
             if return documents
+        return_len: bool
+            if return tokens len
         Returns
         -------
         Scores
@@ -170,6 +173,7 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
             "top_n": top_n,
             "max_chunks_per_doc": max_chunks_per_doc,
             "return_documents": return_documents,
+            "return_len": return_len,
         }
         request_body.update(kwargs)
         response = requests.post(url, json=request_body, headers=self.auth_headers)
@@ -1145,13 +1149,17 @@ class Client:
         response_data = response.json()
         return response_data
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, model_name: Optional[str] = None, worker_ip: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         """
         Get a list of cached models.
-
         Parameters
         ----------
-        None
+        model_name: Optional[str]
+            The name of model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
 
         Returns
         -------
@@ -1164,16 +1172,81 @@ class Client:
             Raised when the request fails, including the reason for the failure.
         """
 
-        url = f"{self.base_url}/v1/cached/list_cached_models"
-        response = requests.get(url, headers=self._headers)
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_name": model_name,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list cached model, detail: {_get_error_string(response)}"
             )
 
+        response_data = response.json()
+        response_data = response_data.get("list")
+        return response_data
+
+    def list_deletable_models(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Get the cached models with the model path cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        Dict[str, Dict[str,str]]]
+            Dictionary with keys "model_name" and values model_file_location.
+        """
+        url = f"{self.base_url}/v1/cache/models/files"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to get paths by model name, detail: {_get_error_string(response)}"
+            )
+
         response_data = response.json()
         return response_data
 
+    def confirm_and_remove_model(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> bool:
+        """
+        Remove the cached models with the model name cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        str
+            The response of the server.
+        """
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.delete(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to remove cached models, detail: {_get_error_string(response)}"
+            )
+
+        response_data = response.json()
+        return response_data.get("result", False)
+
     def get_model_registration(
         self, model_type: str, model_name: str
     ) -> Dict[str, Any]:

xinference/constants.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
 XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"
+XINFERENCE_ENV_CSG_TOKEN = "XINFERENCE_CSG_TOKEN"
 XINFERENCE_ENV_HOME_PATH = "XINFERENCE_HOME"
 XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD = (
     "XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD"

xinference/core/cache_tracker.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 from logging import getLogger
 from typing import Any, Dict, List, Optional
 
@@ -102,33 +101,54 @@ class CacheTrackerActor(xo.Actor):
     def get_model_version_count(self, model_name: str) -> int:
         return len(self.get_model_versions(model_name))
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, worker_ip: str, model_name: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         cached_models = []
-        for model_name, model_versions in self._model_name_to_version_info.items():
-            for version_info in model_versions:
-                cache_status = version_info.get("cache_status", None)
-                if cache_status == True:
-                    ret = version_info.copy()
-                    ret["model_name"] = model_name
+        for name, versions in self._model_name_to_version_info.items():
+            # only return assigned cached model if model_name is not none
+            # else return all cached model
+            if model_name and model_name != name:
+                continue
+            for version_info in versions:
+                cache_status = version_info.get("cache_status", False)
+                # search cached model
+                if cache_status:
+                    res = version_info.copy()
+                    res["model_name"] = name
+                    paths = res.get("model_file_location", {})
+                    # only return assigned worker's device path
+                    if worker_ip in paths.keys():
+                        res["model_file_location"] = paths[worker_ip]
+                        cached_models.append(res)
+        return cached_models
 
-                    re_dict = version_info.get("model_file_location", None)
-                    if re_dict is not None and isinstance(re_dict, dict):
-                        if re_dict:
-                            actor_ip_address, path = next(iter(re_dict.items()))
-                        else:
-                            raise ValueError("The dictionary is empty.")
-                    else:
-                        raise ValueError("re_dict must be a non-empty dictionary.")
+    def list_deletable_models(self, model_version: str, worker_ip: str) -> str:
+        model_file_location = ""
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # search assign model version
+                if model_version == version_info.get("model_version", None):
+                    # check if exist
+                    if version_info.get("cache_status", False):
+                        paths = version_info.get("model_file_location", {})
+                        # only return assigned worker's device path
+                        if worker_ip in paths.keys():
+                            model_file_location = paths[worker_ip]
+        return model_file_location
 
-                    ret["actor_ip_address"] = actor_ip_address
-                    ret["path"] = path
-                    if os.path.isdir(path):
-                        files = os.listdir(path)
-                        resolved_file = os.path.realpath(os.path.join(path, files[0]))
-                        if resolved_file:
-                            ret["real_path"] = os.path.dirname(resolved_file)
-                    else:
-                        ret["real_path"] = os.path.realpath(path)
-                    cached_models.append(ret)
-        cached_models = sorted(cached_models, key=lambda x: x["model_name"])
-        return cached_models
+    def confirm_and_remove_model(self, model_version: str, worker_ip: str):
+        # find remove path
+        rm_path = self.list_deletable_models(model_version, worker_ip)
+        # search _model_name_to_version_info if exist this path, and delete
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # check if exist
+                if version_info.get("cache_status", False):
+                    paths = version_info.get("model_file_location", {})
+                    # only delete assigned worker's device path
+                    if worker_ip in paths.keys() and rm_path == paths[worker_ip]:
+                        del paths[worker_ip]
+                        # if path is empty, update cache status
+                        if not paths:
+                            version_info["cache_status"] = False

xinference/core/event.py

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import queue
-from collections import defaultdict
+from collections import defaultdict, deque
 from enum import Enum
 from typing import Dict, List, TypedDict
 
@@ -37,8 +36,8 @@ class Event(TypedDict):
 class EventCollectorActor(xo.StatelessActor):
     def __init__(self):
         super().__init__()
-        self._model_uid_to_events: Dict[str, queue.Queue] = defaultdict(  # type: ignore
-            lambda: queue.Queue(maxsize=MAX_EVENT_COUNT_PER_MODEL)
+        self._model_uid_to_events: Dict[str, deque] = defaultdict(  # type: ignore
+            lambda: deque(maxlen=MAX_EVENT_COUNT_PER_MODEL)
         )
 
     @classmethod
@@ -50,7 +49,7 @@ class EventCollectorActor(xo.StatelessActor):
         if event_queue is None:
             return []
         else:
-            return [dict(e, event_type=e["event_type"].name) for e in event_queue.queue]
+            return [dict(e, event_type=e["event_type"].name) for e in iter(event_queue)]
 
     def report_event(self, model_uid: str, event: Event):
-        self._model_uid_to_events[model_uid].put(event)
+        self._model_uid_to_events[model_uid].append(event)

xinference/core/model.py

@@ -264,12 +264,14 @@ class ModelActor(xo.StatelessActor):
         return isinstance(self._model, VLLMModel)
 
     def allow_batching(self) -> bool:
-        from ..model.llm.pytorch.core import PytorchChatModel
+        from ..model.llm.pytorch.core import PytorchModel
+
+        model_ability = self._model_description.get("model_ability", [])
 
         return (
             XINFERENCE_TRANSFORMERS_ENABLE_BATCHING
-            and isinstance(self._model, PytorchChatModel)
-            and self._model.__class__.__name__ == PytorchChatModel.__name__
+            and isinstance(self._model, PytorchModel)
+            and "vision" not in model_ability
         )
 
     async def load(self):
@@ -393,18 +395,25 @@ class ModelActor(xo.StatelessActor):
     @request_limit
     @xo.generator
     async def generate(self, prompt: str, *args, **kwargs):
-        if hasattr(self._model, "generate"):
-            return await self._call_wrapper(
-                self._model.generate, prompt, *args, **kwargs
-            )
-        if hasattr(self._model, "async_generate"):
-            return await self._call_wrapper(
-                self._model.async_generate, prompt, *args, **kwargs
+        if self.allow_batching():
+            return await self.handle_batching_request(
+                prompt, "generate", *args, **kwargs
             )
-        raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
+        else:
+            kwargs.pop("raw_params", None)
+            if hasattr(self._model, "generate"):
+                return await self._call_wrapper(
+                    self._model.generate, prompt, *args, **kwargs
+                )
+            if hasattr(self._model, "async_generate"):
+                return await self._call_wrapper(
+                    self._model.async_generate, prompt, *args, **kwargs
+                )
+            raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
 
+    @staticmethod
     async def _queue_consumer(
-        self, queue: Queue, timeout: Optional[float] = None
+        queue: Queue, timeout: Optional[float] = None
     ) -> AsyncIterator[Any]:
         from .scheduler import (
             XINFERENCE_STREAMING_ABORT_FLAG,
@@ -429,9 +438,38 @@ class ModelActor(xo.StatelessActor):
                 yield res
 
     @staticmethod
-    def get_stream_from_args(*args) -> bool:
-        assert args[2] is None or isinstance(args[2], dict)
-        return False if args[2] is None else args[2].get("stream", False)
+    def _get_stream_from_args(ability: str, *args) -> bool:
+        if ability == "chat":
+            assert args[2] is None or isinstance(args[2], dict)
+            return False if args[2] is None else args[2].get("stream", False)
+        else:
+            assert args[0] is None or isinstance(args[0], dict)
+            return False if args[0] is None else args[0].get("stream", False)
+
+    async def handle_batching_request(self, prompt: str, ability: str, *args, **kwargs):
+        stream = self._get_stream_from_args(ability, *args)
+        assert self._scheduler_ref is not None
+        if stream:
+            assert self._scheduler_ref is not None
+            queue: Queue[Any] = Queue()
+            ret = self._queue_consumer(queue)
+            await self._scheduler_ref.add_request(prompt, queue, *args, **kwargs)
+            gen = self._to_json_async_gen(ret)
+            self._current_generator = weakref.ref(gen)
+            return gen
+        else:
+            from .scheduler import XINFERENCE_NON_STREAMING_ABORT_FLAG
+
+            assert self._loop is not None
+            future = ConcurrentFuture()
+            await self._scheduler_ref.add_request(prompt, future, *args, **kwargs)
+            fut = asyncio.wrap_future(future, loop=self._loop)
+            result = await fut
+            if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
+                raise RuntimeError(
+                    f"This request has been cancelled by another `abort_request` request."
+                )
+            return await asyncio.to_thread(json_dumps, result)
 
     @log_async(logger=logger)
     @request_limit
@@ -441,34 +479,11 @@ class ModelActor(xo.StatelessActor):
         response = None
         try:
             if self.allow_batching():
-                stream = self.get_stream_from_args(*args)
-                assert self._scheduler_ref is not None
-                if stream:
-                    assert self._scheduler_ref is not None
-                    queue: Queue[Any] = Queue()
-                    ret = self._queue_consumer(queue)
-                    await self._scheduler_ref.add_request(
-                        prompt, queue, *args, **kwargs
-                    )
-                    gen = self._to_json_async_gen(ret)
-                    self._current_generator = weakref.ref(gen)
-                    return gen
-                else:
-                    from .scheduler import XINFERENCE_NON_STREAMING_ABORT_FLAG
-
-                    assert self._loop is not None
-                    future = ConcurrentFuture()
-                    await self._scheduler_ref.add_request(
-                        prompt, future, *args, **kwargs
-                    )
-                    fut = asyncio.wrap_future(future, loop=self._loop)
-                    result = await fut
-                    if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
-                        raise RuntimeError(
-                            f"This request has been cancelled by another `abort_request` request."
-                        )
-                    return await asyncio.to_thread(json_dumps, result)
+                return await self.handle_batching_request(
+                    prompt, "chat", *args, **kwargs
+                )
             else:
+                kwargs.pop("raw_params", None)
                 if hasattr(self._model, "chat"):
                     response = await self._call_wrapper(
                         self._model.chat, prompt, *args, **kwargs
@@ -528,6 +543,7 @@ class ModelActor(xo.StatelessActor):
         top_n: Optional[int],
         max_chunks_per_doc: Optional[int],
         return_documents: Optional[bool],
+        return_len: Optional[bool],
         *args,
         **kwargs,
     ):
@@ -539,6 +555,7 @@ class ModelActor(xo.StatelessActor):
                 top_n,
                 max_chunks_per_doc,
                 return_documents,
+                return_len,
                 *args,
                 **kwargs,
             )