xinference 0.11.3__py3-none-any.whl → 0.12.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-05-31T17:12:13+0800",
+ "date": "2024-06-14T17:17:50+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "69c09cd068a530cd2fdcac07e4e81f03d48f04f9",
- "version": "0.11.3"
+ "full-revisionid": "34a57df449f0890415c424802d3596f3c8758412",
+ "version": "0.12.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -122,6 +122,14 @@ class TextToImageRequest(BaseModel):
     user: Optional[str] = None
 
 
+class SpeechRequest(BaseModel):
+    model: str
+    input: str
+    voice: Optional[str]
+    response_format: Optional[str] = "mp3"
+    speed: Optional[float] = 1.0
+
+
 class RegisterModelRequest(BaseModel):
     model: str
     persist: bool
@@ -337,6 +345,16 @@ class RESTfulAPI:
                 else None
             ),
         )
+        self._router.add_api_route(
+            "/v1/models/{model_uid}/requests/{request_id}/abort",
+            self.abort_request,
+            methods=["POST"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/models/instance",
             self.launch_model_by_version,
@@ -418,6 +436,16 @@ class RESTfulAPI:
                 else None
             ),
         )
+        self._router.add_api_route(
+            "/v1/audio/speech",
+            self.create_speech,
+            methods=["POST"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/images/generations",
             self.create_images,
@@ -494,11 +522,31 @@ class RESTfulAPI:
             ),
         )
         self._router.add_api_route(
-            "/v1/cached/list_cached_models",
+            "/v1/cache/models",
             self.list_cached_models,
             methods=["GET"],
             dependencies=(
-                [Security(self._auth_service, scopes=["models:list"])]
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models/files",
+            self.list_model_files,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:list"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/v1/cache/models",
+            self.confirm_and_remove_model,
+            methods=["DELETE"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["cache:delete"])]
                 if self.is_authenticated()
                 else None
             ),
@@ -1179,6 +1227,38 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def create_speech(self, request: Request) -> Response:
+        body = SpeechRequest.parse_obj(await request.json())
+        model_uid = body.model
+        try:
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            out = await model.speech(
+                input=body.input,
+                voice=body.voice,
+                response_format=body.response_format,
+                speed=body.speed,
+            )
+            return Response(media_type="application/octet-stream", content=out)
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def create_images(self, request: Request) -> Response:
         body = TextToImageRequest.parse_obj(await request.json())
         model_uid = body.model
@@ -1341,9 +1421,11 @@ class RESTfulAPI:
         model_family = desc.get("model_family", "")
         function_call_models = [
             "chatglm3",
+            "glm4-chat",
             "gorilla-openfunctions-v1",
             "qwen-chat",
             "qwen1.5-chat",
+            "qwen2-instruct",
         ]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen-chat" == model_family
@@ -1366,7 +1448,11 @@ class RESTfulAPI:
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-            if not is_vllm or model_family not in ["qwen-chat", "qwen1.5-chat"]:
+            if not is_vllm or model_family not in [
+                "qwen-chat",
+                "qwen1.5-chat",
+                "qwen2-instruct",
+            ]:
                 raise HTTPException(
                     status_code=400,
                     detail="Streaming support for tool calls is available only when using vLLM backend and Qwen models.",
@@ -1495,10 +1581,17 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def list_cached_models(self) -> JSONResponse:
+    async def list_cached_models(
+        self, model_name: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
         try:
-            data = await (await self._get_supervisor_ref()).list_cached_models()
-            return JSONResponse(content=data)
+            data = await (await self._get_supervisor_ref()).list_cached_models(
+                model_name, worker_ip
+            )
+            resp = {
+                "list": data,
+            }
+            return JSONResponse(content=resp)
         except ValueError as re:
             logger.error(re, exc_info=True)
             raise HTTPException(status_code=400, detail=str(re))
@@ -1518,6 +1611,15 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def abort_request(self, model_uid: str, request_id: str) -> JSONResponse:
+        try:
+            supervisor_ref = await self._get_supervisor_ref()
+            res = await supervisor_ref.abort_request(model_uid, request_id)
+            return JSONResponse(content=res)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def list_vllm_supported_model_families(self) -> JSONResponse:
         try:
             from ..model.llm.vllm.core import (
@@ -1554,6 +1656,41 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def list_model_files(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).list_deletable_models(
+                model_version, worker_ip
+            )
+            response = {
+                "model_version": model_version,
+                "worker_ip": worker_ip,
+                "paths": data,
+            }
+            return JSONResponse(content=response)
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def confirm_and_remove_model(
+        self, model_version: str = Query(None), worker_ip: str = Query(None)
+    ) -> JSONResponse:
+        try:
+            res = await (await self._get_supervisor_ref()).confirm_and_remove_model(
+                model_version=model_version, worker_ip=worker_ip
+            )
+            return JSONResponse(content={"result": res})
+        except ValueError as re:
+            logger.error(re, exc_info=True)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
 
 def run(
     supervisor_address: str,

xinference/client/restful/restful_client.py

@@ -684,6 +684,49 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
         response_data = response.json()
         return response_data
 
+    def speech(
+        self,
+        input: str,
+        voice: str = "",
+        response_format: str = "mp3",
+        speed: float = 1.0,
+    ):
+        """
+        Generates audio from the input text.
+
+        Parameters
+        ----------
+
+        input: str
+            The text to generate audio for. The maximum length is 4096 characters.
+        voice: str
+            The voice to use when generating the audio.
+        response_format: str
+            The format to audio in.
+        speed: str
+            The speed of the generated audio.
+
+        Returns
+        -------
+        bytes
+            The generated audio binary.
+        """
+        url = f"{self._base_url}/v1/audio/speech"
+        params = {
+            "model": self._model_uid,
+            "input": input,
+            "voice": voice,
+            "response_format": response_format,
+            "speed": speed,
+        }
+        response = requests.post(url, json=params, headers=self.auth_headers)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to speech the text, detail: {_get_error_string(response)}"
+            )
+
+        return response.content
+
 
 class Client:
     def __init__(self, base_url, api_key: Optional[str] = None):
@@ -1102,13 +1145,17 @@ class Client:
         response_data = response.json()
         return response_data
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, model_name: Optional[str] = None, worker_ip: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         """
         Get a list of cached models.
-
         Parameters
         ----------
-        None
+        model_name: Optional[str]
+            The name of model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
 
         Returns
         -------
@@ -1121,16 +1168,81 @@ class Client:
             Raised when the request fails, including the reason for the failure.
         """
 
-        url = f"{self.base_url}/v1/cached/list_cached_models"
-        response = requests.get(url, headers=self._headers)
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_name": model_name,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list cached model, detail: {_get_error_string(response)}"
             )
 
+        response_data = response.json()
+        response_data = response_data.get("list")
+        return response_data
+
+    def list_deletable_models(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Get the cached models with the model path cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        Dict[str, Dict[str,str]]]
+            Dictionary with keys "model_name" and values model_file_location.
+        """
+        url = f"{self.base_url}/v1/cache/models/files"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.get(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to get paths by model name, detail: {_get_error_string(response)}"
+            )
+
         response_data = response.json()
         return response_data
 
+    def confirm_and_remove_model(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> bool:
+        """
+        Remove the cached models with the model name cached on the server.
+        Parameters
+        ----------
+        model_version: str
+            The version of the model.
+        worker_ip: Optional[str]
+            Specify the worker ip where the model is located in a distributed scenario.
+        Returns
+        -------
+        str
+            The response of the server.
+        """
+        url = f"{self.base_url}/v1/cache/models"
+        params = {
+            "model_version": model_version,
+            "worker_ip": worker_ip,
+        }
+        response = requests.delete(url, headers=self._headers, params=params)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to remove cached models, detail: {_get_error_string(response)}"
+            )
+
+        response_data = response.json()
+        return response_data.get("result", False)
+
     def get_model_registration(
         self, model_type: str, model_name: str
     ) -> Dict[str, Any]:
@@ -1181,3 +1293,30 @@ class Client:
 
         response_data = response.json()
         return response_data
+
+    def abort_request(self, model_uid: str, request_id: str):
+        """
+        Abort a request.
+        Abort a submitted request. If the request is finished or not found, this method will be a no-op.
+        Currently, this interface is only supported when batching is enabled for models on transformers backend.
+
+        Parameters
+        ----------
+        model_uid: str
+            Model uid.
+        request_id: str
+            Request id.
+        Returns
+        -------
+        Dict
+            Return empty dict.
+        """
+        url = f"{self.base_url}/v1/models/{model_uid}/requests/{request_id}/abort"
+        response = requests.post(url, headers=self._headers)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to abort request, detail: {_get_error_string(response)}"
+            )
+
+        response_data = response.json()
+        return response_data

xinference/constants.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
 XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"
+XINFERENCE_ENV_CSG_TOKEN = "XINFERENCE_CSG_TOKEN"
 XINFERENCE_ENV_HOME_PATH = "XINFERENCE_HOME"
 XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD = (
     "XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD"
@@ -27,6 +28,7 @@ XINFERENCE_ENV_DISABLE_HEALTH_CHECK = "XINFERENCE_DISABLE_HEALTH_CHECK"
 XINFERENCE_ENV_DISABLE_VLLM = "XINFERENCE_DISABLE_VLLM"
 XINFERENCE_ENV_ENABLE_SGLANG = "XINFERENCE_ENABLE_SGLANG"
 XINFERENCE_ENV_DISABLE_METRICS = "XINFERENCE_DISABLE_METRICS"
+XINFERENCE_ENV_TRANSFORMERS_ENABLE_BATCHING = "XINFERENCE_TRANSFORMERS_ENABLE_BATCHING"
 
 
 def get_xinference_home() -> str:
@@ -70,3 +72,6 @@ XINFERENCE_ENABLE_SGLANG = bool(int(os.environ.get(XINFERENCE_ENV_ENABLE_SGLANG,
 XINFERENCE_DISABLE_METRICS = bool(
     int(os.environ.get(XINFERENCE_ENV_DISABLE_METRICS, 0))
 )
+XINFERENCE_TRANSFORMERS_ENABLE_BATCHING = bool(
+    int(os.environ.get(XINFERENCE_ENV_TRANSFORMERS_ENABLE_BATCHING, 0))
+)

xinference/core/cache_tracker.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 from logging import getLogger
 from typing import Any, Dict, List, Optional
 
@@ -102,33 +101,54 @@ class CacheTrackerActor(xo.Actor):
     def get_model_version_count(self, model_name: str) -> int:
         return len(self.get_model_versions(model_name))
 
-    def list_cached_models(self) -> List[Dict[Any, Any]]:
+    def list_cached_models(
+        self, worker_ip: str, model_name: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
         cached_models = []
-        for model_name, model_versions in self._model_name_to_version_info.items():
-            for version_info in model_versions:
-                cache_status = version_info.get("cache_status", None)
-                if cache_status == True:
-                    ret = version_info.copy()
-                    ret["model_name"] = model_name
+        for name, versions in self._model_name_to_version_info.items():
+            # only return assigned cached model if model_name is not none
+            # else return all cached model
+            if model_name and model_name != name:
+                continue
+            for version_info in versions:
+                cache_status = version_info.get("cache_status", False)
+                # search cached model
+                if cache_status:
+                    res = version_info.copy()
+                    res["model_name"] = name
+                    paths = res.get("model_file_location", {})
+                    # only return assigned worker's device path
+                    if worker_ip in paths.keys():
+                        res["model_file_location"] = paths[worker_ip]
+                        cached_models.append(res)
+        return cached_models
 
-                    re_dict = version_info.get("model_file_location", None)
-                    if re_dict is not None and isinstance(re_dict, dict):
-                        if re_dict:
-                            actor_ip_address, path = next(iter(re_dict.items()))
-                        else:
-                            raise ValueError("The dictionary is empty.")
-                    else:
-                        raise ValueError("re_dict must be a non-empty dictionary.")
+    def list_deletable_models(self, model_version: str, worker_ip: str) -> str:
+        model_file_location = ""
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # search assign model version
+                if model_version == version_info.get("model_version", None):
+                    # check if exist
+                    if version_info.get("cache_status", False):
+                        paths = version_info.get("model_file_location", {})
+                        # only return assigned worker's device path
+                        if worker_ip in paths.keys():
+                            model_file_location = paths[worker_ip]
+        return model_file_location
 
-                    ret["actor_ip_address"] = actor_ip_address
-                    ret["path"] = path
-                    if os.path.isdir(path):
-                        files = os.listdir(path)
-                        resolved_file = os.path.realpath(os.path.join(path, files[0]))
-                        if resolved_file:
-                            ret["real_path"] = os.path.dirname(resolved_file)
-                    else:
-                        ret["real_path"] = os.path.realpath(path)
-                    cached_models.append(ret)
-        cached_models = sorted(cached_models, key=lambda x: x["model_name"])
-        return cached_models
+    def confirm_and_remove_model(self, model_version: str, worker_ip: str):
+        # find remove path
+        rm_path = self.list_deletable_models(model_version, worker_ip)
+        # search _model_name_to_version_info if exist this path, and delete
+        for model, model_versions in self._model_name_to_version_info.items():
+            for version_info in model_versions:
+                # check if exist
+                if version_info.get("cache_status", False):
+                    paths = version_info.get("model_file_location", {})
+                    # only delete assigned worker's device path
+                    if worker_ip in paths.keys() and rm_path == paths[worker_ip]:
+                        del paths[worker_ip]
+                        # if path is empty, update cache status
+                        if not paths:
+                            version_info["cache_status"] = False