xinference 0.8.4__py3-none-any.whl → 0.9.0__py3-none-any.whl

xinference/__init__.py

@@ -18,6 +18,12 @@ from . import _version
 __version__ = _version.get_versions()["version"]
 
 
+try:
+    import intel_extension_for_pytorch  # noqa: F401
+except:
+    pass
+
+
 def _install():
     from xoscar.backends.router import Router
 

xinference/_compat.py

@@ -0,0 +1,52 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pydantic.version import VERSION as PYDANTIC_VERSION
+
+PYDANTIC_V2 = PYDANTIC_VERSION.startswith("2.")
+
+
+if PYDANTIC_V2:
+    from pydantic.v1 import (  # noqa: F401
+        BaseModel,
+        Field,
+        Protocol,
+        ValidationError,
+        create_model,
+        create_model_from_namedtuple,
+        create_model_from_typeddict,
+        parse_file_as,
+        validate_arguments,
+        validator,
+    )
+    from pydantic.v1.error_wrappers import ErrorWrapper  # noqa: F401
+    from pydantic.v1.parse import load_str_bytes  # noqa: F401
+    from pydantic.v1.types import StrBytes  # noqa: F401
+    from pydantic.v1.utils import ROOT_KEY  # noqa: F401
+else:
+    from pydantic import (  # noqa: F401
+        BaseModel,
+        Field,
+        Protocol,
+        ValidationError,
+        create_model,
+        create_model_from_namedtuple,
+        create_model_from_typeddict,
+        parse_file_as,
+        validate_arguments,
+        validator,
+    )
+    from pydantic.error_wrappers import ErrorWrapper  # noqa: F401
+    from pydantic.parse import load_str_bytes  # noqa: F401
+    from pydantic.types import StrBytes  # noqa: F401
+    from pydantic.utils import ROOT_KEY  # noqa: F401

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-02-04T17:16:50+0800",
+ "date": "2024-02-22T15:40:53+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "1b9b8c805e4e23a4c34fb05a96b819fee3ca8d50",
- "version": "0.8.4"
+ "full-revisionid": "c653c975847f9f6a81382033a9c8f5bd81bf70f2",
+ "version": "0.9.0"
 }
 '''  # END VERSION_JSON
 

xinference/api/oauth2/auth_service.py

@@ -14,13 +14,12 @@
 from datetime import timedelta
 from typing import List, Optional
 
-import pydantic
 from fastapi import Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordBearer, SecurityScopes
 from jose import JWTError, jwt
-from pydantic import BaseModel, ValidationError
 from typing_extensions import Annotated
 
+from ..._compat import BaseModel, ValidationError, parse_file_as
 from .types import AuthStartupConfig, User
 from .utils import create_access_token, get_password_hash, verify_password
 
@@ -43,7 +42,7 @@ class AuthService:
 
     def init_auth_config(self):
         if self._auth_config_file:
-            config: AuthStartupConfig = pydantic.parse_file_as(
+            config: AuthStartupConfig = parse_file_as(
                 path=self._auth_config_file, type_=AuthStartupConfig
             )
             for user in config.user_config:

xinference/api/oauth2/types.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 from typing import List
 
-from pydantic import BaseModel
+from ..._compat import BaseModel
 
 
 class LoginUserForm(BaseModel):

xinference/api/restful_api.py

@@ -44,13 +44,14 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from PIL import Image
-from pydantic import BaseModel, Field
 from sse_starlette.sse import EventSourceResponse
 from starlette.responses import JSONResponse as StarletteJSONResponse
 from starlette.responses import RedirectResponse
 from uvicorn import Config, Server
 from xoscar.utils import get_next_port
 
+from .._compat import BaseModel, Field
+from .._version import get_versions
 from ..constants import XINFERENCE_DEFAULT_ENDPOINT_PORT
 from ..core.event import Event, EventCollectorActor, EventType
 from ..core.supervisor import SupervisorActor
@@ -190,7 +191,8 @@ class RESTfulAPI:
                 "Report error event failed, model: %s, content: %s", model_uid, content
             )
 
-    async def login_for_access_token(self, form_data: LoginUserForm) -> JSONResponse:
+    async def login_for_access_token(self, request: Request) -> JSONResponse:
+        form_data = LoginUserForm.parse_obj(await request.json())
         result = self._auth_service.generate_token_for_user(
             form_data.username, form_data.password
         )
@@ -220,6 +222,9 @@ class RESTfulAPI:
         self._router.add_api_route(
             "/v1/cluster/info", self.get_cluster_device_info, methods=["GET"]
         )
+        self._router.add_api_route(
+            "/v1/cluster/version", self.get_cluster_version, methods=["GET"]
+        )
         self._router.add_api_route(
             "/v1/cluster/devices", self._get_devices_count, methods=["GET"]
         )
@@ -230,9 +235,11 @@ class RESTfulAPI:
             "/v1/ui/{model_uid}",
             self.build_gradio_interface,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/token", self.login_for_access_token, methods=["POST"]
@@ -245,142 +252,176 @@ class RESTfulAPI:
             "/v1/models/instances",
             self.get_instance_info,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_type}/{model_name}/versions",
             self.get_model_versions,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models",
             self.list_models,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         self._router.add_api_route(
             "/v1/models/{model_uid}",
             self.describe_model,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_uid}/events",
             self.get_model_events,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/instance",
             self.launch_model_by_version,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models",
             self.launch_model,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/experimental/speculative_llms",
             self.launch_speculative_llm,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:start"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:start"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/models/{model_uid}",
             self.terminate_model,
             methods=["DELETE"],
-            dependencies=[Security(self._auth_service, scopes=["models:stop"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:stop"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/completions",
             self.create_completion,
             methods=["POST"],
             response_model=Completion,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/embeddings",
             self.create_embedding,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/rerank",
             self.rerank,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/audio/transcriptions",
             self.create_transcriptions,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/audio/translations",
             self.create_translations,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/images/generations",
             self.create_images,
             methods=["POST"],
             response_model=ImageList,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/images/variations",
             self.create_variations,
             methods=["POST"],
             response_model=ImageList,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/chat/completions",
             self.create_chat_completion,
             methods=["POST"],
             response_model=ChatCompletion,
-            dependencies=[Security(self._auth_service, scopes=["models:read"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         # for custom models
@@ -388,33 +429,41 @@ class RESTfulAPI:
             "/v1/model_registrations/{model_type}",
             self.register_model,
             methods=["POST"],
-            dependencies=[Security(self._auth_service, scopes=["models:register"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:register"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}/{model_name}",
             self.unregister_model,
             methods=["DELETE"],
-            dependencies=[Security(self._auth_service, scopes=["models:unregister"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:unregister"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}",
             self.list_model_registrations,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
         self._router.add_api_route(
             "/v1/model_registrations/{model_type}/{model_name}",
             self.get_model_registrations,
             methods=["GET"],
-            dependencies=[Security(self._auth_service, scopes=["models:list"])]
-            if self.is_authenticated()
-            else None,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:list"])]
+                if self.is_authenticated()
+                else None
+            ),
         )
 
         # Clear the global Registry for the MetricsMiddleware, or
@@ -705,13 +754,15 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
     async def build_gradio_interface(
-        self, model_uid: str, body: BuildGradioInterfaceRequest, request: Request
+        self, model_uid: str, request: Request
     ) -> JSONResponse:
         """
         Separate build_interface with launch_model
         build_interface requires RESTful Client for API calls
         but calling API in async function does not return
         """
+        payload = await request.json()
+        body = BuildGradioInterfaceRequest.parse_obj(payload)
         assert self._app is not None
         assert body.model_type == "LLM"
 
@@ -781,9 +832,8 @@ class RESTfulAPI:
     async def get_address(self) -> JSONResponse:
         return JSONResponse(content=self._supervisor_address)
 
-    async def create_completion(
-        self, request: Request, body: CreateCompletionRequest
-    ) -> Response:
+    async def create_completion(self, request: Request) -> Response:
+        body = CreateCompletionRequest.parse_obj(await request.json())
         exclude = {
             "prompt",
             "model",
@@ -843,8 +893,9 @@ class RESTfulAPI:
                 self.handle_request_limit_error(e)
                 raise HTTPException(status_code=500, detail=str(e))
 
-    async def create_embedding(self, request: CreateEmbeddingRequest) -> Response:
-        model_uid = request.model
+    async def create_embedding(self, request: Request) -> Response:
+        body = CreateEmbeddingRequest.parse_obj(await request.json())
+        model_uid = body.model
 
         try:
             model = await (await self._get_supervisor_ref()).get_model(model_uid)
@@ -858,7 +909,7 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
         try:
-            embedding = await model.create_embedding(request.input)
+            embedding = await model.create_embedding(body.input)
             return Response(embedding, media_type="application/json")
         except RuntimeError as re:
             logger.error(re, exc_info=True)
@@ -870,8 +921,9 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def rerank(self, request: RerankRequest) -> Response:
-        model_uid = request.model
+    async def rerank(self, request: Request) -> Response:
+        body = RerankRequest.parse_obj(await request.json())
+        model_uid = body.model
         try:
             model = await (await self._get_supervisor_ref()).get_model(model_uid)
         except ValueError as ve:
@@ -885,11 +937,11 @@ class RESTfulAPI:
 
         try:
             scores = await model.rerank(
-                request.documents,
-                request.query,
-                top_n=request.top_n,
-                max_chunks_per_doc=request.max_chunks_per_doc,
-                return_documents=request.return_documents,
+                body.documents,
+                body.query,
+                top_n=body.top_n,
+                max_chunks_per_doc=body.max_chunks_per_doc,
+                return_documents=body.return_documents,
             )
             return Response(scores, media_type="application/json")
         except RuntimeError as re:
@@ -990,8 +1042,9 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def create_images(self, request: TextToImageRequest) -> Response:
-        model_uid = request.model
+    async def create_images(self, request: Request) -> Response:
+        body = TextToImageRequest.parse_obj(await request.json())
+        model_uid = body.model
         try:
             model = await (await self._get_supervisor_ref()).get_model(model_uid)
         except ValueError as ve:
@@ -1004,12 +1057,12 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
         try:
-            kwargs = json.loads(request.kwargs) if request.kwargs else {}
+            kwargs = json.loads(body.kwargs) if body.kwargs else {}
             image_list = await model.text_to_image(
-                prompt=request.prompt,
-                n=request.n,
-                size=request.size,
-                response_format=request.response_format,
+                prompt=body.prompt,
+                n=body.n,
+                size=body.size,
+                response_format=body.response_format,
                 **kwargs,
             )
             return Response(content=image_list, media_type="application/json")
@@ -1070,11 +1123,8 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def create_chat_completion(
-        self,
-        request: Request,
-        body: CreateChatCompletion,
-    ) -> Response:
+    async def create_chat_completion(self, request: Request) -> Response:
+        body = CreateChatCompletion.parse_obj(await request.json())
         exclude = {
             "prompt",
             "model",
@@ -1092,10 +1142,12 @@ class RESTfulAPI:
         if body.logit_bias is not None:
             raise HTTPException(status_code=501, detail="Not implemented")
 
+        messages = body.messages and list(body.messages) or None
+
         if (
-            not body.messages
-            or body.messages[-1].get("role") not in ["user", "system", "tool"]
-            or not body.messages[-1].get("content")
+            not messages
+            or messages[-1].get("role") not in ["user", "system", "tool"]
+            or not messages[-1].get("content")
         ):
             raise HTTPException(
                 status_code=400, detail="Invalid input. Please specify the prompt."
@@ -1103,7 +1155,7 @@ class RESTfulAPI:
 
         system_messages = []
         non_system_messages = []
-        for msg in body.messages:
+        for msg in messages:
             assert (
                 msg.get("content") != SPECIAL_TOOL_PROMPT
             ), f"Invalid message content {SPECIAL_TOOL_PROMPT}"
@@ -1116,13 +1168,13 @@ class RESTfulAPI:
             raise HTTPException(
                 status_code=400, detail="Multiple system messages are not supported."
             )
-        if len(system_messages) == 1 and body.messages[0]["role"] != "system":
+        if len(system_messages) == 1 and messages[0]["role"] != "system":
             raise HTTPException(
                 status_code=400, detail="System message should be the first one."
             )
         assert non_system_messages
 
-        has_tool_message = body.messages[-1].get("role") == "tool"
+        has_tool_message = messages[-1].get("role") == "tool"
         if has_tool_message:
             prompt = SPECIAL_TOOL_PROMPT
             system_prompt = system_messages[0]["content"] if system_messages else None
@@ -1157,7 +1209,12 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
         model_family = desc.get("model_family", "")
-        function_call_models = ["chatglm3", "gorilla-openfunctions-v1", "qwen-chat"]
+        function_call_models = [
+            "chatglm3",
+            "gorilla-openfunctions-v1",
+            "qwen-chat",
+            "qwen1.5-chat",
+        ]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen-chat" == model_family
 
@@ -1219,11 +1276,10 @@ class RESTfulAPI:
                 self.handle_request_limit_error(e)
                 raise HTTPException(status_code=500, detail=str(e))
 
-    async def register_model(
-        self, model_type: str, request: RegisterModelRequest
-    ) -> JSONResponse:
-        model = request.model
-        persist = request.persist
+    async def register_model(self, model_type: str, request: Request) -> JSONResponse:
+        body = RegisterModelRequest.parse_obj(await request.json())
+        model = body.model
+        persist = body.persist
 
         try:
             await (await self._get_supervisor_ref()).register_model(
@@ -1292,9 +1348,21 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def get_cluster_device_info(self) -> JSONResponse:
+    async def get_cluster_device_info(
+        self, detailed: bool = Query(False)
+    ) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).get_cluster_device_info(
+                detailed=detailed
+            )
+            return JSONResponse(content=data)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def get_cluster_version(self) -> JSONResponse:
         try:
-            data = await (await self._get_supervisor_ref()).get_cluster_device_info()
+            data = get_versions()
             return JSONResponse(content=data)
         except Exception as e:
             logger.error(e, exc_info=True)