xinference 0.14.0.post1__py3-none-any.whl → 0.14.1.post1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-08-05T11:58:50+0800",
+ "date": "2024-08-12T12:36:32+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "111299317120411f407b015b2b7dbf8402aa35c8",
- "version": "0.14.0.post1"
+ "full-revisionid": "9afee766a3c5cc53e6035490400a4291b78e72ff",
+ "version": "0.14.1.post1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -65,6 +65,7 @@ from ..types import (
     CreateCompletion,
     ImageList,
     PeftModelConfig,
+    VideoList,
     max_tokens_field,
 )
 from .oauth2.auth_service import AuthService
@@ -123,6 +124,14 @@ class TextToImageRequest(BaseModel):
     user: Optional[str] = None
 
 
+class TextToVideoRequest(BaseModel):
+    model: str
+    prompt: Union[str, List[str]] = Field(description="The input to embed.")
+    n: Optional[int] = 1
+    kwargs: Optional[str] = None
+    user: Optional[str] = None
+
+
 class SpeechRequest(BaseModel):
     model: str
     input: str
@@ -158,6 +167,7 @@ class BuildGradioImageInterfaceRequest(BaseModel):
     model_id: str
     controlnet: Union[None, List[Dict[str, Union[str, None]]]]
     model_revision: str
+    model_ability: List[str]
 
 
 class RESTfulAPI:
@@ -511,6 +521,17 @@ class RESTfulAPI:
                 else None
             ),
         )
+        self._router.add_api_route(
+            "/v1/video/generations",
+            self.create_videos,
+            methods=["POST"],
+            response_model=VideoList,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/chat/completions",
             self.create_chat_completion,
@@ -1031,6 +1052,7 @@ class RESTfulAPI:
                 model_revision=body.model_revision,
                 controlnet=body.controlnet,
                 access_token=access_token,
+                model_ability=body.model_ability,
             ).build()
 
             gr.mount_gradio_app(self._app, interface, f"/{model_uid}")
@@ -1544,6 +1566,38 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def create_videos(self, request: Request) -> Response:
+        body = TextToVideoRequest.parse_obj(await request.json())
+        model_uid = body.model
+        try:
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            kwargs = json.loads(body.kwargs) if body.kwargs else {}
+            video_list = await model.text_to_video(
+                prompt=body.prompt,
+                n=body.n,
+                **kwargs,
+            )
+            return Response(content=video_list, media_type="application/json")
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def create_chat_completion(self, request: Request) -> Response:
         raw_body = await request.json()
         body = CreateChatCompletion.parse_obj(raw_body)

xinference/client/handlers.py

@@ -1,9 +1,6 @@
 from .restful.restful_client import (  # noqa: F401
     RESTfulAudioModelHandle as AudioModelHandle,
 )
-from .restful.restful_client import (  # noqa: F401
-    RESTfulChatglmCppChatModelHandle as ChatglmCppChatModelHandle,
-)
 from .restful.restful_client import (  # noqa: F401
     RESTfulChatModelHandle as ChatModelHandle,
 )

xinference/client/restful/restful_client.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import json
 import typing
+import warnings
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union
 
 import requests
@@ -24,13 +25,13 @@ if TYPE_CHECKING:
         ChatCompletion,
         ChatCompletionChunk,
         ChatCompletionMessage,
-        ChatglmCppGenerateConfig,
         Completion,
         CompletionChunk,
         Embedding,
         ImageList,
         LlamaCppGenerateConfig,
         PytorchGenerateConfig,
+        VideoList,
     )
 
 
@@ -370,6 +371,44 @@ class RESTfulImageModelHandle(RESTfulModelHandle):
         return response_data
 
 
+class RESTfulVideoModelHandle(RESTfulModelHandle):
+    def text_to_video(
+        self,
+        prompt: str,
+        n: int = 1,
+        **kwargs,
+    ) -> "VideoList":
+        """
+        Creates a video by the input text.
+
+        Parameters
+        ----------
+        prompt: `str` or `List[str]`
+            The prompt or prompts to guide video generation. If not defined, you need to pass `prompt_embeds`.
+        n: `int`, defaults to 1
+            The number of videos to generate per prompt. Must be between 1 and 10.
+        Returns
+        -------
+        VideoList
+            A list of video objects.
+        """
+        url = f"{self._base_url}/v1/video/generations"
+        request_body = {
+            "model": self._model_uid,
+            "prompt": prompt,
+            "n": n,
+            "kwargs": json.dumps(kwargs),
+        }
+        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to create the video, detail: {_get_error_string(response)}"
+            )
+
+        response_data = response.json()
+        return response_data
+
+
 class RESTfulGenerateModelHandle(RESTfulModelHandle):
     def generate(
         self,
@@ -470,81 +509,14 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
             Report the failure to generate the chat from the server. Detailed information provided in error message.
 
         """
-
-        url = f"{self._base_url}/v1/chat/completions"
-
-        if chat_history is None:
-            chat_history = []
-
-        chat_history = handle_system_prompts(chat_history, system_prompt)
-        chat_history.append({"role": "user", "content": prompt})  # type: ignore
-
-        request_body: Dict[str, Any] = {
-            "model": self._model_uid,
-            "messages": chat_history,
-        }
-        if tools is not None:
-            request_body["tools"] = tools
-        if generate_config is not None:
-            for key, value in generate_config.items():
-                request_body[key] = value
-
-        stream = bool(generate_config and generate_config.get("stream"))
-        response = requests.post(
-            url, json=request_body, stream=stream, headers=self.auth_headers
+        warnings.warn(
+            "The parameters `prompt`, `system_prompt` and `chat_history` will be deprecated in version v0.15.0, "
+            "and will be replaced by the parameter `messages`, "
+            "similar to the OpenAI API: https://platform.openai.com/docs/guides/chat-completions/getting-started",
+            category=DeprecationWarning,
+            stacklevel=2,
         )
 
-        if response.status_code != 200:
-            raise RuntimeError(
-                f"Failed to generate chat completion, detail: {_get_error_string(response)}"
-            )
-
-        if stream:
-            return streaming_response_iterator(response.iter_lines())
-
-        response_data = response.json()
-        return response_data
-
-
-class RESTfulChatglmCppChatModelHandle(RESTfulModelHandle):
-    def chat(
-        self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List["ChatCompletionMessage"]] = None,
-        tools: Optional[List[Dict]] = None,
-        generate_config: Optional["ChatglmCppGenerateConfig"] = None,
-    ) -> Union["ChatCompletion", Iterator["ChatCompletionChunk"]]:
-        """
-        Given a list of messages comprising a conversation, the ChatGLM model will return a response via RESTful APIs.
-
-        Parameters
-        ----------
-        prompt: str
-            The user's input.
-        system_prompt: Optional[str]
-            The system context provide to Model prior to any chats.
-        chat_history: Optional[List["ChatCompletionMessage"]]
-            A list of messages comprising the conversation so far.
-        tools: Optional[List[Dict]]
-            A tool list.
-        generate_config: Optional["ChatglmCppGenerateConfig"]
-            Additional configuration for ChatGLM chat generation.
-
-        Returns
-        -------
-        Union["ChatCompletion", Iterator["ChatCompletionChunk"]]
-            Stream is a parameter in generate_config.
-            When stream is set to True, the function will return Iterator["ChatCompletionChunk"].
-            When stream is set to False, the function will return "ChatCompletion".
-
-        Raises
-        ------
-        RuntimeError
-            Report the failure to generate the chat from the server. Detailed information provided in error message.
-
-        """
-
         url = f"{self._base_url}/v1/chat/completions"
 
         if chat_history is None:
@@ -580,60 +552,6 @@ class RESTfulChatglmCppChatModelHandle(RESTfulModelHandle):
         return response_data
 
 
-class RESTfulChatglmCppGenerateModelHandle(RESTfulChatglmCppChatModelHandle):
-    def generate(
-        self,
-        prompt: str,
-        generate_config: Optional["ChatglmCppGenerateConfig"] = None,
-    ) -> Union["Completion", Iterator["CompletionChunk"]]:
-        """
-        Given a prompt, the ChatGLM model will generate a response via RESTful APIs.
-
-        Parameters
-        ----------
-        prompt: str
-            The user's input.
-        generate_config: Optional["ChatglmCppGenerateConfig"]
-            Additional configuration for ChatGLM chat generation.
-
-        Returns
-        -------
-        Union["Completion", Iterator["CompletionChunk"]]
-            Stream is a parameter in generate_config.
-            When stream is set to True, the function will return Iterator["CompletionChunk"].
-            When stream is set to False, the function will return "Completion".
-
-        Raises
-        ------
-        RuntimeError
-            Report the failure to generate the content from the server. Detailed information provided in error message.
-
-        """
-
-        url = f"{self._base_url}/v1/completions"
-
-        request_body: Dict[str, Any] = {"model": self._model_uid, "prompt": prompt}
-        if generate_config is not None:
-            for key, value in generate_config.items():
-                request_body[key] = value
-
-        stream = bool(generate_config and generate_config.get("stream"))
-
-        response = requests.post(
-            url, json=request_body, stream=stream, headers=self.auth_headers
-        )
-        if response.status_code != 200:
-            raise RuntimeError(
-                f"Failed to generate completion, detail: {response.json()['detail']}"
-            )
-
-        if stream:
-            return streaming_response_iterator(response.iter_lines())
-
-        response_data = response.json()
-        return response_data
-
-
 class RESTfulAudioModelHandle(RESTfulModelHandle):
     def transcriptions(
         self,
@@ -1090,7 +1008,6 @@ class Client:
         -------
         ModelHandle
             The corresponding Model Handler based on the Model specified in the uid:
-              - :obj:`xinference.client.handlers.ChatglmCppChatModelHandle` -> provide handle to ChatGLM Model
               - :obj:`xinference.client.handlers.GenerateModelHandle` -> provide handle to basic generate Model. e.g. Baichuan.
               - :obj:`xinference.client.handlers.ChatModelHandle` -> provide handle to chat Model. e.g. Baichuan-chat.
 
@@ -1111,11 +1028,7 @@ class Client:
         desc = response.json()
 
         if desc["model_type"] == "LLM":
-            if desc["model_format"] == "ggmlv3" and "chatglm" in desc["model_name"]:
-                return RESTfulChatglmCppGenerateModelHandle(
-                    model_uid, self.base_url, auth_headers=self._headers
-                )
-            elif "chat" in desc["model_ability"]:
+            if "chat" in desc["model_ability"]:
                 return RESTfulChatModelHandle(
                     model_uid, self.base_url, auth_headers=self._headers
                 )
@@ -1141,6 +1054,10 @@ class Client:
             return RESTfulAudioModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers
             )
+        elif desc["model_type"] == "video":
+            return RESTfulVideoModelHandle(
+                model_uid, self.base_url, auth_headers=self._headers
+            )
         elif desc["model_type"] == "flexible":
             return RESTfulFlexibleModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers

xinference/constants.py

@@ -47,6 +47,7 @@ XINFERENCE_TENSORIZER_DIR = os.path.join(XINFERENCE_HOME, "tensorizer")
 XINFERENCE_MODEL_DIR = os.path.join(XINFERENCE_HOME, "model")
 XINFERENCE_LOG_DIR = os.path.join(XINFERENCE_HOME, "logs")
 XINFERENCE_IMAGE_DIR = os.path.join(XINFERENCE_HOME, "image")
+XINFERENCE_VIDEO_DIR = os.path.join(XINFERENCE_HOME, "video")
 XINFERENCE_AUTH_DIR = os.path.join(XINFERENCE_HOME, "auth")
 XINFERENCE_CSG_ENDPOINT = str(
     os.environ.get(XINFERENCE_ENV_CSG_ENDPOINT, "https://hub-stg.opencsg.com/")

xinference/core/chat_interface.py

@@ -24,7 +24,6 @@ from gradio.components import Markdown, Textbox
 from gradio.layouts import Accordion, Column, Row
 
 from ..client.restful.restful_client import (
-    RESTfulChatglmCppChatModelHandle,
     RESTfulChatModelHandle,
     RESTfulGenerateModelHandle,
 )
@@ -116,9 +115,7 @@ class GradioInterface:
             client = RESTfulClient(self.endpoint)
             client._set_token(self._access_token)
             model = client.get_model(self.model_uid)
-            assert isinstance(
-                model, (RESTfulChatModelHandle, RESTfulChatglmCppChatModelHandle)
-            )
+            assert isinstance(model, RESTfulChatModelHandle)
 
             response_content = ""
             for chunk in model.chat(

xinference/core/image_interface.py

@@ -36,6 +36,7 @@ class ImageInterface:
         model_name: str,
         model_id: str,
         model_revision: str,
+        model_ability: List[str],
         controlnet: Union[None, List[Dict[str, Union[str, None]]]],
         access_token: Optional[str],
     ):
@@ -45,6 +46,7 @@ class ImageInterface:
         self.model_name = model_name
         self.model_id = model_id
         self.model_revision = model_revision
+        self.model_ability = model_ability
         self.controlnet = controlnet
         self.access_token = (
             access_token.replace("Bearer ", "") if access_token is not None else None
@@ -76,6 +78,7 @@ class ImageInterface:
             n: int,
             size_width: int,
             size_height: int,
+            num_inference_steps: int,
             negative_prompt: Optional[str] = None,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
@@ -86,11 +89,15 @@ class ImageInterface:
             assert isinstance(model, RESTfulImageModelHandle)
 
             size = f"{int(size_width)}*{int(size_height)}"
+            num_inference_steps = (
+                None if num_inference_steps == -1 else num_inference_steps  # type: ignore
+            )
 
             response = model.text_to_image(
                 prompt=prompt,
                 n=n,
                 size=size,
+                num_inference_steps=num_inference_steps,
                 negative_prompt=negative_prompt,
                 response_format="b64_json",
             )
@@ -125,13 +132,23 @@ class ImageInterface:
                     n = gr.Number(label="Number of Images", value=1)
                     size_width = gr.Number(label="Width", value=1024)
                     size_height = gr.Number(label="Height", value=1024)
+                    num_inference_steps = gr.Number(
+                        label="Inference Step Number", value=-1
+                    )
 
                 with gr.Column():
                     image_output = gr.Gallery()
 
             generate_button.click(
                 text_generate_image,
-                inputs=[prompt, n, size_width, size_height, negative_prompt],
+                inputs=[
+                    prompt,
+                    n,
+                    size_width,
+                    size_height,
+                    num_inference_steps,
+                    negative_prompt,
+                ],
                 outputs=image_output,
             )
 
@@ -145,6 +162,7 @@ class ImageInterface:
             n: int,
             size_width: int,
             size_height: int,
+            num_inference_steps: int,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
 
@@ -157,6 +175,9 @@ class ImageInterface:
                 size = f"{int(size_width)}*{int(size_height)}"
             else:
                 size = None
+            num_inference_steps = (
+                None if num_inference_steps == -1 else num_inference_steps  # type: ignore
+            )
 
             bio = io.BytesIO()
             image.save(bio, format="png")
@@ -168,6 +189,7 @@ class ImageInterface:
                 image=bio.getvalue(),
                 size=size,
                 response_format="b64_json",
+                num_inference_steps=num_inference_steps,
             )
 
             images = []
@@ -200,6 +222,9 @@ class ImageInterface:
                     n = gr.Number(label="Number of image", value=1)
                     size_width = gr.Number(label="Width", value=-1)
                     size_height = gr.Number(label="Height", value=-1)
+                    num_inference_steps = gr.Number(
+                        label="Inference Step Number", value=-1
+                    )
 
                 with gr.Row():
                     with gr.Column(scale=1):
@@ -216,6 +241,7 @@ class ImageInterface:
                     n,
                     size_width,
                     size_height,
+                    num_inference_steps,
                 ],
                 outputs=output_gallery,
             )
@@ -247,9 +273,11 @@ class ImageInterface:
                     </div>
                     """
             )
-            with gr.Tab("Text to Image"):
-                self.text2image_interface()
-            with gr.Tab("Image to Image"):
-                self.image2image_interface()
+            if "text2image" in self.model_ability:
+                with gr.Tab("Text to Image"):
+                    self.text2image_interface()
+            if "image2image" in self.model_ability:
+                with gr.Tab("Image to Image"):
+                    self.image2image_interface()
 
         return app

xinference/core/model.py

@@ -133,6 +133,7 @@ class ModelActor(xo.StatelessActor):
     async def __pre_destroy__(self):
         from ..model.embedding.core import EmbeddingModel
         from ..model.llm.pytorch.core import PytorchModel as LLMPytorchModel
+        from ..model.llm.sglang.core import SGLANGModel
         from ..model.llm.vllm.core import VLLMModel as LLMVLLMModel
 
         if self.allow_batching():
@@ -145,8 +146,11 @@ class ModelActor(xo.StatelessActor):
                     f"Destroy scheduler actor failed, address: {self.address}, error: {e}"
                 )
 
+        if hasattr(self._model, "stop") and callable(self._model.stop):
+            self._model.stop()
+
         if (
-            isinstance(self._model, (LLMPytorchModel, LLMVLLMModel))
+            isinstance(self._model, (LLMPytorchModel, LLMVLLMModel, SGLANGModel))
             and self._model.model_spec.model_format == "pytorch"
         ) or isinstance(self._model, EmbeddingModel):
             try:
@@ -174,6 +178,7 @@ class ModelActor(xo.StatelessActor):
     ):
         super().__init__()
         from ..model.llm.pytorch.core import PytorchModel
+        from ..model.llm.sglang.core import SGLANGModel
         from ..model.llm.vllm.core import VLLMModel
 
         self._worker_address = worker_address
@@ -187,7 +192,7 @@ class ModelActor(xo.StatelessActor):
         self._current_generator = lambda: None
         self._lock = (
             None
-            if isinstance(self._model, (PytorchModel, VLLMModel))
+            if isinstance(self._model, (PytorchModel, VLLMModel, SGLANGModel))
             else asyncio.locks.Lock()
         )
         self._worker_ref = None
@@ -771,6 +776,27 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for flexible infer."
         )
 
+    @log_async(logger=logger)
+    @request_limit
+    async def text_to_video(
+        self,
+        prompt: str,
+        n: int = 1,
+        *args,
+        **kwargs,
+    ):
+        if hasattr(self._model, "text_to_video"):
+            return await self._call_wrapper_json(
+                self._model.text_to_video,
+                prompt,
+                n,
+                *args,
+                **kwargs,
+            )
+        raise AttributeError(
+            f"Model {self._model.model_spec} is not for creating video."
+        )
+
     async def record_metrics(self, name, op, kwargs):
         worker_ref = await self._get_worker_ref()
         await worker_ref.record_metrics(name, op, kwargs)

xinference/core/supervisor.py

@@ -64,6 +64,7 @@ if TYPE_CHECKING:
     from ..model.image import ImageModelFamilyV1
     from ..model.llm import LLMFamilyV1
     from ..model.rerank import RerankModelSpec
+    from ..model.video import VideoModelFamilyV1
     from .worker import WorkerActor
 
 
@@ -484,6 +485,31 @@ class SupervisorActor(xo.StatelessActor):
         res["model_instance_count"] = instance_cnt
         return res
 
+    async def _to_video_model_reg(
+        self, model_family: "VideoModelFamilyV1", is_builtin: bool
+    ) -> Dict[str, Any]:
+        from ..model.video import get_cache_status
+
+        instance_cnt = await self.get_instance_count(model_family.model_name)
+        version_cnt = await self.get_model_version_count(model_family.model_name)
+
+        if self.is_local_deployment():
+            # TODO: does not work when the supervisor and worker are running on separate nodes.
+            cache_status = get_cache_status(model_family)
+            res = {
+                **model_family.dict(),
+                "cache_status": cache_status,
+                "is_builtin": is_builtin,
+            }
+        else:
+            res = {
+                **model_family.dict(),
+                "is_builtin": is_builtin,
+            }
+        res["model_version_count"] = version_cnt
+        res["model_instance_count"] = instance_cnt
+        return res
+
     async def _to_flexible_model_reg(
         self, model_spec: "FlexibleModelSpec", is_builtin: bool
     ) -> Dict[str, Any]:
@@ -602,6 +628,17 @@ class SupervisorActor(xo.StatelessActor):
                         {"model_name": model_spec.model_name, "is_builtin": False}
                     )
 
+            ret.sort(key=sort_helper)
+            return ret
+        elif model_type == "video":
+            from ..model.video import BUILTIN_VIDEO_MODELS
+
+            for model_name, family in BUILTIN_VIDEO_MODELS.items():
+                if detailed:
+                    ret.append(await self._to_video_model_reg(family, is_builtin=True))
+                else:
+                    ret.append({"model_name": model_name, "is_builtin": True})
+
             ret.sort(key=sort_helper)
             return ret
         elif model_type == "rerank":