xinference 0.14.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

xinference/model/video/diffusers.py

@@ -0,0 +1,180 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import logging
+import os
+import sys
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing import TYPE_CHECKING, List, Union
+
+import numpy as np
+import PIL.Image
+import torch
+
+from ...constants import XINFERENCE_VIDEO_DIR
+from ...device_utils import move_model_to_available_device
+from ...types import Video, VideoList
+
+if TYPE_CHECKING:
+    from .core import VideoModelFamilyV1
+
+
+logger = logging.getLogger(__name__)
+
+
+def export_to_video_imageio(
+    video_frames: Union[List[np.ndarray], List["PIL.Image.Image"]],
+    output_video_path: str,
+    fps: int = 8,
+) -> str:
+    """
+    Export the video frames to a video file using imageio lib to Avoid "green screen" issue (for example CogVideoX)
+    """
+    import imageio
+
+    if isinstance(video_frames[0], PIL.Image.Image):
+        video_frames = [np.array(frame) for frame in video_frames]
+    with imageio.get_writer(output_video_path, fps=fps) as writer:
+        for frame in video_frames:
+            writer.append_data(frame)
+    return output_video_path
+
+
+class DiffUsersVideoModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "VideoModelFamilyV1",
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._model = None
+        self._kwargs = kwargs
+
+    @property
+    def model_spec(self):
+        return self._model_spec
+
+    def load(self):
+        import torch
+
+        torch_dtype = self._kwargs.get("torch_dtype")
+        if sys.platform != "darwin" and torch_dtype is None:
+            # The following params crashes on Mac M2
+            self._kwargs["torch_dtype"] = torch.float16
+            self._kwargs["variant"] = "fp16"
+            self._kwargs["use_safetensors"] = True
+        if isinstance(torch_dtype, str):
+            self._kwargs["torch_dtype"] = getattr(torch, torch_dtype)
+
+        if self._model_spec.model_family == "CogVideoX":
+            from diffusers import CogVideoXPipeline
+
+            self._model = CogVideoXPipeline.from_pretrained(
+                self._model_path, **self._kwargs
+            )
+        else:
+            raise Exception(
+                f"Unsupported model family: {self._model_spec.model_family}"
+            )
+
+        if self._kwargs.get("cpu_offload", False):
+            logger.debug("CPU offloading model")
+            self._model.enable_model_cpu_offload()
+        elif not self._kwargs.get("device_map"):
+            logger.debug("Loading model to available device")
+            self._model = move_model_to_available_device(self._model)
+        # Recommended if your computer has < 64 GB of RAM
+        self._model.enable_attention_slicing()
+
+    def text_to_video(
+        self,
+        prompt: str,
+        n: int = 1,
+        num_inference_steps: int = 50,
+        guidance_scale: int = 6,
+        response_format: str = "b64_json",
+        **kwargs,
+    ) -> VideoList:
+        import gc
+
+        # cv2 bug will cause the video cannot be normally displayed
+        # thus we use the imageio one
+        # from diffusers.utils import export_to_video
+        from ...device_utils import empty_cache
+
+        logger.debug(
+            "diffusers text_to_video args: %s",
+            kwargs,
+        )
+        assert self._model is not None
+        if self._kwargs.get("cpu_offload"):
+            # if enabled cpu offload,
+            # the model.device would be CPU
+            device = "cuda"
+        else:
+            device = self._model.device
+        prompt_embeds, _ = self._model.encode_prompt(
+            prompt=prompt,
+            do_classifier_free_guidance=True,
+            num_videos_per_prompt=n,
+            max_sequence_length=226,
+            device=device,
+            dtype=torch.float16,
+        )
+        assert callable(self._model)
+        output = self._model(
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            prompt_embeds=prompt_embeds,
+            **kwargs,
+        )
+
+        # clean cache
+        gc.collect()
+        empty_cache()
+
+        os.makedirs(XINFERENCE_VIDEO_DIR, exist_ok=True)
+        urls = []
+        for f in output.frames:
+            path = os.path.join(XINFERENCE_VIDEO_DIR, uuid.uuid4().hex + ".mp4")
+            p = export_to_video_imageio(f, path, fps=8)
+            urls.append(p)
+        if response_format == "url":
+            return VideoList(
+                created=int(time.time()),
+                data=[Video(url=url, b64_json=None) for url in urls],
+            )
+        elif response_format == "b64_json":
+
+            def _gen_base64_video(_video_url):
+                try:
+                    with open(_video_url, "rb") as f:
+                        return base64.b64encode(f.read()).decode()
+                finally:
+                    os.remove(_video_url)
+
+            with ThreadPoolExecutor() as executor:
+                results = list(map(partial(executor.submit, _gen_base64_video), urls))  # type: ignore
+                video_list = [Video(url=None, b64_json=s.result()) for s in results]
+            return VideoList(created=int(time.time()), data=video_list)
+        else:
+            raise ValueError(f"Unsupported response format: {response_format}")

xinference/model/video/model_spec.json

@@ -0,0 +1,11 @@
+[
+  {
+    "model_name": "CogVideoX-2b",
+    "model_family": "CogVideoX",
+    "model_id": "THUDM/CogVideoX-2b",
+    "model_revision": "4bbfb1de622b80bc1b77b6e9aced75f816be0e38",
+    "model_ability": [
+      "text2video"
+    ]
+  }
+]

xinference/model/video/model_spec_modelscope.json

@@ -0,0 +1,12 @@
+[
+  {
+    "model_name": "CogVideoX-2b",
+    "model_family": "CogVideoX",
+    "model_hub": "modelscope",
+    "model_id": "ZhipuAI/CogVideoX-2b",
+    "model_revision": "master",
+    "model_ability": [
+      "text2video"
+    ]
+  }
+]

xinference/types.py

@@ -52,6 +52,16 @@ class ImageList(TypedDict):
     data: List[Image]
 
 
+class Video(TypedDict):
+    url: Optional[str]
+    b64_json: Optional[str]
+
+
+class VideoList(TypedDict):
+    created: int
+    data: List[Video]
+
+
 class EmbeddingUsage(TypedDict):
     prompt_tokens: int
     total_tokens: int
@@ -206,30 +216,6 @@ class ChatCompletionChunk(TypedDict):
     usage: NotRequired[CompletionUsage]
 
 
-class ChatglmCppModelConfig(TypedDict, total=False):
-    pass
-
-
-class ChatglmCppGenerateConfig(TypedDict, total=False):
-    max_tokens: int
-    top_p: float
-    temperature: float
-    stream: bool
-    lora_name: Optional[str]
-    stream_options: Optional[Union[dict, None]]
-
-
-class QWenCppModelConfig(TypedDict, total=False):
-    pass
-
-
-class QWenCppGenerateConfig(TypedDict, total=False):
-    max_tokens: int
-    top_p: float
-    temperature: float
-    stream: bool
-
-
 StoppingCriteria = Callable[[List[int], List[float]], bool]
 
 

xinference/web/ui/build/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.4bafd904.css",
-    "main.js": "./static/js/main.af906659.js",
+    "main.js": "./static/js/main.17ca0398.js",
     "static/media/icon.webp": "./static/media/icon.4603d52c63041e5dfbfd.webp",
     "index.html": "./index.html",
     "main.4bafd904.css.map": "./static/css/main.4bafd904.css.map",
-    "main.af906659.js.map": "./static/js/main.af906659.js.map"
+    "main.17ca0398.js.map": "./static/js/main.17ca0398.js.map"
   },
   "entrypoints": [
     "static/css/main.4bafd904.css",
-    "static/js/main.af906659.js"
+    "static/js/main.17ca0398.js"
   ]
 }

xinference/web/ui/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.af906659.js"></script><link href="./static/css/main.4bafd904.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.17ca0398.js"></script><link href="./static/css/main.4bafd904.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>