xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/model/llm/transformers/minicpmv26.py

@@ -12,26 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
-from typing import Dict, Iterator, List, Optional, Union
+from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 import torch
 from PIL import Image
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....core.scheduler import InferenceRequest
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import (
+    _decode_image,
+    generate_chat_completion,
+    generate_completion_chunk,
+    parse_messages,
+)
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -43,6 +40,7 @@ class MiniCPMV26Model(PytorchChatModel):
         self._device = None
         self._tokenizer = None
         self._model = None
+        self._processor = None
 
     @classmethod
     def match(
@@ -59,7 +57,7 @@ class MiniCPMV26Model(PytorchChatModel):
         return AutoModel
 
     def load(self, **kwargs):
-        from transformers import AutoModel, AutoTokenizer
+        from transformers import AutoModel, AutoProcessor, AutoTokenizer
         from transformers.generation import GenerationConfig
 
         device = self._pytorch_model_config.get("device", "auto")
@@ -100,6 +98,10 @@ class MiniCPMV26Model(PytorchChatModel):
             self.model_path,
             trust_remote_code=True,
         )
+        self._processor = AutoProcessor.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._device = self._model.device
         self._save_tensorizer()
 
     def _message_content_to_chat(self, content):
@@ -120,7 +122,9 @@ class MiniCPMV26Model(PytorchChatModel):
                 frame_idx = uniform_sample(frame_idx, MAX_NUM_FRAMES)
             frames = vr.get_batch(frame_idx).asnumpy()
             frames = [Image.fromarray(v.astype("uint8")) for v in frames]
-            print("num frames:", len(frames))
+            logger.info(
+                f"Num frames: {len(frames)} when decoding video for {self.model_uid}"
+            )
             return frames
 
         def _load_video(_url):
@@ -158,19 +162,13 @@ class MiniCPMV26Model(PytorchChatModel):
             return text, images, frames
         return content, [], []
 
-    def chat(
-        self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        stream = generate_config.get("stream", False) if generate_config else False
-        videoExisted = False
+    def _convert_to_specific_style(self, messages: List[Dict]) -> Tuple:
+        video_existed = False
+        prompt, _, chat_history = parse_messages(messages)
 
         content, images_chat, video_frames = self._message_content_to_chat(prompt)
         if len(video_frames) > 0:
-            videoExisted = True
+            video_existed = True
             images_chat = video_frames
 
         msgs = []
@@ -184,7 +182,7 @@ class MiniCPMV26Model(PytorchChatModel):
             if images_tmp != []:
                 images_history = images_tmp
             if len(video_frames_h) > 0:
-                videoExisted = True
+                video_existed = True
                 images_history = video_frames_h
             if len(query_to_response) == 0 and role == "user":
                 query_to_response.append(
@@ -198,10 +196,19 @@ class MiniCPMV26Model(PytorchChatModel):
                 msgs.extend(query_to_response)
                 query_to_response = []
         msgs.append({"role": "user", "content": images_chat + [content]})
+        return msgs, video_existed
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        stream = generate_config.get("stream", False) if generate_config else False
+        msgs, video_existed = self._convert_to_specific_style(messages)
 
         # Set decode params for video
         params = {}
-        if videoExisted:
+        if video_existed:
             params = {"use_image_id": False, "max_slice_nums": 1}
 
         chat = self._model.chat(
@@ -216,57 +223,140 @@ class MiniCPMV26Model(PytorchChatModel):
             it = self.chat_stream(chat)
             return self._to_chat_completion_chunks(it)
         else:
-            c = Completion(
-                id=str(uuid.uuid1()),
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[
-                    CompletionChoice(
-                        index=0, text=chat, finish_reason="stop", logprobs=None
-                    )
-                ],
-                usage=CompletionUsage(
-                    prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-                ),
-            )
-            return self._to_chat_completion(c)
+            return generate_chat_completion(self.model_uid, chat)
 
     def chat_stream(self, chat) -> Iterator[CompletionChunk]:
         completion_id = str(uuid.uuid1())
         for new_text in chat:
-            completion_choice = CompletionChoice(
-                text=new_text, index=0, logprobs=None, finish_reason=None
-            )
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=-1,
                 completion_tokens=-1,
                 total_tokens=-1,
             )
-            chunk["usage"] = completion_usage
-            yield chunk
-
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=-1,
             completion_tokens=-1,
             total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )
+
+    def prepare_sanitize_generate_config(self, req: InferenceRequest):
+        """
+        Refer to https://huggingface.co/openbmb/MiniCPM-V-2_6/blob/main/modeling_minicpmv.py
+        """
+        raw_config = req.inference_kwargs.get("raw_params", {})
+        temperature = raw_config.get("temperature", None)
+        if temperature is None:
+            raw_config["temperature"] = 0.7
+        top_p = raw_config.get("top_p", None)
+        if top_p is None:
+            raw_config["top_p"] = 0.8
+        top_k = raw_config.get("top_k", None)
+        if top_k is None:
+            raw_config["top_k"] = 100
+        repetition_penalty = raw_config.get("repetition_penalty", None)
+        if repetition_penalty is None:
+            raw_config["repetition_penalty"] = 1.05
+        return raw_config
+
+    def _handle_input_ids_and_images(self, msgs: List[Dict]) -> Dict:
+        """
+        Copied from https://huggingface.co/openbmb/MiniCPM-V-2_6/blob/main/modeling_minicpmv.py#L315
+        """
+        from copy import deepcopy
+
+        copy_msgs = deepcopy(msgs)
+
+        images = []
+        for i, msg in enumerate(copy_msgs):
+            role = msg["role"]
+            content = msg["content"]
+            assert role in ["user", "assistant"]
+            if i == 0:
+                assert role == "user", "The role of first msg should be user"
+            if isinstance(content, str):
+                content = [content]
+            cur_msgs = []
+            for c in content:
+                if isinstance(c, Image.Image):
+                    images.append(c)
+                    cur_msgs.append("(<image>./</image>)")
+                elif isinstance(c, str):
+                    cur_msgs.append(c)
+            msg["content"] = "\n".join(cur_msgs)
+
+        return {
+            "prompt": self._processor.tokenizer.apply_chat_template(
+                copy_msgs, tokenize=False, add_generation_prompt=True
+            ),
+            "input_image": images,
+        }
+
+    def _get_full_prompt(self, messages: List[Dict], tools):
+        msgs, video_existed = self._convert_to_specific_style(messages)
+        if video_existed:
+            raise RuntimeError(
+                f"Continuous batching does not support video inputs for this model: {self.model_uid}"
+            )
+        return self._handle_input_ids_and_images(msgs)
+
+    def build_prefill_kwargs(self, prompts: List, req_list: List[InferenceRequest]):
+        prompts_lists = [x["prompt"] for x in prompts]
+        input_images_lists = [x["input_image"] for x in prompts]
+        inputs = self._processor(
+            prompts_lists,
+            input_images_lists,
+            max_slice_nums=None,
+            use_image_id=None,
+            return_tensors="pt",
+            max_length=8192,
+        ).to(self._model.device)
+        inputs.pop("image_sizes")
+
+        masked_input_ids = inputs["input_ids"] * inputs["attention_mask"]
+        for i in range(masked_input_ids.shape[0]):
+            non_zero_values = masked_input_ids[i][masked_input_ids[i] != 0].tolist()
+            req_list[i].prompt_tokens = non_zero_values
+            req_list[i].extra_kwargs["attention_mask_seq_len"] = len(non_zero_values)
+            req_list[i].padding_len = masked_input_ids.shape[1] - len(non_zero_values)
+
+        model_inputs = {
+            "input_ids": inputs["input_ids"],
+            "image_bound": inputs["image_bound"],
+            "pixel_values": inputs["pixel_values"],
+            "tgt_sizes": inputs["tgt_sizes"],
+        }
+        model_inputs["inputs_embeds"], _ = self._model.get_vllm_embedding(model_inputs)
+
+        return {
+            "inputs_embeds": model_inputs["inputs_embeds"],
+            "attention_mask": inputs["attention_mask"],
+        }
+
+    def build_decode_position_ids(
+        self, batch_size: int, seq_length: int, reqs: List[InferenceRequest]
+    ):
+        return None
+
+    def batch_inference(self, req_list: List[InferenceRequest]):
+        """
+        This method is rewritten
+        because the specific inference process is performed by `self._model.llm`,
+        not `self._model` itself
+        """
+        from .utils import batch_inference_one_step
+
+        self.prepare_batch_inference(req_list)
+        batch_inference_one_step(
+            self, req_list, self.model_uid, self._model.llm, self._tokenizer
         )
-        chunk["usage"] = completion_usage
-        yield chunk
+        self.handle_batch_inference_results(req_list)

xinference/model/llm/transformers/omnilmm.py

@@ -16,20 +16,13 @@ import json
 import logging
 import operator
 import tempfile
-import time
-import uuid
 from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 from ....thirdparty.omnilmm.chat import OmniLMMChat, img2base64
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChoice,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk
 from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, parse_messages
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -96,15 +89,14 @@ class OmniLMMModel(PytorchChatModel):
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         if generate_config and generate_config.get("stream"):
             raise Exception(
                 f"Chat with model {self.model_family.model_name} does not support stream."
             )
+        prompt, _, chat_history = parse_messages(messages)
         image_first, prompt = self._message_content_to_OmniLMM(prompt)
 
         msgs = []
@@ -135,19 +127,4 @@ class OmniLMMModel(PytorchChatModel):
         input = {"image": im_64, "question": json.dumps(msgs, ensure_ascii=True)}
         answer = self._model.chat(input=input)
 
-        return ChatCompletion(
-            id="chat" + str(uuid.uuid1()),
-            object="chat.completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                ChatCompletionChoice(
-                    index=0,
-                    message={"role": "assistant", "content": answer},
-                    finish_reason="stop",
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-            ),
-        )
+        return generate_chat_completion(self.model_uid, answer)

xinference/model/llm/transformers/qwen2_audio.py

@@ -0,0 +1,168 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import uuid
+from io import BytesIO
+from typing import Dict, Iterator, List, Optional, Union
+from urllib.request import urlopen
+
+import numpy as np
+
+from ....model.utils import select_device
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
+from .core import PytorchChatModel, PytorchGenerateConfig
+
+logger = logging.getLogger(__name__)
+
+
+class Qwen2AudioChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._processor = None
+        self._model = None
+        self._device = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        llm_family = model_family.model_family or model_family.model_name
+        if "qwen2-audio".lower() in llm_family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+
+        device = self._pytorch_model_config.get("device", "auto")
+        device = select_device(device)
+        self._device = device
+        # for multiple GPU, set back to auto to make multiple devices work
+        device = "auto" if device == "cuda" else device
+
+        self._processor = AutoProcessor.from_pretrained(
+            self.model_path,
+            device_map=device,
+            # trust_remote_code=True,
+            code_revision=self.model_spec.model_revision,
+        )
+        self._model = Qwen2AudioForConditionalGeneration.from_pretrained(
+            self.model_path,
+            device_map=device,
+            # trust_remote_code=True,
+            revision=self.model_spec.model_revision,
+        )
+
+    def _transform_messages(
+        self,
+        messages: List[Dict],
+    ):
+        import librosa
+
+        text = self._processor.apply_chat_template(
+            messages, add_generation_prompt=True, tokenize=False
+        )
+        audios: List[np.ndarray] = []
+        for msg in messages:
+            content = msg["content"]
+            if isinstance(content, List):
+                for item in content:  # type: ignore
+                    if item.get("type") == "audio" and "audio_url" in item:
+                        audio = librosa.load(
+                            BytesIO(urlopen(item["audio_url"]).read()),
+                            sr=self._processor.feature_extractor.sampling_rate,
+                        )[0]
+                        audios.append(audio)
+
+        return text, audios
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        text, audios = self._transform_messages(messages)
+        inputs = self._processor(
+            text=text, audios=audios, return_tensors="pt", padding=True
+        )
+        inputs.input_ids = inputs.input_ids.to(self._device)
+        generate_config = generate_config if generate_config else {}
+        stream = generate_config.get("stream", False) if generate_config else False
+
+        if stream:
+            it = self._generate_stream(inputs, generate_config)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(inputs, generate_config)
+            return c
+
+    def _generate(self, inputs, config: PytorchGenerateConfig = {}) -> ChatCompletion:
+        generate_ids = self._model.generate(
+            **inputs,
+            max_length=config.get("max_tokens", 512),
+        )
+        generate_ids = generate_ids[:, inputs.input_ids.size(1) :]
+        response = self._processor.batch_decode(
+            generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )[0]
+        return generate_chat_completion(self.model_uid, response)
+
+    def _generate_stream(
+        self, inputs, config: PytorchGenerateConfig = {}
+    ) -> Iterator[CompletionChunk]:
+        from threading import Thread
+
+        from transformers import TextIteratorStreamer
+
+        tokenizer = self._processor.tokenizer
+        streamer = TextIteratorStreamer(
+            tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True
+        )
+
+        gen_kwargs = {
+            "max_new_tokens": config.get("max_tokens", 512),
+            "streamer": streamer,
+            **inputs,
+        }
+
+        thread = Thread(target=self._model.generate, kwargs=gen_kwargs)
+        thread.start()
+
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=-1,
+                completion_tokens=-1,
+                total_tokens=-1,
+                has_choice=True,
+                has_content=True,
+            )
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )