xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/model/llm/transformers/qwen2_vl.py

@@ -0,0 +1,234 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import importlib.util
+import logging
+import sys
+import uuid
+from typing import Iterator, List, Optional, Union
+
+from ....model.utils import select_device
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    CompletionChunk,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
+from .core import PytorchChatModel, PytorchGenerateConfig
+
+logger = logging.getLogger(__name__)
+
+
+class Qwen2VLChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tokenizer = None
+        self._model = None
+        self._device = None
+        self._processor = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        llm_family = model_family.model_family or model_family.model_name
+        if "qwen2-vl-instruct".lower() in llm_family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
+
+        device = self._pytorch_model_config.get("device", "auto")
+        device = select_device(device)
+        self._device = device
+        # for multiple GPU, set back to auto to make multiple devices work
+        device = "auto" if device == "cuda" else device
+
+        self._processor = AutoProcessor.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._tokenizer = self._processor.tokenizer
+        flash_attn_installed = importlib.util.find_spec("flash_attn") is not None
+        if flash_attn_installed:
+            self._model = Qwen2VLForConditionalGeneration.from_pretrained(
+                self.model_path,
+                torch_dtype="bfloat16",
+                device_map=device,
+                attn_implementation="flash_attention_2",
+                trust_remote_code=True,
+            ).eval()
+        else:
+            self._model = Qwen2VLForConditionalGeneration.from_pretrained(
+                self.model_path, device_map=device, trust_remote_code=True
+            ).eval()
+
+    def _transform_messages(
+        self,
+        messages: List[ChatCompletionMessage],
+    ):
+        transformed_messages = []
+        for msg in messages:
+            new_content = []
+            role = msg["role"]
+            content = msg["content"]
+            if isinstance(content, str):
+                new_content.append({"type": "text", "text": content})
+            elif isinstance(content, List):
+                for item in content:  # type: ignore
+                    if "text" in item:
+                        new_content.append({"type": "text", "text": item["text"]})
+                    elif "image_url" in item:
+                        new_content.append(
+                            {"type": "image", "image": item["image_url"]["url"]}
+                        )
+                    elif "video_url" in item:
+                        new_content.append(
+                            {"type": "video", "video": item["video_url"]["url"]}
+                        )
+            new_message = {"role": role, "content": new_content}
+            transformed_messages.append(new_message)
+
+        return transformed_messages
+
+    def chat(
+        self,
+        messages: List[ChatCompletionMessage],  # type: ignore
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)
+
+        generate_config = generate_config if generate_config else {}
+
+        stream = generate_config.get("stream", False) if generate_config else False
+
+        if stream:
+            it = self._generate_stream(messages, generate_config)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(messages, generate_config)
+            return c
+
+    def _generate(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> ChatCompletion:
+        from qwen_vl_utils import process_vision_info
+
+        # Preparation for inference
+        text = self._processor.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+        image_inputs, video_inputs = process_vision_info(messages)
+        inputs = self._processor(
+            text=[text],
+            images=image_inputs,
+            videos=video_inputs,
+            padding=True,
+            return_tensors="pt",
+        )
+        inputs = inputs.to("cuda")
+
+        # Inference: Generation of the output
+        generated_ids = self._model.generate(
+            **inputs,
+            max_new_tokens=config.get("max_tokens", 512),
+            temperature=config.get("temperature", 1),
+        )
+        generated_ids_trimmed = [
+            out_ids[len(in_ids) :]
+            for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+        ]
+        output_text = self._processor.batch_decode(
+            generated_ids_trimmed,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
+        )[0]
+        return generate_chat_completion(self.model_uid, output_text)
+
+    def _generate_stream(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> Iterator[CompletionChunk]:
+        from threading import Thread
+
+        from qwen_vl_utils import process_vision_info
+        from transformers import TextIteratorStreamer
+
+        text = self._processor.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+        image_inputs, video_inputs = process_vision_info(messages)
+        inputs = self._processor(
+            text=[text],
+            images=image_inputs,
+            videos=video_inputs,
+            padding=True,
+            return_tensors="pt",
+        )
+        inputs = inputs.to(self._model.device)
+
+        tokenizer = self._tokenizer
+        streamer = TextIteratorStreamer(
+            tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True
+        )
+
+        gen_kwargs = {
+            "max_new_tokens": config.get("max_tokens", 512),
+            "temperature": config.get("temperature", 1),
+            "streamer": streamer,
+            **inputs,
+        }
+        error = None
+
+        def model_generate():
+            try:
+                return self._model.generate(**gen_kwargs)
+            except Exception:
+                nonlocal error
+                error = sys.exc_info()
+                streamer.end()
+                raise
+
+        thread = Thread(target=model_generate)
+        thread.start()
+
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=-1,
+                completion_tokens=-1,
+                total_tokens=-1,
+                has_choice=True,
+                has_content=True,
+            )
+
+        if error:
+            _, err, tb = error  # type: ignore
+            raise err.with_traceback(tb)
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )

xinference/model/llm/transformers/qwen_vl.py

@@ -15,7 +15,6 @@ import base64
 import logging
 import operator
 import tempfile
-import time
 import typing
 import uuid
 from typing import Dict, Iterator, List, Optional, Tuple, Union
@@ -25,16 +24,9 @@ from transformers import PreTrainedTokenizer
 
 from ....core.scheduler import InferenceRequest
 from ....model.utils import select_device
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
 from .utils import pad_prefill_tokens
 
@@ -53,7 +45,7 @@ class QwenVLChatModel(PytorchChatModel):
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         llm_family = model_family.model_family or model_family.model_name
-        if "qwen" in llm_family and "vision" in model_family.model_ability:
+        if "qwen-" in llm_family and "vision" in model_family.model_ability:
             return True
         return False
 
@@ -129,18 +121,12 @@ class QwenVLChatModel(PytorchChatModel):
             return self._tokenizer.from_list_format(content)
         return content
 
-    def _get_prompt_and_chat_history(
-        self,
-        prompt: Union[str, List[Dict]],
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
-    ):
-        prompt = self._message_content_to_qwen(prompt)
-        # Convert openai history to qwen vl history
+    def _get_prompt_and_chat_history(self, messages: List[Dict]):
         qwen_history = []
         query_to_response: List = []
-        for h in chat_history or []:
-            role = h["role"]
-            content = self._message_content_to_qwen(h["content"])
+        for message in messages[:-1]:
+            role = message["role"]
+            content = self._message_content_to_qwen(message["content"])
             if len(query_to_response) == 0 and role == "user":
                 query_to_response.append(content)
             if len(query_to_response) == 1 and role == "assistant":
@@ -148,18 +134,15 @@ class QwenVLChatModel(PytorchChatModel):
             if len(query_to_response) == 2:
                 qwen_history.append(query_to_response)
                 query_to_response = []
+        prompt = self._message_content_to_qwen(messages[-1]["content"])
         return prompt, qwen_history
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        prompt, qwen_history = self._get_prompt_and_chat_history(
-            prompt, chat_history=chat_history
-        )
+        prompt, qwen_history = self._get_prompt_and_chat_history(messages)
 
         stream = generate_config.get("stream", False) if generate_config else False
         stream_options = (
@@ -174,33 +157,17 @@ class QwenVLChatModel(PytorchChatModel):
             it = self._generate_stream(prompt, qwen_history, include_usage)  # type: ignore
             return self._to_chat_completion_chunks(it)
         else:
-            c = self._generate(prompt, qwen_history)  # type: ignore
-            return self._to_chat_completion(c)
+            return self._generate(prompt, qwen_history)  # type: ignore
 
-    def _generate(self, prompt: str, qwen_history: List) -> Completion:
+    def _generate(self, prompt: str, qwen_history: List) -> ChatCompletion:
         response, history = self._model.chat(
             self._tokenizer, query=prompt, history=qwen_history
         )
-        c = Completion(
-            id=str(uuid.uuid1()),
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                CompletionChoice(
-                    index=0, text=response, finish_reason="stop", logprobs=None
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-            ),
-        )
-        return c
+        return generate_chat_completion(self.model_uid, response)
 
     def _generate_stream(
         self, prompt: str, qwen_history: List, include_usage
     ) -> Iterator[CompletionChunk]:
-        # response, history = model.chat(tokenizer, message, history=history)
         response_generator = self._model.chat_stream(
             self._tokenizer, query=prompt, history=qwen_history
         )
@@ -212,57 +179,40 @@ class QwenVLChatModel(PytorchChatModel):
         for response in response_generator:
             inc_content = response[len(full_response) :]
             full_response = response
-            completion_choice = CompletionChoice(
-                text=inc_content, index=0, logprobs=None, finish_reason=None
-            )
-            completion_chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
             completion_tokens = completion_tokens + 1
             total_tokens = prompt_tokens + completion_tokens
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=inc_content,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
             )
-            completion_chunk["usage"] = completion_usage
-            yield completion_chunk
-
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        completion_chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
             total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
         )
-        completion_chunk["usage"] = completion_usage
-        yield completion_chunk
         if include_usage:
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[],
-            )
-            chunk["usage"] = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
             )
-            yield chunk
 
     @staticmethod
     def get_batch_size_and_seq_len_indexes_from_kv() -> Tuple[int, int]:
@@ -359,10 +309,8 @@ class QwenVLChatModel(PytorchChatModel):
 
         return raw_text, context_tokens
 
-    def _get_full_prompt(self, prompt, system_prompt, chat_history, tools):
-        prompt, qwen_history = self._get_prompt_and_chat_history(
-            prompt, chat_history=chat_history
-        )
+    def _get_full_prompt(self, messages: List[Dict], tools):
+        prompt, qwen_history = self._get_prompt_and_chat_history(messages)
         _, context_tokens = self.make_context(self._tokenizer, prompt, qwen_history)
         return context_tokens
 

xinference/model/llm/transformers/utils.py

@@ -321,7 +321,7 @@ def generate_stream(
 
     if stream:
         completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason=finish_reason
+            text=output, index=0, logprobs=None, finish_reason=finish_reason
         )
     else:
         completion_choice = CompletionChoice(
@@ -430,39 +430,6 @@ def pad_prefill_tokens(
     return prompt_tokens
 
 
-def _get_completion_chunk(
-    output: str,
-    chunk_id: str,
-    finish_reason: Optional[str],
-    model_uid: str,
-    r: InferenceRequest,
-    just_usage: bool,
-):
-    completion_choice = (
-        [
-            CompletionChoice(
-                text=output, index=0, logprobs=None, finish_reason=finish_reason
-            )
-        ]
-        if not just_usage
-        else []
-    )
-    completion_chunk = CompletionChunk(
-        id=chunk_id,
-        object="text_completion",
-        created=int(time.time()),
-        model=model_uid,
-        choices=completion_choice,
-    )
-    completion_usage = CompletionUsage(
-        prompt_tokens=len(r.prompt_tokens),
-        completion_tokens=len(r.new_tokens),
-        total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
-    )
-    completion_chunk["usage"] = completion_usage
-    return completion_chunk
-
-
 def _get_completion(
     output: str,
     chunk_id: str,
@@ -551,6 +518,8 @@ def _batch_inference_one_step_internal(
     bos_flag: str = "<bos_stream>",
     eos_flag: str = "<eos_stream>",
 ):
+    from ..utils import generate_completion_chunk
+
     # need to judge stopped here,
     # since some requests state may change to stopped due to invalid parameters, e.g. max_src_len
     valid_req_list = [r for r in req_list if not r.stopped]
@@ -710,11 +679,28 @@ def _batch_inference_one_step_internal(
                     output = output[r.last_output_length :]
                     r.last_output_length += len(output)
 
-                    completion_chunk = _get_completion_chunk(
-                        output, r.chunk_id, r.finish_reason, model_uid, r, False
+                    completion_chunk = generate_completion_chunk(
+                        chunk_text=output,
+                        finish_reason=None,
+                        chunk_id=r.chunk_id,
+                        model_uid=model_uid,
+                        prompt_tokens=len(r.prompt_tokens),
+                        completion_tokens=len(r.new_tokens),
+                        total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
                     )
                     r.completion.append(completion_chunk)
                     if r.stopped:
+                        # OpenAI compatible chunk
+                        completion_chunk = generate_completion_chunk(
+                            chunk_text="",
+                            finish_reason=r.finish_reason,
+                            chunk_id=r.chunk_id,
+                            model_uid=model_uid,
+                            prompt_tokens=len(r.prompt_tokens),
+                            completion_tokens=len(r.new_tokens),
+                            total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
+                        )
+                        r.completion.append(completion_chunk)
                         r.completion.append(eos_flag)
 
                     # last round, handle stream result
@@ -723,8 +709,16 @@ def _batch_inference_one_step_internal(
                     # these tokens are real generated and should be counted.
                     if r.stopped and _i == decode_round - 1 and include_usage:
                         r.completion.append(
-                            _get_completion_chunk(
-                                "", r.chunk_id, r.finish_reason, model_uid, r, True
+                            generate_completion_chunk(
+                                chunk_text=None,
+                                finish_reason=None,
+                                chunk_id=r.chunk_id,
+                                model_uid=model_uid,
+                                prompt_tokens=len(r.prompt_tokens),
+                                completion_tokens=len(r.new_tokens),
+                                total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
+                                has_choice=False,
+                                has_content=False,
                             )
                         )
             else: