xinference 0.14.4.post1__py3-none-any.whl → 0.15.0__py3-none-any.whl

xinference/model/llm/transformers/deepseek_vl.py

@@ -15,7 +15,6 @@ import base64
 import logging
 import os.path
 import tempfile
-import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO
@@ -25,16 +24,9 @@ import requests
 import torch
 
 from ....model.utils import select_device
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -147,9 +139,7 @@ class DeepSeekVLChatModel(PytorchChatModel):
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         if not generate_config:
@@ -162,44 +152,40 @@ class DeepSeekVLChatModel(PytorchChatModel):
             if isinstance(stream_options, dict)
             else False
         )
-        prompt, images = self._message_content_to_deepseek(prompt)
-        prompt_messages: List[Dict[str, Any]] = [
-            {
-                "role": "User",
-                "content": prompt,
-            },
-            {"role": "Assistant", "content": ""},
-        ]
-        if images:
-            prompt_messages[0]["images"] = images
-
-        # Convert openai history to qwen vl history
-        deepseek_history = []
-        for h in chat_history or []:
-            role = h["role"]
+
+        prompt = ""
+        deepseek_messages = []
+        for i, message in enumerate(messages):
+            role = message["role"]
+            content = message["content"]
             if role == "user":
-                content, images = self._message_content_to_deepseek(h["content"])
-                msg: Dict[str, Any] = {
-                    "role": "User",
-                    "content": content,
-                }
-                if images:
-                    msg["images"] = images
-                deepseek_history.append(msg)
+                if isinstance(content, str):
+                    deepseek_messages.append({"role": "User", "content": content})
+                else:
+                    content, images = self._message_content_to_deepseek(content)
+                    msg: Dict[str, Any] = {
+                        "role": "User",
+                        "content": content,
+                    }
+                    if images:
+                        msg["images"] = images
+                    deepseek_messages.append(msg)
+                if i == len(messages) - 1:
+                    prompt = content
             elif role == "assistant":
-                deepseek_history.append({"role": "Assistant", "content": h["content"]})
+                deepseek_messages.append({"role": "Assistant", "content": content})
             else:
-                logger.error("Unexpected msg in chat history: %s", h)
-
-        deepseek_history.extend(prompt_messages)
+                logger.error(
+                    f"Unexpected message in messages: role: {role}, message: {message}"
+                )
 
         from ....thirdparty.deepseek_vl.serve.inference import generate
         from ....thirdparty.deepseek_vl.utils.io import load_pil_images
 
         # load images and prepare for inputs
-        pil_images = load_pil_images(deepseek_history)
+        pil_images = load_pil_images(deepseek_messages)
         prepare_inputs = self._vl_chat_processor(
-            conversations=deepseek_history, images=pil_images, force_batchify=True
+            conversations=deepseek_messages, images=pil_images, force_batchify=True
         ).to(self._model.device, self._model.dtype)
 
         temperature = generate_config.get("temperature", 0.2)
@@ -226,31 +212,16 @@ class DeepSeekVLChatModel(PytorchChatModel):
             it = self._generate_stream(streamer, stop_str, include_usage, prompt)
             return self._to_chat_completion_chunks(it)
         else:
-            c = self._generate(streamer, stop_str)
-            return self._to_chat_completion(c)
+            return self._generate(streamer, stop_str)
 
-    def _generate(self, streamer, stop_str) -> Completion:
+    def _generate(self, streamer, stop_str) -> ChatCompletion:
         generated_text = ""
         for new_text in streamer:
             if new_text.endswith(stop_str):
                 new_text = new_text[: -len(stop_str)]
             generated_text += new_text
 
-        c = Completion(
-            id=str(uuid.uuid1()),
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                CompletionChoice(
-                    index=0, text=generated_text, finish_reason="stop", logprobs=None
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-            ),
-        )
-        return c
+        return generate_chat_completion(self.model_uid, generated_text)
 
     def _generate_stream(
         self, streamer, stop_str, include_usage, prompt
@@ -262,54 +233,40 @@ class DeepSeekVLChatModel(PytorchChatModel):
         for i, new_text in enumerate(streamer):
             if new_text.endswith(stop_str):
                 new_text = new_text[: -len(stop_str)]
-            completion_choice = CompletionChoice(
-                text=new_text, index=0, logprobs=None, finish_reason=None
-            )
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
             completion_tokens = i
             total_tokens = prompt_tokens + completion_tokens
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=True,
+                has_content=True,
             )
-            chunk["usage"] = completion_usage
-            yield chunk
-
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
             total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk
+
         if include_usage:
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[],
-            )
-            chunk["usage"] = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
             )
-            yield chunk

xinference/model/llm/transformers/glm4v.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import typing
 import uuid
 from concurrent.futures import ThreadPoolExecutor
@@ -22,18 +21,10 @@ from typing import Dict, Iterator, List, Optional, Union
 import torch
 
 from ....core.scheduler import InferenceRequest
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import _decode_image, generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
 from .utils import get_max_src_len
 
@@ -102,66 +93,45 @@ class Glm4VModel(PytorchChatModel):
         self._tokenizer = tokenizer
         self._save_tensorizer()
 
-    def _message_content_to_chat(self, content):
-        if not isinstance(content, str):
-            texts = []
-            image_urls = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "text":
-                    texts.append(c["text"])
-                elif c_type == "image_url":
-                    image_urls.append(c["image_url"]["url"])
-            image_futures = []
-            with ThreadPoolExecutor() as executor:
-                for image_url in image_urls:
-                    fut = executor.submit(_decode_image, image_url)
-                    image_futures.append(fut)
-            images = [fut.result() for fut in image_futures]
-            text = " ".join(texts)
-            if len(images) == 0:
-                return text, []
-            elif len(images) == 1:
-                return text, images
+    @staticmethod
+    def _get_processed_msgs(messages: List[Dict]) -> List[Dict]:
+        res = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if isinstance(content, str):
+                res.append({"role": role, "content": content})
             else:
-                raise RuntimeError("Only one image per message is supported")
-        return content, []
-
-    def _get_chat_msgs(
-        self,
-        prompt: Union[str, List[Dict]],
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
-    ):
-        content, images_chat = self._message_content_to_chat(prompt)
-
-        msgs = []
-        query_to_response: List[Dict] = []
-        images_history = []
-        for h in chat_history or []:
-            role = h["role"]
-            content_h, images_tmp = self._message_content_to_chat(h["content"])
-            if images_tmp:
-                images_history = images_tmp
-            if len(query_to_response) == 0 and role == "user":
-                query_to_response.append({"role": "user", "content": content_h})
-            if len(query_to_response) == 1 and role == "assistant":
-                query_to_response.append({"role": "assistant", "content": content_h})
-            if len(query_to_response) == 2:
-                msgs.extend(query_to_response)
-                query_to_response = []
-        image = None
-        if len(images_chat) > 0:
-            image = images_chat[0]
-        elif len(images_history) > 0:
-            image = images_history[0]
-        msgs.append({"role": "user", "content": content, "image": image})
-        return msgs
+                texts = []
+                image_urls = []
+                for c in content:
+                    c_type = c.get("type")
+                    if c_type == "text":
+                        texts.append(c["text"])
+                    else:
+                        assert (
+                            c_type == "image_url"
+                        ), "Please follow the image input of the OpenAI API."
+                        image_urls.append(c["image_url"]["url"])
+                if len(image_urls) > 1:
+                    raise RuntimeError("Only one image per message is supported")
+                image_futures = []
+                with ThreadPoolExecutor() as executor:
+                    for image_url in image_urls:
+                        fut = executor.submit(_decode_image, image_url)
+                        image_futures.append(fut)
+                images = [fut.result() for fut in image_futures]
+                assert len(images) <= 1
+                text = " ".join(texts)
+                if images:
+                    res.append({"role": role, "content": text, "image": images[0]})
+                else:
+                    res.append({"role": role, "content": text})
+        return res
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         from transformers import TextIteratorStreamer
@@ -170,7 +140,7 @@ class Glm4VModel(PytorchChatModel):
             generate_config = {}
 
         stream = generate_config.get("stream", False)
-        msgs = self._get_chat_msgs(prompt, chat_history)
+        msgs = self._get_processed_msgs(messages)
 
         inputs = self._tokenizer.apply_chat_template(
             msgs,
@@ -213,64 +183,38 @@ class Glm4VModel(PytorchChatModel):
                 response = self._tokenizer.decode(outputs[0])
                 if response.endswith(stop_str):
                     response = response[: -len(stop_str)]
-            c = Completion(
-                id=str(uuid.uuid1()),
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[
-                    CompletionChoice(
-                        index=0, text=response, finish_reason="stop", logprobs=None
-                    )
-                ],
-                usage=CompletionUsage(
-                    prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-                ),
-            )
-            return self._to_chat_completion(c)
+            return generate_chat_completion(self.model_uid, response)
 
     def chat_stream(self, streamer, stop_str) -> Iterator[CompletionChunk]:
         completion_id = str(uuid.uuid1())
         for new_text in streamer:
             if not new_text.endswith(stop_str):
-                completion_choice = CompletionChoice(
-                    text=new_text, index=0, logprobs=None, finish_reason=None
-                )
-                chunk = CompletionChunk(
-                    id=completion_id,
-                    object="text_completion",
-                    created=int(time.time()),
-                    model=self.model_uid,
-                    choices=[completion_choice],
-                )
-                completion_usage = CompletionUsage(
+                yield generate_completion_chunk(
+                    chunk_text=new_text,
+                    finish_reason=None,
+                    chunk_id=completion_id,
+                    model_uid=self.model_uid,
                     prompt_tokens=-1,
                     completion_tokens=-1,
                     total_tokens=-1,
+                    has_choice=True,
+                    has_content=True,
                 )
-                chunk["usage"] = completion_usage
-                yield chunk
 
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=-1,
             completion_tokens=-1,
             total_tokens=-1,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk
 
-    def _get_full_prompt(self, prompt, system_prompt, chat_history, tools):
-        msgs = self._get_chat_msgs(prompt, chat_history)
+    def _get_full_prompt(self, messages, tools):
+        msgs = self._get_processed_msgs(messages)
         inputs = self._tokenizer.apply_chat_template(
             msgs,
             add_generation_prompt=True,

xinference/model/llm/transformers/intern_vl.py

@@ -12,24 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Iterator, List, Optional, Union
 
 import torch
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import (
+    _decode_image,
+    generate_chat_completion,
+    generate_completion_chunk,
+    parse_messages,
+)
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -78,7 +74,7 @@ def _message_content_to_intern(content, image_cnt):
 
 def _get_prompt_and_chat_history(
     prompt: Union[str, List[Dict]],
-    chat_history: Optional[List[ChatCompletionMessage]] = None,
+    chat_history: Optional[List[Dict]] = None,
 ):
     # Convert openai history to intern vl history
     images = []
@@ -332,9 +328,7 @@ class InternVLChatModel(PytorchChatModel):
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         from ....thirdparty.internvl.conversation import get_conv_template
@@ -366,6 +360,7 @@ class InternVLChatModel(PytorchChatModel):
             else False
         )
 
+        prompt, _, chat_history = parse_messages(messages)
         content, history, images, videos = _get_prompt_and_chat_history(
             prompt, chat_history
         )
@@ -434,10 +429,9 @@ class InternVLChatModel(PytorchChatModel):
             chunk = self._generate_stream(generate_kwargs, input_ids, include_usage)
             return self._to_chat_completion_chunks(chunk)
         else:
-            chunk = self._generate(generate_kwargs, input_ids, template)
-            return self._to_chat_completion(chunk)
+            return self._generate(generate_kwargs, input_ids, template)
 
-    def _generate(self, generate_kwargs, input_ids, template):
+    def _generate(self, generate_kwargs, input_ids, template) -> ChatCompletion:
         prompt_tokens = len(input_ids[0])
         generation_output = self._model.generate(**generate_kwargs)
         completion_tokens = len(generation_output[0])
@@ -445,23 +439,13 @@ class InternVLChatModel(PytorchChatModel):
             generation_output, skip_special_tokens=True
         )[0]
         response = response.split(template.sep)[0].strip()
-        chunk = Completion(
-            id=str(uuid.uuid1()),
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                CompletionChoice(
-                    index=0, text=response, finish_reason="stop", logprobs=None
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=prompt_tokens + completion_tokens,
-            ),
+        return generate_chat_completion(
+            self.model_uid,
+            response,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
         )
-        return chunk
 
     def _generate_stream(self, generate_kwargs, input_ids, include_usage):
         from threading import Thread
@@ -483,58 +467,43 @@ class InternVLChatModel(PytorchChatModel):
 
         completion_id = str(uuid.uuid1())
         prompt_tokens = len(input_ids[0])
-        completion_tokens = 0
+        total_tokens, completion_tokens = 0, 0
         # Loop through the streamer to get the new text as it is generated
         for i, new_text in enumerate(streamer):
             if new_text == self._model.conv_template.sep:
                 break
-            completion_choice = CompletionChoice(
-                text=new_text, index=0, logprobs=None, finish_reason=None
-            )
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
             completion_tokens = max(completion_tokens, len(streamer.token_cache))
             total_tokens = prompt_tokens + completion_tokens
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
             )
-            chunk["usage"] = completion_usage
-            yield chunk
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
             total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk
+
         if include_usage:
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[],
-            )
-            chunk["usage"] = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
             )
-            yield chunk