xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/model/llm/transformers/glm4v.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import typing
 import uuid
 from concurrent.futures import ThreadPoolExecutor
@@ -22,18 +21,10 @@ from typing import Dict, Iterator, List, Optional, Union
 import torch
 
 from ....core.scheduler import InferenceRequest
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import _decode_image, generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
 from .utils import get_max_src_len
 
@@ -102,66 +93,45 @@ class Glm4VModel(PytorchChatModel):
         self._tokenizer = tokenizer
         self._save_tensorizer()
 
-    def _message_content_to_chat(self, content):
-        if not isinstance(content, str):
-            texts = []
-            image_urls = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "text":
-                    texts.append(c["text"])
-                elif c_type == "image_url":
-                    image_urls.append(c["image_url"]["url"])
-            image_futures = []
-            with ThreadPoolExecutor() as executor:
-                for image_url in image_urls:
-                    fut = executor.submit(_decode_image, image_url)
-                    image_futures.append(fut)
-            images = [fut.result() for fut in image_futures]
-            text = " ".join(texts)
-            if len(images) == 0:
-                return text, []
-            elif len(images) == 1:
-                return text, images
+    @staticmethod
+    def _get_processed_msgs(messages: List[Dict]) -> List[Dict]:
+        res = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if isinstance(content, str):
+                res.append({"role": role, "content": content})
             else:
-                raise RuntimeError("Only one image per message is supported")
-        return content, []
-
-    def _get_chat_msgs(
-        self,
-        prompt: Union[str, List[Dict]],
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
-    ):
-        content, images_chat = self._message_content_to_chat(prompt)
-
-        msgs = []
-        query_to_response: List[Dict] = []
-        images_history = []
-        for h in chat_history or []:
-            role = h["role"]
-            content_h, images_tmp = self._message_content_to_chat(h["content"])
-            if images_tmp:
-                images_history = images_tmp
-            if len(query_to_response) == 0 and role == "user":
-                query_to_response.append({"role": "user", "content": content_h})
-            if len(query_to_response) == 1 and role == "assistant":
-                query_to_response.append({"role": "assistant", "content": content_h})
-            if len(query_to_response) == 2:
-                msgs.extend(query_to_response)
-                query_to_response = []
-        image = None
-        if len(images_chat) > 0:
-            image = images_chat[0]
-        elif len(images_history) > 0:
-            image = images_history[0]
-        msgs.append({"role": "user", "content": content, "image": image})
-        return msgs
+                texts = []
+                image_urls = []
+                for c in content:
+                    c_type = c.get("type")
+                    if c_type == "text":
+                        texts.append(c["text"])
+                    else:
+                        assert (
+                            c_type == "image_url"
+                        ), "Please follow the image input of the OpenAI API."
+                        image_urls.append(c["image_url"]["url"])
+                if len(image_urls) > 1:
+                    raise RuntimeError("Only one image per message is supported")
+                image_futures = []
+                with ThreadPoolExecutor() as executor:
+                    for image_url in image_urls:
+                        fut = executor.submit(_decode_image, image_url)
+                        image_futures.append(fut)
+                images = [fut.result() for fut in image_futures]
+                assert len(images) <= 1
+                text = " ".join(texts)
+                if images:
+                    res.append({"role": role, "content": text, "image": images[0]})
+                else:
+                    res.append({"role": role, "content": text})
+        return res
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         from transformers import TextIteratorStreamer
@@ -170,7 +140,7 @@ class Glm4VModel(PytorchChatModel):
             generate_config = {}
 
         stream = generate_config.get("stream", False)
-        msgs = self._get_chat_msgs(prompt, chat_history)
+        msgs = self._get_processed_msgs(messages)
 
         inputs = self._tokenizer.apply_chat_template(
             msgs,
@@ -213,64 +183,38 @@ class Glm4VModel(PytorchChatModel):
                 response = self._tokenizer.decode(outputs[0])
                 if response.endswith(stop_str):
                     response = response[: -len(stop_str)]
-            c = Completion(
-                id=str(uuid.uuid1()),
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[
-                    CompletionChoice(
-                        index=0, text=response, finish_reason="stop", logprobs=None
-                    )
-                ],
-                usage=CompletionUsage(
-                    prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-                ),
-            )
-            return self._to_chat_completion(c)
+            return generate_chat_completion(self.model_uid, response)
 
     def chat_stream(self, streamer, stop_str) -> Iterator[CompletionChunk]:
         completion_id = str(uuid.uuid1())
         for new_text in streamer:
             if not new_text.endswith(stop_str):
-                completion_choice = CompletionChoice(
-                    text=new_text, index=0, logprobs=None, finish_reason=None
-                )
-                chunk = CompletionChunk(
-                    id=completion_id,
-                    object="text_completion",
-                    created=int(time.time()),
-                    model=self.model_uid,
-                    choices=[completion_choice],
-                )
-                completion_usage = CompletionUsage(
+                yield generate_completion_chunk(
+                    chunk_text=new_text,
+                    finish_reason=None,
+                    chunk_id=completion_id,
+                    model_uid=self.model_uid,
                     prompt_tokens=-1,
                     completion_tokens=-1,
                     total_tokens=-1,
+                    has_choice=True,
+                    has_content=True,
                 )
-                chunk["usage"] = completion_usage
-                yield chunk
 
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=-1,
             completion_tokens=-1,
             total_tokens=-1,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk
 
-    def _get_full_prompt(self, prompt, system_prompt, chat_history, tools):
-        msgs = self._get_chat_msgs(prompt, chat_history)
+    def _get_full_prompt(self, messages, tools):
+        msgs = self._get_processed_msgs(messages)
         inputs = self._tokenizer.apply_chat_template(
             msgs,
             add_generation_prompt=True,

xinference/model/llm/transformers/intern_vl.py

@@ -12,24 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Iterator, List, Optional, Union
 
 import torch
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import (
+    _decode_image,
+    generate_chat_completion,
+    generate_completion_chunk,
+    parse_messages,
+)
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -78,7 +74,7 @@ def _message_content_to_intern(content, image_cnt):
 
 def _get_prompt_and_chat_history(
     prompt: Union[str, List[Dict]],
-    chat_history: Optional[List[ChatCompletionMessage]] = None,
+    chat_history: Optional[List[Dict]] = None,
 ):
     # Convert openai history to intern vl history
     images = []
@@ -332,9 +328,7 @@ class InternVLChatModel(PytorchChatModel):
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         from ....thirdparty.internvl.conversation import get_conv_template
@@ -366,6 +360,7 @@ class InternVLChatModel(PytorchChatModel):
             else False
         )
 
+        prompt, _, chat_history = parse_messages(messages)
         content, history, images, videos = _get_prompt_and_chat_history(
             prompt, chat_history
         )
@@ -434,10 +429,9 @@ class InternVLChatModel(PytorchChatModel):
             chunk = self._generate_stream(generate_kwargs, input_ids, include_usage)
             return self._to_chat_completion_chunks(chunk)
         else:
-            chunk = self._generate(generate_kwargs, input_ids, template)
-            return self._to_chat_completion(chunk)
+            return self._generate(generate_kwargs, input_ids, template)
 
-    def _generate(self, generate_kwargs, input_ids, template):
+    def _generate(self, generate_kwargs, input_ids, template) -> ChatCompletion:
         prompt_tokens = len(input_ids[0])
         generation_output = self._model.generate(**generate_kwargs)
         completion_tokens = len(generation_output[0])
@@ -445,23 +439,13 @@ class InternVLChatModel(PytorchChatModel):
             generation_output, skip_special_tokens=True
         )[0]
         response = response.split(template.sep)[0].strip()
-        chunk = Completion(
-            id=str(uuid.uuid1()),
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                CompletionChoice(
-                    index=0, text=response, finish_reason="stop", logprobs=None
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=prompt_tokens + completion_tokens,
-            ),
+        return generate_chat_completion(
+            self.model_uid,
+            response,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
         )
-        return chunk
 
     def _generate_stream(self, generate_kwargs, input_ids, include_usage):
         from threading import Thread
@@ -483,58 +467,43 @@ class InternVLChatModel(PytorchChatModel):
 
         completion_id = str(uuid.uuid1())
         prompt_tokens = len(input_ids[0])
-        completion_tokens = 0
+        total_tokens, completion_tokens = 0, 0
         # Loop through the streamer to get the new text as it is generated
         for i, new_text in enumerate(streamer):
             if new_text == self._model.conv_template.sep:
                 break
-            completion_choice = CompletionChoice(
-                text=new_text, index=0, logprobs=None, finish_reason=None
-            )
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
             completion_tokens = max(completion_tokens, len(streamer.token_cache))
             total_tokens = prompt_tokens + completion_tokens
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
             )
-            chunk["usage"] = completion_usage
-            yield chunk
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
             total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk
+
         if include_usage:
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[],
-            )
-            chunk["usage"] = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
             )
-            yield chunk

xinference/model/llm/transformers/internlm2.py

@@ -11,23 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import time
 import uuid
 from typing import Any, Dict, Iterator, List, Optional, Union
 
 from ....core.scheduler import InferenceRequest
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChoice,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-    LoRA,
-    PytorchGenerateConfig,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, LoRA, PytorchGenerateConfig
 from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk, parse_messages
 from .core import PytorchChatModel, PytorchModelConfig
 
 
@@ -106,9 +96,7 @@ class Internlm2PytorchChatModel(PytorchChatModel):
 
     def chat(
         self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         kwargs: Dict[str, Any] = {}
@@ -130,6 +118,8 @@ class Internlm2PytorchChatModel(PytorchChatModel):
             if isinstance(stream_options, dict)
             else False
         )
+
+        prompt, system_prompt, chat_history = parse_messages(messages)
         if chat_history:
             input_history = [
                 (chat_history[i]["content"], (chat_history[i + 1]["content"]))
@@ -155,54 +145,42 @@ class Internlm2PytorchChatModel(PytorchChatModel):
                     total_tokens = prompt_tokens + completion_tokens
                     chunk_text = chunk_text[last_chunk_text_length:]
                     last_chunk_text_length += len(chunk_text)
-                    completion_choice = CompletionChoice(
-                        text=chunk_text, index=0, logprobs=None, finish_reason=None
-                    )
-                    yield CompletionChunk(
-                        id=chunk_id,
-                        object="text_completion",
-                        created=int(time.time()),
-                        model=self.model_uid,
-                        choices=[completion_choice],
-                        usage=CompletionUsage(
-                            prompt_tokens=prompt_tokens,
-                            completion_tokens=completion_tokens,
-                            total_tokens=total_tokens,
-                        ),
+
+                    yield generate_completion_chunk(
+                        chunk_text,
+                        finish_reason=None,
+                        chunk_id=chunk_id,
+                        model_uid=self.model_uid,
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
                     )
+                yield generate_completion_chunk(
+                    None,
+                    finish_reason="stop",
+                    chunk_id=chunk_id,
+                    model_uid=self.model_uid,
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    total_tokens=total_tokens,
+                    has_choice=True,
+                    has_content=False,
+                )
                 if include_usage:
-                    chunk = CompletionChunk(
-                        id=chunk_id,
-                        object="text_completion",
-                        created=int(time.time()),
-                        model=self.model_uid,
-                        choices=[],
-                    )
-                    chunk["usage"] = CompletionUsage(
+                    yield generate_completion_chunk(
+                        None,
+                        finish_reason=None,
+                        chunk_id=chunk_id,
+                        model_uid=self.model_uid,
                         prompt_tokens=prompt_tokens,
                         completion_tokens=completion_tokens,
                         total_tokens=total_tokens,
+                        has_choice=False,
                     )
-                    yield chunk
 
             return self._to_chat_completion_chunks(_stream_generator())
         else:
             response, _ = self._model.chat(
                 self._tokenizer, prompt, input_history, **kwargs
             )
-            return ChatCompletion(
-                id="chat" + str(uuid.uuid1()),
-                object="chat.completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[
-                    ChatCompletionChoice(
-                        index=0,
-                        message={"role": "assistant", "content": response},
-                        finish_reason="stop",
-                    )
-                ],
-                usage=CompletionUsage(
-                    prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-                ),
-            )
+            return generate_chat_completion(self.model_uid, response)

xinference/model/llm/transformers/minicpmv25.py

@@ -13,25 +13,21 @@
 # limitations under the License.
 import json
 import logging
-import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Iterator, List, Optional, Union
 
 import torch
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionChunk,
-    CompletionUsage,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import _decode_image
+from ..utils import (
+    _decode_image,
+    generate_chat_completion,
+    generate_completion_chunk,
+    parse_messages,
+)
 from .core import PytorchChatModel, PytorchGenerateConfig
 
 logger = logging.getLogger(__name__)
@@ -125,12 +121,11 @@ class MiniCPMV25Model(PytorchChatModel):
 
     def chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
         stream = generate_config.get("stream", False) if generate_config else False
+        prompt, _, chat_history = parse_messages(messages)
         content, images_chat = self._message_content_to_chat(prompt)
 
         msgs = []
@@ -166,57 +161,29 @@ class MiniCPMV25Model(PytorchChatModel):
             it = self.chat_stream(chat)
             return self._to_chat_completion_chunks(it)
         else:
-            c = Completion(
-                id=str(uuid.uuid1()),
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[
-                    CompletionChoice(
-                        index=0, text=chat, finish_reason="stop", logprobs=None
-                    )
-                ],
-                usage=CompletionUsage(
-                    prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
-                ),
-            )
-            return self._to_chat_completion(c)
+            return generate_chat_completion(self.model_uid, chat)
 
     def chat_stream(self, chat) -> Iterator[CompletionChunk]:
         completion_id = str(uuid.uuid1())
         for new_text in chat:
-            completion_choice = CompletionChoice(
-                text=new_text, index=0, logprobs=None, finish_reason=None
-            )
-            chunk = CompletionChunk(
-                id=completion_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
-            completion_usage = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=-1,
                 completion_tokens=-1,
                 total_tokens=-1,
             )
-            chunk["usage"] = completion_usage
-            yield chunk
 
-        completion_choice = CompletionChoice(
-            text="", index=0, logprobs=None, finish_reason="stop"
-        )
-        chunk = CompletionChunk(
-            id=completion_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[completion_choice],
-        )
-        completion_usage = CompletionUsage(
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
             prompt_tokens=-1,
             completion_tokens=-1,
             total_tokens=-1,
+            has_choice=True,
+            has_content=False,
         )
-        chunk["usage"] = completion_usage
-        yield chunk