xinference 0.10.3__py3-none-any.whl → 0.11.1__py3-none-any.whl

xinference/model/llm/pytorch/internlm2.py

@@ -108,6 +108,12 @@ class Internlm2PytorchChatModel(PytorchChatModel):
             kwargs["max_length"] = int(max_new_tokens)
 
         stream = generate_config.get("stream", False)
+        stream_options = generate_config.pop("stream_options", None)
+        include_usage = (
+            stream_options["include_usage"]
+            if isinstance(stream_options, dict)
+            else False
+        )
         if chat_history:
             input_history = [
                 (chat_history[i]["content"], (chat_history[i + 1]["content"]))
@@ -122,9 +128,15 @@ class Internlm2PytorchChatModel(PytorchChatModel):
             def _stream_generator():
                 last_chunk_text_length = 0
                 chunk_id = "chat-" + str(uuid.uuid1())
+                prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
+                inputs = self._tokenizer([prompt], return_tensors="pt")
+                inputs = inputs.to(self._model.device)
+                prompt_tokens = len(inputs["input_ids"][0])
                 for chunk_text, _ in self._model.stream_chat(
-                    self._tokenizer, prompt, input_history, **kwargs
+                    self._tokenizer, prompt, chat_history, **kwargs
                 ):
+                    completion_tokens = completion_tokens + 1
+                    total_tokens = prompt_tokens + completion_tokens
                     chunk_text = chunk_text[last_chunk_text_length:]
                     last_chunk_text_length += len(chunk_text)
                     completion_choice = CompletionChoice(
@@ -136,7 +148,26 @@ class Internlm2PytorchChatModel(PytorchChatModel):
                         created=int(time.time()),
                         model=self.model_uid,
                         choices=[completion_choice],
+                        usage=CompletionUsage(
+                            prompt_tokens=prompt_tokens,
+                            completion_tokens=completion_tokens,
+                            total_tokens=total_tokens,
+                        ),
+                    )
+                if include_usage:
+                    chunk = CompletionChunk(
+                        id=chunk_id,
+                        object="text_completion",
+                        created=int(time.time()),
+                        model=self.model_uid,
+                        choices=[],
+                    )
+                    chunk["usage"] = CompletionUsage(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
                     )
+                    yield chunk
 
             return self._to_chat_completion_chunks(_stream_generator())
         else:

xinference/model/llm/pytorch/qwen_vl.py

@@ -22,9 +22,11 @@ from typing import Dict, Iterator, List, Optional, Union
 from ....model.utils import select_device
 from ....types import (
     ChatCompletion,
-    ChatCompletionChoice,
     ChatCompletionChunk,
     ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionChunk,
     CompletionUsage,
 )
 from ..llm_family import LLMFamilyV1, LLMSpecV1
@@ -116,10 +118,6 @@ class QwenVLChatModel(PytorchChatModel):
         chat_history: Optional[List[ChatCompletionMessage]] = None,
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        if generate_config and generate_config.get("stream"):
-            raise Exception(
-                f"Chat with model {self.model_family.model_name} does not support stream."
-            )
         prompt = self._message_content_to_qwen(prompt)
         # Convert openai history to qwen vl history
         qwen_history = []
@@ -134,22 +132,106 @@ class QwenVLChatModel(PytorchChatModel):
             if len(query_to_response) == 2:
                 qwen_history.append(query_to_response)
                 query_to_response = []
+
+        stream = generate_config.get("stream", False) if generate_config else False
+        stream_options = (
+            generate_config.pop("stream_options", None) if generate_config else None
+        )
+        include_usage = (
+            stream_options["include_usage"]
+            if isinstance(stream_options, dict)
+            else False
+        )
+        if stream:
+            it = self._generate_stream(prompt, qwen_history, include_usage)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(prompt, qwen_history)
+            return self._to_chat_completion(c)
+
+    def _generate(self, prompt: str, qwen_history: List) -> Completion:
         response, history = self._model.chat(
             self._tokenizer, query=prompt, history=qwen_history
         )
-        return ChatCompletion(
-            id="chat" + str(uuid.uuid1()),
-            object="chat.completion",
+        c = Completion(
+            id=str(uuid.uuid1()),
+            object="text_completion",
             created=int(time.time()),
             model=self.model_uid,
             choices=[
-                ChatCompletionChoice(
-                    index=0,
-                    message={"role": "assistant", "content": response},
-                    finish_reason="stop",
+                CompletionChoice(
+                    index=0, text=response, finish_reason="stop", logprobs=None
                 )
             ],
             usage=CompletionUsage(
                 prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
             ),
         )
+        return c
+
+    def _generate_stream(
+        self, prompt: str, qwen_history: List, include_usage
+    ) -> Iterator[CompletionChunk]:
+        # response, history = model.chat(tokenizer, message, history=history)
+        response_generator = self._model.chat_stream(
+            self._tokenizer, query=prompt, history=qwen_history
+        )
+        completion_id = str(uuid.uuid1())
+        prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
+        input_ids = self._tokenizer(prompt, allowed_special="all").input_ids
+        prompt_tokens = len(input_ids)
+        full_response = ""
+        for response in response_generator:
+            inc_content = response[len(full_response) :]
+            full_response = response
+            completion_choice = CompletionChoice(
+                text=inc_content, index=0, logprobs=None, finish_reason=None
+            )
+            completion_chunk = CompletionChunk(
+                id=completion_id,
+                object="text_completion",
+                created=int(time.time()),
+                model=self.model_uid,
+                choices=[completion_choice],
+            )
+            completion_tokens = completion_tokens + 1
+            total_tokens = prompt_tokens + completion_tokens
+            completion_usage = CompletionUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+            )
+            completion_chunk["usage"] = completion_usage
+            yield completion_chunk
+
+        completion_choice = CompletionChoice(
+            text="", index=0, logprobs=None, finish_reason="stop"
+        )
+        completion_chunk = CompletionChunk(
+            id=completion_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=self.model_uid,
+            choices=[completion_choice],
+        )
+        completion_usage = CompletionUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+        )
+        completion_chunk["usage"] = completion_usage
+        yield completion_chunk
+        if include_usage:
+            chunk = CompletionChunk(
+                id=completion_id,
+                object="text_completion",
+                created=int(time.time()),
+                model=self.model_uid,
+                choices=[],
+            )
+            chunk["usage"] = CompletionUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+            )
+            yield chunk

xinference/model/llm/pytorch/utils.py

@@ -106,6 +106,10 @@ def generate_stream(
     context_len = get_context_length(model.config)
     stream_interval = generate_config.get("stream_interval", 2)
     stream = generate_config.get("stream", False)
+    stream_options = generate_config.pop("stream_options", None)
+    include_usage = (
+        stream_options["include_usage"] if isinstance(stream_options, dict) else False
+    )
 
     len_prompt = len(prompt)
 
@@ -333,6 +337,21 @@ def generate_stream(
 
     yield completion_chunk, completion_usage
 
+    if include_usage:
+        completion_chunk = CompletionChunk(
+            id=str(uuid.uuid1()),
+            object="text_completion",
+            created=int(time.time()),
+            model=model_uid,
+            choices=[],
+        )
+        completion_usage = CompletionUsage(
+            prompt_tokens=input_echo_len,
+            completion_tokens=i,
+            total_tokens=(input_echo_len + i),
+        )
+        yield completion_chunk, completion_usage
+
     # clean
     del past_key_values, out
     gc.collect()
@@ -352,7 +371,10 @@ def generate_stream_falcon(
     context_len = get_context_length(model.config)
     stream_interval = generate_config.get("stream_interval", 2)
     stream = generate_config.get("stream", False)
-
+    stream_options = generate_config.pop("stream_options", None)
+    include_usage = (
+        stream_options["include_usage"] if isinstance(stream_options, dict) else False
+    )
     len_prompt = len(prompt)
 
     temperature = float(generate_config.get("temperature", 1.0))
@@ -488,6 +510,21 @@ def generate_stream_falcon(
 
     yield completion_chunk, completion_usage
 
+    if include_usage:
+        completion_chunk = CompletionChunk(
+            id=str(uuid.uuid1()),
+            object="text_completion",
+            created=int(time.time()),
+            model=model_uid,
+            choices=[],
+        )
+        completion_usage = CompletionUsage(
+            prompt_tokens=input_echo_len,
+            completion_tokens=i,
+            total_tokens=(input_echo_len + i),
+        )
+        yield completion_chunk, completion_usage
+
     # clean
     gc.collect()
     empty_cache()

xinference/model/llm/pytorch/yi_vl.py

@@ -27,9 +27,11 @@ from PIL import Image
 from ....model.utils import select_device
 from ....types import (
     ChatCompletion,
-    ChatCompletionChoice,
     ChatCompletionChunk,
     ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionChunk,
     CompletionUsage,
 )
 from ..llm_family import LLMFamilyV1, LLMSpecV1
@@ -122,38 +124,6 @@ class YiVLChatModel(PytorchChatModel):
                 raise RuntimeError("Only one image per message is supported by Yi VL.")
         return content
 
-    @staticmethod
-    def _parse_text(text):
-        lines = text.split("\n")
-        lines = [line for line in lines if line != ""]
-        count = 0
-        for i, line in enumerate(lines):
-            if "```" in line:
-                count += 1
-                items = line.split("`")
-                if count % 2 == 1:
-                    lines[i] = f'<pre><code class="language-{items[-1]}">'
-                else:
-                    lines[i] = f"<br></code></pre>"
-            else:
-                if i > 0:
-                    if count % 2 == 1:
-                        line = line.replace("`", r"\`")
-                        line = line.replace("<", "&lt;")
-                        line = line.replace(">", "&gt;")
-                        line = line.replace(" ", "&nbsp;")
-                        line = line.replace("*", "&ast;")
-                        line = line.replace("_", "&lowbar;")
-                        line = line.replace("-", "&#45;")
-                        line = line.replace(".", "&#46;")
-                        line = line.replace("!", "&#33;")
-                        line = line.replace("(", "&#40;")
-                        line = line.replace(")", "&#41;")
-                        line = line.replace("$", "&#36;")
-                    lines[i] = "<br>" + line
-        text = "".join(lines)
-        return text
-
     def chat(
         self,
         prompt: Union[str, List[Dict]],
@@ -164,12 +134,18 @@ class YiVLChatModel(PytorchChatModel):
         from transformers import TextIteratorStreamer
 
         # TODO(codingl2k1): implement stream mode.
-        if generate_config and generate_config.get("stream"):
-            raise Exception(
-                f"Chat with model {self.model_family.model_name} does not support stream."
-            )
+
         if not generate_config:
             generate_config = {}
+
+        stream = generate_config.get("stream", False)
+        stream_options = generate_config.pop("stream_options", None)
+        include_usage = (
+            stream_options["include_usage"]
+            if isinstance(stream_options, dict)
+            else False
+        )
+
         from ....thirdparty.llava.conversation import conv_templates
         from ....thirdparty.llava.mm_utils import (
             KeywordsStoppingCriteria,
@@ -196,11 +172,11 @@ class YiVLChatModel(PytorchChatModel):
         )
 
         images = state.get_images(return_pil=True)
-        image = images[0]
-
-        image_tensor = self._image_processor.preprocess(image, return_tensors="pt")[
-            "pixel_values"
-        ][0]
+        if images:
+            image = images[0]
+            image_tensor = self._image_processor.preprocess(image, return_tensors="pt")[
+                "pixel_values"
+            ][0]
 
         stop_str = state.sep
         keywords = [stop_str]
@@ -217,7 +193,9 @@ class YiVLChatModel(PytorchChatModel):
             "input_ids": input_ids,
             "images": image_tensor.unsqueeze(0)
             .to(dtype=torch.bfloat16)
-            .to(self._model.device),
+            .to(self._model.device)
+            if images
+            else None,
             "streamer": streamer,
             "do_sample": True,
             "top_p": float(top_p),
@@ -229,25 +207,92 @@ class YiVLChatModel(PytorchChatModel):
         t = Thread(target=self._model.generate, kwargs=generate_kwargs)
         t.start()
 
+        if stream:
+            it = self._generate_stream(streamer, stop_str, input_ids, include_usage)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(streamer, stop_str)
+            return self._to_chat_completion(c)
+
+    def _generate(self, streamer, stop_str) -> Completion:
         generated_text = ""
         for new_text in streamer:
             generated_text += new_text
             if generated_text.endswith(stop_str):
                 generated_text = generated_text[: -len(stop_str)]
-        r = self._parse_text(generated_text)
-        return ChatCompletion(
-            id="chat" + str(uuid.uuid1()),
-            object="chat.completion",
+
+        c = Completion(
+            id=str(uuid.uuid1()),
+            object="text_completion",
             created=int(time.time()),
             model=self.model_uid,
             choices=[
-                ChatCompletionChoice(
-                    index=0,
-                    message={"role": "assistant", "content": r},
-                    finish_reason="stop",
+                CompletionChoice(
+                    index=0, text=generated_text, finish_reason="stop", logprobs=None
                 )
             ],
             usage=CompletionUsage(
                 prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
             ),
         )
+        return c
+
+    def _generate_stream(
+        self, streamer, stop_str, input_ids, include_usage
+    ) -> Iterator[CompletionChunk]:
+        completion_id = str(uuid.uuid1())
+        prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
+        prompt_tokens = len(input_ids[0])
+        for i, new_text in enumerate(streamer):
+            if not new_text.endswith(stop_str):
+                completion_choice = CompletionChoice(
+                    text=new_text, index=0, logprobs=None, finish_reason=None
+                )
+                chunk = CompletionChunk(
+                    id=completion_id,
+                    object="text_completion",
+                    created=int(time.time()),
+                    model=self.model_uid,
+                    choices=[completion_choice],
+                )
+                completion_tokens = i
+                total_tokens = prompt_tokens + completion_tokens
+                completion_usage = CompletionUsage(
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    total_tokens=total_tokens,
+                )
+                chunk["usage"] = completion_usage
+                yield chunk
+
+        completion_choice = CompletionChoice(
+            text="", index=0, logprobs=None, finish_reason="stop"
+        )
+        chunk = CompletionChunk(
+            id=completion_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=self.model_uid,
+            choices=[completion_choice],
+        )
+        completion_usage = CompletionUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+        )
+        chunk["usage"] = completion_usage
+        yield chunk
+        if include_usage:
+            chunk = CompletionChunk(
+                id=completion_id,
+                object="text_completion",
+                created=int(time.time()),
+                model=self.model_uid,
+                choices=[],
+            )
+            chunk["usage"] = CompletionUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+            )
+            yield chunk

xinference/model/llm/sglang/core.py

@@ -53,6 +53,7 @@ class SGLANGGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     ignore_eos: bool
     stream: bool
+    stream_options: Optional[Union[dict, None]]
 
 
 try:
@@ -157,6 +158,8 @@ class SGLANGModel(LLM):
         )
         generate_config.setdefault("stop", [])
         generate_config.setdefault("stream", False)
+        stream_options = generate_config.get("stream_options")
+        generate_config.setdefault("stream_options", stream_options)
         generate_config.setdefault("ignore_eos", False)
 
         return generate_config
@@ -192,7 +195,7 @@ class SGLANGModel(LLM):
 
     @staticmethod
     def _convert_state_to_completion_chunk(
-        request_id: str, model: str, output_text: str, meta_info: Dict
+        request_id: str, model: str, output_text: str
     ) -> CompletionChunk:
         choices: List[CompletionChoice] = [
             CompletionChoice(
@@ -209,13 +212,6 @@ class SGLANGModel(LLM):
             model=model,
             choices=choices,
         )
-        prompt_tokens = meta_info["prompt_tokens"]
-        completion_tokens = meta_info["completion_tokens"]
-        chunk["usage"] = CompletionUsage(
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=prompt_tokens + completion_tokens,
-        )
         return chunk
 
     @staticmethod
@@ -272,6 +268,9 @@ class SGLANGModel(LLM):
             "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
         )
         stream = sanitized_generate_config.pop("stream")
+        stream_options = sanitized_generate_config.pop("stream_options")
+        if isinstance(stream_options, dict):
+            include_usage = stream_options.pop("include_usage", False)
         request_id = str(uuid.uuid1())
         state = pipeline.run(
             question=prompt,
@@ -289,11 +288,34 @@ class SGLANGModel(LLM):
         else:
 
             async def stream_results() -> AsyncGenerator[CompletionChunk, None]:
+                prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
                 async for out, meta_info in state.text_async_iter(
                     var_name="answer", return_meta_data=True
                 ):
                     chunk = self._convert_state_to_completion_chunk(
-                        request_id, self.model_uid, output_text=out, meta_info=meta_info
+                        request_id, self.model_uid, output_text=out
+                    )
+                    prompt_tokens = meta_info["prompt_tokens"]
+                    completion_tokens = meta_info["completion_tokens"]
+                    total_tokens = prompt_tokens + completion_tokens
+                    chunk["usage"] = CompletionUsage(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
+                    )
+                    yield chunk
+                if include_usage:
+                    chunk = CompletionChunk(
+                        id=request_id,
+                        object="text_completion",
+                        created=int(time.time()),
+                        model=self.model_uid,
+                        choices=[],
+                    )
+                    chunk["usage"] = CompletionUsage(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
                     )
                     yield chunk
 

xinference/model/llm/utils.py

@@ -228,16 +228,14 @@ Begin!"""
                 tools_name_text = []
                 for func_info in tools:
                     parameters = []
-                    required_parameters = func_info["function"]["parameters"].get(
-                        "required", []
-                    )
-                    for name, p in func_info["function"]["parameters"][
-                        "properties"
-                    ].items():
-                        param = dict({"name": name}, **p)
-                        if name in required_parameters:
-                            param["required"] = True
-                        parameters.append(param)
+                    fp = func_info["function"].get("parameters", {})
+                    if fp:
+                        required_parameters = fp.get("required", [])
+                        for name, p in fp["properties"].items():
+                            param = dict({"name": name}, **p)
+                            if name in required_parameters:
+                                param["required"] = True
+                            parameters.append(param)
 
                     name = func_info["function"]["name"]
                     desc = func_info["function"]["description"]
@@ -447,6 +445,17 @@ Begin!"""
                 else:
                     ret += "<AI>" + content.strip()
             return ret
+        elif prompt_style.style_name == "PHI3":
+            ret = f"<|system|>{prompt_style.intra_message_sep}{prompt_style.system_prompt}{prompt_style.inter_message_sep}"
+            for message in chat_history:
+                content = message["content"] or ""
+                role = get_role(message["role"])
+                if content:
+                    ret += f"<|{role}|>{prompt_style.intra_message_sep}{content}{prompt_style.inter_message_sep}"
+                else:
+                    ret += f"<|{role}|>{prompt_style.intra_message_sep}"
+            ret += "<|assistant|>\n"
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 
@@ -473,9 +482,6 @@ Begin!"""
                 for i, choice in enumerate(chunk["choices"])
             ],
         }
-        usage = chunk.get("usage")
-        if usage is not None:
-            chat_chunk["usage"] = usage
         return cast(ChatCompletionChunk, chat_chunk)
 
     @classmethod
@@ -499,6 +505,19 @@ Begin!"""
                 for i, choice in enumerate(chunk["choices"])
             ],
         }
+        return cast(ChatCompletionChunk, chat_chunk)
+
+    @classmethod
+    def _get_final_chat_completion_chunk(
+        cls, chunk: CompletionChunk
+    ) -> ChatCompletionChunk:
+        chat_chunk = {
+            "id": "chat" + chunk["id"],
+            "model": chunk["model"],
+            "created": chunk["created"],
+            "object": "chat.completion.chunk",
+            "choices": [],
+        }
         usage = chunk.get("usage")
         if usage is not None:
             chat_chunk["usage"] = usage
@@ -512,7 +531,12 @@ Begin!"""
         for i, chunk in enumerate(chunks):
             if i == 0:
                 yield cls._get_first_chat_completion_chunk(chunk)
-            yield cls._to_chat_completion_chunk(chunk)
+            # usage
+            choices = chunk.get("choices")
+            if not choices:
+                yield cls._get_final_chat_completion_chunk(chunk)
+            else:
+                yield cls._to_chat_completion_chunk(chunk)
 
     @classmethod
     async def _async_to_chat_completion_chunks(
@@ -523,7 +547,12 @@ Begin!"""
         async for chunk in chunks:
             if i == 0:
                 yield cls._get_first_chat_completion_chunk(chunk)
-            yield cls._to_chat_completion_chunk(chunk)
+            # usage
+            choices = chunk.get("choices")
+            if not choices:
+                yield cls._get_final_chat_completion_chunk(chunk)
+            else:
+                yield cls._to_chat_completion_chunk(chunk)
             i += 1
 
     @staticmethod
@@ -680,6 +709,15 @@ Begin!"""
         else:
             m = {"role": "assistant", "content": content, "tool_calls": []}
             finish_reason = "stop"
+        try:
+            usage = c.get("usage")
+            assert "prompt_tokens" in usage
+        except Exception:
+            usage = {
+                "prompt_tokens": -1,
+                "completion_tokens": -1,
+                "total_tokens": -1,
+            }
         return {
             "id": "chat" + f"cmpl-{_id}",
             "model": model_uid,
@@ -692,11 +730,7 @@ Begin!"""
                     "finish_reason": finish_reason,
                 }
             ],
-            "usage": {
-                "prompt_tokens": -1,
-                "completion_tokens": -1,
-                "total_tokens": -1,
-            },
+            "usage": usage,
         }