xinference 0.15.4__py3-none-any.whl → 0.16.0__py3-none-any.whl

xinference/model/llm/llm_family.json

@@ -206,7 +206,7 @@
           "none"
         ],
         "model_id": "THUDM/glm-4-9b-chat",
-        "model_revision": "f6e0743b285dd808084530f070ad08e504386750"
+        "model_revision": "eb55a443d66541f30869f6caac5ad0d2e95bcbaa"
       },
       {
         "model_format": "ggufv2",
@@ -7923,6 +7923,174 @@
             "00021-of-00021"
           ]
         }
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-bf16"
       }
     ],
     "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",

xinference/model/llm/llm_family_modelscope.json

@@ -5681,6 +5681,114 @@
             "00021-of-00021"
           ]
         }
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "2-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "2-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
       }
     ],
     "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",

xinference/model/llm/transformers/chatglm.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
+import logging
 import typing
 import uuid
 from threading import Thread
@@ -29,6 +30,8 @@ from ..utils import (
 )
 from .core import PytorchChatModel, PytorchModelConfig
 
+logger = logging.getLogger(__name__)
+
 
 class ChatglmPytorchChatModel(PytorchChatModel):
     def __init__(
@@ -445,3 +448,104 @@ class ChatglmPytorchChatModel(PytorchChatModel):
             raw_config["top_p"] = 0.8
 
         return raw_config
+
+    def prepare_batch_inference(self, req_list: List[InferenceRequest]):
+        super(PytorchChatModel, self).prepare_batch_inference(req_list)
+        for r in req_list:
+            try:
+                if not r.stopped and r.is_prefill:
+                    tools = r.generate_config.get("tools", None)
+                    tools = list(tools) if tools is not None else None
+                    tool_choice = r.generate_config.get("tool_choice", "none")
+
+                    r.prompt = self._process_messages(
+                        r.prompt, tools=tools, tool_choice=tool_choice
+                    )
+                    r.full_prompt = self.get_full_context(
+                        r.prompt,
+                        self.model_family.chat_template,  # type: ignore
+                        tokenizer=self._tokenizer,
+                    )
+                    if tools:
+                        r.tools = tools
+            except Exception as e:
+                logger.exception(f"prepare inference error with {e}")
+                r.stopped = True
+                r.error_msg = str(e)
+
+    def handle_chat_result_non_streaming(self, req: InferenceRequest):
+        if req.tools:
+            response = req.completion[0]["choices"][0]["text"]
+            usage = req.completion[0]["usage"]
+            function_call = self._process_response_non_streaming(
+                response, req.tools, use_tool=True
+            )
+            req.completion[0] = self._tool_calls_completion(
+                self.model_family, self.model_uid, function_call
+            )
+            req.completion[0]["usage"] = usage
+        else:
+            req.completion[0] = self._to_chat_completion(req.completion[0])
+
+    def handle_chat_result_streaming(self, req: InferenceRequest):
+        results = []
+        tools = {tool["function"]["name"] for tool in req.tools} if req.tools else {}
+        response = "".join(req.outputs)
+        eos_pos = response.find("<eos_stream>")
+        if eos_pos != -1:
+            response = response[:eos_pos]
+
+        if "<bos_stream>" in req.completion:
+            bos_pos = req.completion.index("<bos_stream>")
+            results.append(
+                self._get_first_chat_completion_chunk(req.completion[bos_pos + 1])
+            )
+
+        if req.stopped:
+            if tools:
+                new_response = self._process_response_streaming(
+                    response, tools, end=True
+                )
+                if new_response:
+                    if isinstance(new_response, dict):  # tool call case
+                        chunk_id = [
+                            c for c in req.completion if not isinstance(c, str)
+                        ][0]["id"]
+                        results.append(
+                            self._tool_calls_completion_chunk(
+                                self.model_family,
+                                self.model_uid,
+                                new_response,
+                                chunk_id=chunk_id,
+                            )
+                        )
+                    else:  # normal case
+                        for c in req.completion:
+                            if c == "<bos_stream>":
+                                continue
+                            elif c == "<eos_stream>":
+                                break
+                            else:
+                                results.append(self._to_chat_completion_chunk(c))
+            else:
+                for c in req.completion:
+                    if c == "<bos_stream>":
+                        continue
+                    elif c == "<eos_stream>":
+                        break
+                    else:
+                        results.append(self._to_chat_completion_chunk(c))
+        else:
+            if response and response[-1] != "�":
+                new_response = self._process_response_streaming(
+                    response, tools, end=False
+                )
+                if new_response is not None:  # normal case
+                    for c in req.completion:
+                        if c == "<bos_stream>":
+                            continue
+                        results.append(self._to_chat_completion_chunk(c))
+
+        if req.stopped and req.include_usage:
+            results.append(self._get_final_chat_completion_chunk(req.completion[-1]))
+        req.completion = results

xinference/model/llm/transformers/core.py

@@ -29,7 +29,6 @@ from ....device_utils import (
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
-    Completion,
     CompletionChoice,
     CompletionChunk,
     CreateCompletionTorch,
@@ -46,9 +45,7 @@ from .utils import get_context_length, get_max_src_len, pad_prefill_tokens
 logger = logging.getLogger(__name__)
 
 NON_DEFAULT_MODEL_LIST: List[str] = [
-    "chatglm3",
-    "chatglm3-32k",
-    "chatglm3-128k",
+    "opt",
     "glm4-chat",
     "glm4-chat-1m",
     "internlm2-chat",
@@ -345,69 +342,6 @@ class PytorchModel(LLM):
             return False
         return True
 
-    def generate(
-        self, prompt: str, generate_config: Optional[PytorchGenerateConfig] = None
-    ) -> Union[Completion, Iterator[CompletionChunk]]:
-        from .utils import generate_stream
-
-        def generator_wrapper(
-            prompt: str, generate_config: PytorchGenerateConfig
-        ) -> Iterator[CompletionChunk]:
-            for completion_chunk, completion_usage in generate_stream(
-                self.model_uid,
-                self._model,
-                self._tokenizer,
-                prompt,
-                self._device,
-                generate_config,
-            ):
-                completion_chunk["usage"] = completion_usage
-                yield completion_chunk
-
-        logger.debug(
-            "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
-        )
-
-        generate_config = self._sanitize_generate_config(generate_config)
-
-        assert self._model is not None
-        assert self._tokenizer is not None
-
-        lora_model = generate_config.pop("lora_name")
-
-        if lora_model is not None and self._peft_model is not None:
-            for lora in self._peft_model:
-                if lora_model == lora.lora_name:
-                    self._model.set_adapter(lora_model)
-                    logger.info(f"Set lora model to {lora_model}")
-                    break
-            else:
-                self._model.disable_adapter()
-                logger.info(f"No lora model {lora_model} found, skip setting")
-
-        stream = generate_config.get("stream", False)
-        if not stream:
-            for completion_chunk, completion_usage in generate_stream(
-                self.model_uid,
-                self._model,
-                self._tokenizer,
-                prompt,
-                self._device,
-                generate_config,
-            ):
-                pass
-            completion = Completion(
-                id=completion_chunk["id"],
-                object=completion_chunk["object"],
-                created=completion_chunk["created"],
-                model=completion_chunk["model"],
-                choices=completion_chunk["choices"],
-                usage=completion_usage,
-            )
-            return completion
-        else:
-            return generator_wrapper(prompt, generate_config)
-
     def build_prefill_attention_mask(
         self, batch_size: int, seq_length: int, reqs: List[InferenceRequest]
     ):
@@ -730,7 +664,12 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
         messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        tools = generate_config.pop("tools", []) if generate_config else None
+        raise NotImplementedError
+
+    def load(self):
+        super().load()
+
+    def _get_full_prompt(self, messages: List[Dict], tools):
         model_family = self.model_family.model_family or self.model_family.model_name
         full_context_kwargs = {}
         if (
@@ -746,29 +685,6 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
             tokenizer=self._tokenizer,
             **full_context_kwargs,
         )
-
-        generate_config = self._sanitize_generate_config(generate_config)
-
-        stream = generate_config.get("stream", False)
-        if stream:
-            it = self.generate(full_prompt, generate_config)
-            assert isinstance(it, Iterator)
-            return self._to_chat_completion_chunks(it)
-        else:
-            c = self.generate(full_prompt, generate_config)
-            assert not isinstance(c, Iterator)
-            if tools:
-                return self._tool_calls_completion(self.model_family, self.model_uid, c)
-            return self._to_chat_completion(c)
-
-    def load(self):
-        super().load()
-
-    def _get_full_prompt(self, messages: List[Dict], tools):
-        assert self.model_family.chat_template is not None
-        full_prompt = self.get_full_context(
-            messages, self.model_family.chat_template, tokenizer=self._tokenizer
-        )
         return full_prompt
 
     def prepare_batch_inference(self, req_list: List[InferenceRequest]):
@@ -776,12 +692,39 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
         for r in req_list:
             try:
                 if not r.stopped and r.is_prefill:
-                    r.full_prompt = self._get_full_prompt(r.prompt, None)
+                    tools = r.generate_config.get("tools", None)
+                    r.full_prompt = self._get_full_prompt(r.prompt, tools)
+                    if tools:
+                        r.tools = tools
             except Exception as e:
                 logger.exception(f"prepare inference error with {e}")
                 r.stopped = True
                 r.error_msg = str(e)
 
+    def handle_chat_result_non_streaming(self, req: InferenceRequest):
+        if req.tools:
+            req.completion[0] = self._tool_calls_completion(
+                self.model_family, self.model_uid, req.completion[0]
+            )
+        else:
+            req.completion[0] = self._to_chat_completion(req.completion[0])
+
+    def handle_chat_result_streaming(self, req: InferenceRequest):
+        results = []
+        for i, c in enumerate(req.completion):
+            if c == "<bos_stream>":
+                results.append(
+                    self._get_first_chat_completion_chunk(req.completion[i + 1])
+                )
+            elif c == "<eos_stream>":
+                break
+            else:
+                results.append(self._to_chat_completion_chunk(c))
+
+        if req.stopped and req.include_usage:
+            results.append(self._get_final_chat_completion_chunk(req.completion[-1]))
+        req.completion = results
+
     def handle_batch_inference_results(self, req_list: List[InferenceRequest]):
         for req in req_list:
             if req.error_msg is None and req.completion:
@@ -800,23 +743,6 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
                     continue
 
                 if req.stream:
-                    results = []
-                    for i, c in enumerate(req.completion):
-                        if c == "<bos_stream>":
-                            results.append(
-                                self._get_first_chat_completion_chunk(
-                                    req.completion[i + 1]
-                                )
-                            )
-                        elif c == "<eos_stream>":
-                            break
-                        else:
-                            results.append(self._to_chat_completion_chunk(c))
-
-                    if req.stopped and req.include_usage:
-                        results.append(
-                            self._get_final_chat_completion_chunk(req.completion[-1])
-                        )
-                    req.completion = results
+                    self.handle_chat_result_streaming(req)
                 else:
-                    req.completion[0] = self._to_chat_completion(req.completion[0])
+                    self.handle_chat_result_non_streaming(req)