xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/model/llm/lmdeploy/core.py

@@ -12,25 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import time
 import uuid
 from typing import AsyncGenerator, Dict, Iterator, List, Optional, TypedDict, Union
 
 import torch
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionChunkChoice,
-    ChatCompletionMessage,
-    Completion,
-    CompletionChoice,
-    CompletionUsage,
-    LoRA,
-)
+from ....types import ChatCompletion, ChatCompletionChunk, Completion, LoRA
 from ..core import LLM
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import ChatModelMixin
+from ..utils import ChatModelMixin, generate_chat_completion, generate_completion_chunk
 
 logger = logging.getLogger(__name__)
 
@@ -74,8 +64,8 @@ class LMDeployGenerateConfig(TypedDict, total=False):
     repetition_penalty: Optional[float]
     ignore_eos: Optional[bool]
     random_seed: Optional[int]
-    stop_words: Optional[List[str]]
-    bad_words: Optional[List[str]]
+    stop_words: Optional[List[int]]
+    bad_words: Optional[List[int]]
     min_new_tokens: Optional[int]
     skip_special_tokens: Optional[bool]
     logprobs: Optional[int]
@@ -164,9 +154,6 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
             raise ValueError(f"Can not find correct chat template.")
 
         chat_template_config = ChatTemplateConfig(chat_temp_name)
-        chat_template_config.meta_instruction = (
-            self.model_family.prompt_style.system_prompt
-        )
         count = torch.cuda.device_count()
         if count > 1:
             self._model_config.setdefault("tp", torch.cuda.device_count())
@@ -192,9 +179,7 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
 
     async def async_chat(
         self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[Dict] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
         stream = (
@@ -213,75 +198,69 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
             else False
         )
 
-        chat_history = chat_history or []
-
         if stream:
-            chunk = self._chat_stream(prompt, chat_history, include_usage)
+            chunk = self._chat_stream(messages, include_usage)
             return self._async_to_chat_completion_chunks(chunk)
         else:
-            chunk = await self._chat(prompt, chat_history)
-            return self._to_chat_completion(chunk)
+            return await self._chat(messages)
 
-    async def _chat_stream(self, prompt, chat_history, include_usage):
+    async def _chat_stream(self, messages, include_usage):
         from lmdeploy.messages import Response
 
         prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
         completion_id = str(uuid.uuid1())
+        finish_reason = None
         async for output in self._generate(
-            prompt,
-            chat_history,
+            messages,
             session_id=-1,
             stream_response=True,
         ):
             new_text = output.text if isinstance(output, Response) else output.response
-
-            completion_choice = ChatCompletionChunkChoice(
-                text=new_text,
-                index=0,
-                logprobs=None,
-                finish_reason=output.finish_reason,
-            )
-            chunk = ChatCompletionChunk(
-                id=completion_id,
-                object="chat.completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[completion_choice],
-            )
             prompt_tokens = output.input_token_len
             completion_tokens = output.generate_token_len
             total_tokens = prompt_tokens + completion_tokens
-            completion_usage = CompletionUsage(
+            finish_reason = output.finish_reason
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
             )
-            chunk["usage"] = completion_usage
-            print(chunk)
-            yield chunk
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason=finish_reason,
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
+        )
         if include_usage:
-            chunk = ChatCompletionChunk(
-                id=completion_id,
-                object="chat.completion",
-                created=int(time.time()),
-                model=self.model_uid,
-                choices=[],
-            )
-            chunk["usage"] = CompletionUsage(
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
             )
-            yield chunk
 
-    async def _chat(self, prompt, chat_history):
+    async def _chat(self, messages) -> ChatCompletion:
         from lmdeploy.messages import Response
 
-        response, finish_reason = "", ""
+        response, finish_reason = "", None
         prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
         async for output in self._generate(
-            prompt,
-            chat_history,
+            messages,
             session_id=-1,
             stream_response=False,
         ):
@@ -291,30 +270,20 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
             total_tokens = output.input_token_len + output.generate_token_len
             finish_reason = output.finish_reason
 
-        chunk = ChatCompletion(
-            id=str(uuid.uuid1()),
-            object="chat.completion",
-            created=int(time.time()),
-            model=self.model_uid,
-            choices=[
-                CompletionChoice(
-                    index=0, text=response, finish_reason=finish_reason, logprobs=None
-                )
-            ],
-            usage=CompletionUsage(
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-            ),
+        return generate_chat_completion(
+            self.model_uid,
+            response,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            finish_reason=finish_reason,
         )
-        return chunk
 
     # copy from lmdeploy
     # Reference: lmdeploy.serve.async_engine.py
     async def _generate(
         self,
-        prompt,
-        chat_history,
+        messages: List[Dict],
         session_id: int,
         generate_config: Optional[Dict] = None,
         tools: Optional[List[object]] = None,
@@ -332,6 +301,8 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
         from lmdeploy.serve.async_engine import GenOut
         from lmdeploy.tokenizer import DetokenizeState
 
+        from ..utils import get_stop_token_ids_from_config_file
+
         session_id = -1
 
         if str(session_id) not in self._model.id2step:
@@ -343,7 +314,9 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
                 generate_config, self._model.tokenizer
             )
         if generate_config.stop_words is None:  # type: ignore
-            generate_config.stop_words = self._model.stop_words  # type: ignore
+            stop_token_ids = get_stop_token_ids_from_config_file(self.model_path)
+            if stop_token_ids is not None:
+                generate_config.stop_words = stop_token_ids  # type: ignore
         if generate_config.random_seed is None and sequence_start:  # type: ignore
             generate_config.random_seed = random.getrandbits(64)  # type: ignore
         if generate_config.n > 1:  # type: ignore
@@ -353,7 +326,7 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
             )
             generate_config.n = 1  # type: ignore
 
-        prompt_input = await self._get_prompt_input(prompt, chat_history)
+        prompt_input = await self._get_prompt_input(messages)
         prompt = prompt_input["prompt"]
         input_ids = prompt_input["input_ids"]
         finish_reason = None
@@ -482,8 +455,7 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
     # Reference: lmdeploy.serve.vl_async_engine.py
     async def _get_prompt_input(
         self,
-        prompt: Union[str, List[Dict]],
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         sequence_start: bool = True,
         tools: Optional[List[object]] = None,
         **kwargs,
@@ -493,13 +465,9 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
         IMAGE_DUMMY_TOKEN_INDEX = 0
         import numpy as np
 
-        assert self.model_family.prompt_style is not None
-        prompt_style = self.model_family.prompt_style.copy()
-        chat_history = chat_history or []
-
-        decorated, _ = self.get_prompt(prompt, chat_history, prompt_style)  # type: ignore
-        chat_history.append(ChatCompletionMessage(role="user", content=prompt))  # type: ignore
-        prompt = chat_history  # type: ignore
+        model_family = self.model_family.model_family or self.model_family.model_name
+        decorated, _ = self.get_specific_prompt(model_family, messages)  # type: ignore
+        prompt = messages  # type: ignore
 
         decorated = decorated.replace("<image>", "<img><IMAGE_TOKEN></img>")
 

xinference/model/llm/mlx/core.py

@@ -17,22 +17,20 @@ import platform
 import sys
 import time
 import uuid
-from typing import Dict, Iterable, Iterator, List, Optional, TypedDict, Union
+from typing import Dict, Iterator, List, Optional, TypedDict, Union
 
 from ....fields import max_tokens_field
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
-    ChatCompletionMessage,
     Completion,
-    CompletionChoice,
     CompletionChunk,
     CompletionUsage,
     LoRA,
 )
 from ..core import LLM
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import ChatModelMixin
+from ..utils import QWEN_TOOL_CALL_FAMILY, ChatModelMixin, generate_completion_chunk
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +52,7 @@ class MLXGenerateConfig(TypedDict, total=False):
     stop_token_ids: Optional[Union[int, List[int]]]
     stream: bool
     stream_options: Optional[Union[dict, None]]
+    tools: Optional[List[Dict]]
 
 
 class MLXModel(LLM):
@@ -211,23 +210,21 @@ class MLXModel(LLM):
             else:
                 output += out
 
-            completion_choice = CompletionChoice(
-                text=output, index=0, logprobs=None, finish_reason=None
-            )
-            completion_chunk = CompletionChunk(
-                id=chunk_id,
-                object="text_completion",
-                created=int(time.time()),
-                model=model_uid,
-                choices=[completion_choice],
-            )
             completion_usage = CompletionUsage(
                 prompt_tokens=input_echo_len,
                 completion_tokens=i,
                 total_tokens=(input_echo_len + i),
             )
 
-            yield completion_chunk, completion_usage
+            yield generate_completion_chunk(
+                chunk_text=output,
+                finish_reason=None,
+                chunk_id=chunk_id,
+                model_uid=model_uid,
+                prompt_tokens=input_echo_len,
+                completion_tokens=i,
+                total_tokens=(input_echo_len + i),
+            ), completion_usage
 
         logger.info(
             f"Average generation speed: {i / (time.time() - start):.2f} tokens/s."
@@ -238,29 +235,31 @@ class MLXModel(LLM):
         else:
             finish_reason = "stop"
 
-        if stream:
-            completion_choice = CompletionChoice(
-                text="", index=0, logprobs=None, finish_reason=finish_reason
-            )
-        else:
-            completion_choice = CompletionChoice(
-                text=output, index=0, logprobs=None, finish_reason=finish_reason
-            )
-
-        completion_chunk = CompletionChunk(
-            id=chunk_id,
-            object="text_completion",
-            created=int(time.time()),
-            model=model_uid,
-            choices=[completion_choice],
-        )
         completion_usage = CompletionUsage(
             prompt_tokens=input_echo_len,
             completion_tokens=i,
             total_tokens=(input_echo_len + i),
         )
-
-        yield completion_chunk, completion_usage
+        if stream:
+            yield generate_completion_chunk(
+                "",
+                finish_reason=finish_reason,
+                chunk_id=chunk_id,
+                model_uid=model_uid,
+                prompt_tokens=input_echo_len,
+                completion_tokens=i,
+                total_tokens=(input_echo_len + i),
+            ), completion_usage
+        else:
+            yield generate_completion_chunk(
+                output,
+                finish_reason=finish_reason,
+                chunk_id=chunk_id,
+                model_uid=model_uid,
+                prompt_tokens=input_echo_len,
+                completion_tokens=i,
+                total_tokens=(input_echo_len + i),
+            ), completion_usage
 
         if include_usage:
             completion_chunk = CompletionChunk(
@@ -270,11 +269,6 @@ class MLXModel(LLM):
                 model=model_uid,
                 choices=[],
             )
-            completion_usage = CompletionUsage(
-                prompt_tokens=input_echo_len,
-                completion_tokens=i,
-                total_tokens=(input_echo_len + i),
-            )
             yield completion_chunk, completion_usage
 
     def generate(
@@ -345,20 +339,13 @@ class MLXChatModel(MLXModel, ChatModelMixin):
         generate_config: Optional[MLXGenerateConfig],
     ) -> MLXGenerateConfig:
         generate_config = super()._sanitize_generate_config(generate_config)
-        if (
-            (not generate_config.get("stop"))
-            and self.model_family.prompt_style
-            and self.model_family.prompt_style.stop
-        ):
-            generate_config["stop"] = self.model_family.prompt_style.stop.copy()
+        if (not generate_config.get("stop")) and self.model_family.stop:
+            generate_config["stop"] = self.model_family.stop.copy()
         if (
             generate_config.get("stop_token_ids", None) is None
-            and self.model_family.prompt_style
-            and self.model_family.prompt_style.stop_token_ids
+            and self.model_family.stop_token_ids
         ):
-            generate_config[
-                "stop_token_ids"
-            ] = self.model_family.prompt_style.stop_token_ids.copy()
+            generate_config["stop_token_ids"] = self.model_family.stop_token_ids.copy()
 
         return generate_config
 
@@ -377,28 +364,20 @@ class MLXChatModel(MLXModel, ChatModelMixin):
 
     def chat(
         self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[MLXGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        tools = generate_config.pop("tools", []) if generate_config else None  # type: ignore
-        full_prompt = self.get_full_prompt(
-            self.model_family, prompt, system_prompt, chat_history, tools
+        model_family = self.model_family.model_family or self.model_family.model_name
+        tools = generate_config.pop("tools", []) if generate_config else None
+        full_context_kwargs = {}
+        if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+            full_context_kwargs["tools"] = tools
+        assert self.model_family.chat_template is not None
+        full_prompt = self.get_full_context(
+            messages, self.model_family.chat_template, **full_context_kwargs
         )
 
         generate_config = self._sanitize_generate_config(generate_config)
-        # TODO(codingl2k1): qwen hacky to set stop for function call.
-        model_family = self.model_family.model_family or self.model_family.model_name
-        if tools and model_family in ["qwen-chat", "qwen1.5-chat"]:
-            stop = generate_config.get("stop")
-            if isinstance(stop, str):
-                generate_config["stop"] = [stop, "Observation:"]
-            elif isinstance(stop, Iterable):
-                assert not isinstance(stop, str)
-                generate_config["stop"] = list(stop) + ["Observation:"]
-            else:
-                generate_config["stop"] = "Observation:"
 
         stream = generate_config.get("stream", False)
         if stream:
@@ -409,7 +388,5 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             c = self.generate(full_prompt, generate_config)
             assert not isinstance(c, Iterator)
             if tools:
-                return self._tool_calls_completion(
-                    self.model_family, self.model_uid, c, tools
-                )
+                return self._tool_calls_completion(self.model_family, self.model_uid, c)
             return self._to_chat_completion(c)

xinference/model/llm/sglang/core.py

@@ -21,7 +21,6 @@ from typing import AsyncGenerator, Dict, List, Optional, TypedDict, Union
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
-    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -29,7 +28,7 @@ from ....types import (
 )
 from .. import LLM, LLMFamilyV1, LLMSpecV1
 from ..llm_family import CustomLLMFamilyV1
-from ..utils import ChatModelMixin
+from ..utils import ChatModelMixin, generate_completion_chunk
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +82,9 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "mixtral-instruct-v0.1",
     "gemma-it",
     "gemma-2-it",
+    "deepseek-v2.5",
+    "deepseek-v2-chat",
+    "deepseek-v2-chat-0628",
 ]
 
 
@@ -319,6 +321,7 @@ class SGLANGModel(LLM):
         self,
         prompt: str,
         generate_config: Optional[SGLANGGenerateConfig] = None,
+        request_id: Optional[str] = None,
     ) -> Union[Completion, AsyncGenerator[CompletionChunk, None]]:
         sanitized_generate_config = self._sanitize_generate_config(generate_config)
         logger.debug(
@@ -332,8 +335,8 @@ class SGLANGModel(LLM):
             if isinstance(stream_options, dict)
             else False
         )
-
-        request_id = str(uuid.uuid1())
+        if not request_id:
+            request_id = str(uuid.uuid1())
         if not stream:
             state = await self._non_stream_generate(prompt, **sanitized_generate_config)
             return self._convert_state_to_completion(
@@ -346,12 +349,14 @@ class SGLANGModel(LLM):
 
             async def stream_results() -> AsyncGenerator[CompletionChunk, None]:
                 prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
+                finish_reason = None
                 async for meta_info, out in self._stream_generate(
                     prompt, **sanitized_generate_config
                 ):
                     chunk = self._convert_state_to_completion_chunk(
                         request_id, self.model_uid, output_text=out
                     )
+                    finish_reason = meta_info["finish_reason"]
                     prompt_tokens = meta_info["prompt_tokens"]
                     completion_tokens = meta_info["completion_tokens"]
                     total_tokens = prompt_tokens + completion_tokens
@@ -361,6 +366,26 @@ class SGLANGModel(LLM):
                         total_tokens=total_tokens,
                     )
                     yield chunk
+
+                finish_reason = (
+                    "stop"
+                    if finish_reason is None
+                    or (
+                        isinstance(finish_reason, str)
+                        and finish_reason.lower() == "none"
+                    )
+                    else finish_reason
+                )
+                yield generate_completion_chunk(
+                    "",
+                    finish_reason=finish_reason,
+                    chunk_id=request_id,
+                    model_uid=self.model_uid,
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    total_tokens=total_tokens,
+                )
+
                 if include_usage:
                     chunk = CompletionChunk(
                         id=request_id,
@@ -409,26 +434,19 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
     ) -> Dict:
         if not generate_config:
             generate_config = {}
-        if self.model_family.prompt_style:
-            if (
-                not generate_config.get("stop")
-            ) and self.model_family.prompt_style.stop:
-                generate_config["stop"] = self.model_family.prompt_style.stop.copy()
+        if self.model_family.stop:
+            if (not generate_config.get("stop")) and self.model_family.stop:
+                generate_config["stop"] = self.model_family.stop.copy()
         return generate_config
 
     async def async_chat(
         self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        messages: List[Dict],
         generate_config: Optional[Dict] = None,
+        request_id: Optional[str] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
-        assert self.model_family.prompt_style is not None
-        prompt_style = self.model_family.prompt_style.copy()
-        if system_prompt:
-            prompt_style.system_prompt = system_prompt
-        chat_history = chat_history or []
-        full_prompt = self.get_prompt(prompt, chat_history, prompt_style)
+        assert self.model_family.chat_template is not None
+        full_prompt = self.get_full_context(messages, self.model_family.chat_template)
 
         generate_config = self._sanitize_chat_config(generate_config)
         stream = generate_config.get("stream", None)