xinference 0.12.3__py3-none-any.whl → 0.13.0__py3-none-any.whl

xinference/model/llm/pytorch/qwen_vl.py

@@ -16,9 +16,14 @@ import logging
 import operator
 import tempfile
 import time
+import typing
 import uuid
-from typing import Dict, Iterator, List, Optional, Union
+from typing import Dict, Iterator, List, Optional, Tuple, Union
 
+import torch
+from transformers import PreTrainedTokenizer
+
+from ....core.scheduler import InferenceRequest
 from ....model.utils import select_device
 from ....types import (
     ChatCompletion,
@@ -31,6 +36,7 @@ from ....types import (
 )
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import pad_prefill_tokens
 
 logger = logging.getLogger(__name__)
 
@@ -40,6 +46,7 @@ class QwenVLChatModel(PytorchChatModel):
         super().__init__(*args, **kwargs)
         self._tokenizer = None
         self._model = None
+        self._device = None
 
     @classmethod
     def match(
@@ -62,6 +69,7 @@ class QwenVLChatModel(PytorchChatModel):
 
         device = self._pytorch_model_config.get("device", "auto")
         device = select_device(device)
+        self._device = device
         # for multiple GPU, set back to auto to make multiple devices work
         device = "auto" if device == "cuda" else device
 
@@ -120,13 +128,11 @@ class QwenVLChatModel(PytorchChatModel):
             return self._tokenizer.from_list_format(content)
         return content
 
-    def chat(
+    def _get_prompt_and_chat_history(
         self,
         prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
         chat_history: Optional[List[ChatCompletionMessage]] = None,
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+    ):
         prompt = self._message_content_to_qwen(prompt)
         # Convert openai history to qwen vl history
         qwen_history = []
@@ -141,6 +147,18 @@ class QwenVLChatModel(PytorchChatModel):
             if len(query_to_response) == 2:
                 qwen_history.append(query_to_response)
                 query_to_response = []
+        return prompt, qwen_history
+
+    def chat(
+        self,
+        prompt: Union[str, List[Dict]],
+        system_prompt: Optional[str] = None,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        prompt, qwen_history = self._get_prompt_and_chat_history(
+            prompt, chat_history=chat_history
+        )
 
         stream = generate_config.get("stream", False) if generate_config else False
         stream_options = (
@@ -152,10 +170,10 @@ class QwenVLChatModel(PytorchChatModel):
             else False
         )
         if stream:
-            it = self._generate_stream(prompt, qwen_history, include_usage)
+            it = self._generate_stream(prompt, qwen_history, include_usage)  # type: ignore
             return self._to_chat_completion_chunks(it)
         else:
-            c = self._generate(prompt, qwen_history)
+            c = self._generate(prompt, qwen_history)  # type: ignore
             return self._to_chat_completion(c)
 
     def _generate(self, prompt: str, qwen_history: List) -> Completion:
@@ -244,3 +262,146 @@ class QwenVLChatModel(PytorchChatModel):
                 total_tokens=total_tokens,
             )
             yield chunk
+
+    @staticmethod
+    def get_batch_size_and_seq_len_indexes_from_kv() -> Tuple[int, int]:
+        """
+        Qwen-vl is very special for its kv_cache impl.
+        Its dimension is `bs * seq_len * head_num * dim`.
+        See https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/modeling_qwen.py
+        """
+        return 0, 1
+
+    @staticmethod
+    @typing.no_type_check
+    def make_context(
+        tokenizer: PreTrainedTokenizer,
+        query: str,
+        history: List[Tuple[str, str]] = None,
+        system: str = "",
+        max_window_size: int = 6144,
+        chat_format: str = "chatml",
+    ):
+        """
+        This function is from https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py.
+        Use this function to get input_ids with image.
+        """
+        if history is None:
+            history = []
+
+        if chat_format == "chatml":
+            im_start, im_end = "<|im_start|>", "<|im_end|>"
+            im_start_tokens = [tokenizer.im_start_id]
+            im_end_tokens = [tokenizer.im_end_id]
+            nl_tokens = tokenizer.encode("\n")
+
+            def _tokenize_str(role, content):
+                return f"{role}\n{content}", tokenizer.encode(
+                    role, allowed_special=set(tokenizer.IMAGE_ST)
+                ) + nl_tokens + tokenizer.encode(
+                    content, allowed_special=set(tokenizer.IMAGE_ST)
+                )
+
+            system_text, system_tokens_part = _tokenize_str("system", system)
+            system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
+
+            raw_text = ""
+            context_tokens = []
+
+            for turn_query, turn_response in reversed(history):
+                query_text, query_tokens_part = _tokenize_str("user", turn_query)
+                query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
+                if turn_response is not None:
+                    response_text, response_tokens_part = _tokenize_str(
+                        "assistant", turn_response
+                    )
+                    response_tokens = (
+                        im_start_tokens + response_tokens_part + im_end_tokens
+                    )
+
+                    next_context_tokens = (
+                        nl_tokens + query_tokens + nl_tokens + response_tokens
+                    )
+                    prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
+                else:
+                    next_context_tokens = nl_tokens + query_tokens + nl_tokens
+                    prev_chat = f"\n{im_start}{query_text}{im_end}\n"
+
+                current_context_size = (
+                    len(system_tokens) + len(next_context_tokens) + len(context_tokens)
+                )
+                if current_context_size < max_window_size:
+                    context_tokens = next_context_tokens + context_tokens
+                    raw_text = prev_chat + raw_text
+                else:
+                    break
+
+            context_tokens = system_tokens + context_tokens
+            raw_text = f"{im_start}{system_text}{im_end}" + raw_text
+            context_tokens += (
+                nl_tokens
+                + im_start_tokens
+                + _tokenize_str("user", query)[1]
+                + im_end_tokens
+                + nl_tokens
+                + im_start_tokens
+                + tokenizer.encode("assistant")
+                + nl_tokens
+            )
+            raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
+
+        elif chat_format == "raw":
+            raw_text = query
+            context_tokens = tokenizer.encode(raw_text)
+        else:
+            raise NotImplementedError(f"Unknown chat format {chat_format!r}")
+
+        return raw_text, context_tokens
+
+    def _get_full_prompt(self, prompt, system_prompt, chat_history, tools):
+        prompt, qwen_history = self._get_prompt_and_chat_history(
+            prompt, chat_history=chat_history
+        )
+        _, context_tokens = self.make_context(self._tokenizer, prompt, qwen_history)
+        return context_tokens
+
+    def prepare_sanitize_generate_config(self, req: InferenceRequest):
+        """
+        Refer to https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/generation_config.json
+        """
+        raw_config = req.inference_kwargs.get("raw_params", {})
+        top_p = raw_config.get("top_p", None)
+        if top_p is None:
+            raw_config["top_p"] = 0.3
+        top_k = raw_config.get("top_k", None)
+        if top_k is None:
+            raw_config["top_k"] = 0
+        return raw_config
+
+    def build_prefill_inputs(self, prompts: List, req_list: List[InferenceRequest]):
+        context_len = self.get_context_len()
+        inputs = pad_prefill_tokens(prompts, context_len, req_list)
+        input_ids = torch.as_tensor(
+            pad_prefill_tokens(inputs, context_len, req_list), device=self._device
+        )
+        return input_ids
+
+    def build_prefill_position_ids(
+        self, batch_size: int, seq_length: int, reqs: List[InferenceRequest]
+    ):
+        """
+        Qwen-vl fill `1` for position_ids padding
+        """
+        res = []
+        for r in reqs:
+            real_seq_len = seq_length - r.padding_len
+            res.append(
+                torch.cat(
+                    [
+                        torch.full((r.padding_len,), 1, dtype=torch.long),
+                        torch.arange(0, real_seq_len, dtype=torch.long),
+                    ]
+                )
+            )
+            r.extra_kwargs["max_position_id"] = real_seq_len - 1
+        return torch.stack(res).to(self._device)

xinference/model/llm/pytorch/utils.py

@@ -17,7 +17,7 @@ import logging
 import os
 import time
 import uuid
-from typing import Dict, Iterable, Iterator, List, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Optional, Tuple
 
 import torch
 from transformers.cache_utils import DynamicCache
@@ -39,6 +39,10 @@ from ....types import (
     max_tokens_field,
 )
 
+if TYPE_CHECKING:
+    from ...llm.pytorch.core import PytorchModel
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -414,6 +418,19 @@ def get_max_src_len(context_len: int, r: InferenceRequest) -> int:
     return context_len - max_new_tokens - 8
 
 
+def pad_prefill_tokens(
+    input_ids: List[List[int]], context_len: int, req_list: List[InferenceRequest]
+):
+    prompt_tokens = []
+    for i, input_id in enumerate(input_ids):
+        req = req_list[i]
+        max_src_len = get_max_src_len(context_len, req)
+        req.prompt_tokens = input_id[-max_src_len:]
+        prompt_tokens.append(req.prompt_tokens)
+    _pad_seqs_inplace(prompt_tokens, req_list, 0)
+    return prompt_tokens
+
+
 def _get_completion_chunk(
     output: str,
     chunk_id: str,
@@ -481,23 +498,33 @@ def _get_completion(
     return completion
 
 
+def _get_pad_param(seq_len_idx: int, pad_len: int) -> Tuple:
+    dimensions = [0] * 8
+    dimensions[-2 * (seq_len_idx + 1)] = pad_len
+    return tuple(dimensions)
+
+
 def _merge_kv_cache(
-    past_kv: Tuple[Tuple[torch.Tensor]], new_kv: Tuple[Tuple[torch.Tensor]]
+    xinf_model_obj: "PytorchModel",
+    past_kv: Tuple[Tuple[torch.Tensor]],
+    new_kv: Tuple[Tuple[torch.Tensor]],
 ):
     from torch.nn.functional import pad
 
+    _, seq_len_idx = xinf_model_obj.get_batch_size_and_seq_len_indexes_from_kv()
     past_cache = DynamicCache.from_legacy_cache(past_kv)
     new_cache = DynamicCache.from_legacy_cache(new_kv)
-    past_seq_len = past_cache.get_seq_length()
-    new_seq_len = new_cache.get_seq_length()
+    past_seq_len = past_kv[0][0].shape[seq_len_idx]
+    new_seq_len = new_kv[0][0].shape[seq_len_idx]
     if past_seq_len != new_seq_len:
         padding_target = new_cache if past_seq_len > new_seq_len else past_cache
         padding_len = abs(past_seq_len - new_seq_len)
+        pad_param = _get_pad_param(seq_len_idx, padding_len)
         for idx in range(len(padding_target)):
             k = padding_target.key_cache[idx]
             v = padding_target.value_cache[idx]
-            _k = pad(k, (0, 0, padding_len, 0))
-            _v = pad(v, (0, 0, padding_len, 0))
+            _k = pad(k, pad_param)
+            _v = pad(v, pad_param)
             padding_target.key_cache[idx] = _k
             padding_target.value_cache[idx] = _v
 
@@ -509,36 +536,19 @@ def _merge_kv_cache(
     return ret_kv.to_legacy_cache()
 
 
-def _get_attention_mask_and_position_ids(kv, reqs: List[InferenceRequest]):
-    batch_size, seq_length, device = (
-        kv[0][0].shape[0],
-        kv[0][0].shape[2],
-        kv[0][0].device,
-    )
-    seq_length = seq_length + 1
-    position_ids = torch.as_tensor([[seq_length - 1]], dtype=torch.long, device=device)
-    attention_mask = torch.ones(
-        (batch_size, seq_length), dtype=torch.long, device=device
-    )
-    padding_lens = torch.as_tensor([r.padding_len for r in reqs])
-    mask = torch.arange(seq_length).expand(
-        batch_size, seq_length
-    ) < padding_lens.unsqueeze(1)
-    attention_mask[mask] = 0
-    return attention_mask, position_ids
+def get_batch_size_and_seq_len_from_kv_cache(kv, xinf_model_obj: "PytorchModel"):
+    bs_idx, seq_len_idx = xinf_model_obj.get_batch_size_and_seq_len_indexes_from_kv()
+    return kv[0][0].shape[bs_idx], kv[0][0].shape[seq_len_idx] + 1
 
 
 @torch.inference_mode()
 def _batch_inference_one_step_internal(
+    xinf_model_obj: "PytorchModel",
     req_list: List[InferenceRequest],
     model_uid,
     model,
     tokenizer,
-    device,
-    context_len: int,
-    stop_tokens: Tuple[int],
     decode_round: int = 16,
-    require_attention_mask: bool = False,
     bos_flag: str = "<bos_stream>",
     eos_flag: str = "<eos_stream>",
 ):
@@ -548,7 +558,9 @@ def _batch_inference_one_step_internal(
     if not valid_req_list:
         return
     generate_config_mapping: Dict[InferenceRequest, Tuple] = {
-        r: r.get_generate_configs(tokenizer.eos_token_id, stop_tokens)
+        r: r.get_generate_configs(
+            tokenizer.eos_token_id, xinf_model_obj.get_builtin_stop_token_ids()
+        )
         for r in valid_req_list
     }
     s_time = time.time()
@@ -564,15 +576,8 @@ def _batch_inference_one_step_internal(
             decode_reqs.append(r)
 
     if prompts:  # prefill first
-        input_ids: List[List[int]] = tokenizer(prompts, padding=False).input_ids
-        prompt_tokens = []
-        for i, input_id in enumerate(input_ids):
-            req = valid_req_list[i]
-            max_src_len = get_max_src_len(context_len, req)
-            req.prompt_tokens = input_id[-max_src_len:]
-            prompt_tokens.append(req.prompt_tokens)
-        _pad_seqs_inplace(prompt_tokens, valid_req_list, 0)
-        out = model(torch.as_tensor(prompt_tokens, device=device), use_cache=True)
+        prefill_kws = xinf_model_obj.build_prefill_kwargs(prompts, prefill_reqs)
+        out = model(**prefill_kws, use_cache=True)
 
         logits = out.logits
         past_key_values = out.past_key_values
@@ -599,7 +604,9 @@ def _batch_inference_one_step_internal(
         if decode_reqs:
             decode_kv = decode_reqs[0].kv_cache
             # prefill and decode kv cache need to be merged at `batch_size` and `seq_len` dimensions.
-            merged_kv_cache = _merge_kv_cache(decode_kv, past_key_values)
+            merged_kv_cache = _merge_kv_cache(
+                xinf_model_obj, decode_kv, past_key_values
+            )
             for r in valid_req_list:
                 r.kv_cache = merged_kv_cache
             empty_cache()
@@ -612,20 +619,14 @@ def _batch_inference_one_step_internal(
     output_mapping: Dict[InferenceRequest, str] = {}
     # here, only decode phase, just run some rounds
     for _i in range(decode_round):
+        batch_size, seq_len = get_batch_size_and_seq_len_from_kv_cache(
+            past_key_values, xinf_model_obj
+        )
         decode_tokens: List[List[int]] = [[r.new_tokens[-1]] for r in valid_req_list]
-        inf_kws = {}
-        if require_attention_mask:
-            attention_mask, position_ids = _get_attention_mask_and_position_ids(
-                past_key_values, valid_req_list
-            )
-            inf_kws["position_ids"] = position_ids
-            inf_kws["attention_mask"] = attention_mask
-        out = model(
-            input_ids=torch.as_tensor(decode_tokens, device=device),
-            use_cache=True,
-            past_key_values=past_key_values,
-            **inf_kws,
+        inf_kws = xinf_model_obj.build_decode_kwargs(
+            decode_tokens, valid_req_list, batch_size, seq_len
         )
+        out = model(**inf_kws, use_cache=True, past_key_values=past_key_values)
         logits = out.logits
         past_key_values = out.past_key_values
 
@@ -755,27 +756,17 @@ def _batch_inference_one_step_internal(
 
 
 def batch_inference_one_step(
+    xinf_model_obj: "PytorchModel",
     req_list: List[InferenceRequest],
     model_uid,
     model,
     tokenizer,
-    device,
-    context_len: int,
-    stop_token_ids: Tuple[int],
-    require_attention_mask: bool = False,
 ):
     from ....core.model import OutOfMemoryError
 
     try:
         _batch_inference_one_step_internal(
-            req_list,
-            model_uid,
-            model,
-            tokenizer,
-            device,
-            context_len,
-            stop_token_ids,
-            require_attention_mask=require_attention_mask,
+            xinf_model_obj, req_list, model_uid, model, tokenizer
         )
     except OutOfMemoryError:
         logger.exception(

xinference/model/llm/utils.py

@@ -47,6 +47,11 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen2-moe-instruct",
 ]
 
+GLM4_TOOL_CALL_FAMILY = [
+    "glm4-chat",
+    "glm4-chat-1m",
+]
+
 
 class ChatModelMixin:
     @staticmethod
@@ -617,9 +622,13 @@ Begin!"""
 
     @staticmethod
     def _eval_glm_chat_arguments(c, tools):
-        if isinstance(c[0], str):
-            return c[0], None, None
-        return None, c[0]["name"], c[0]["parameters"]
+        try:
+            if isinstance(c[0], str):
+                return c[0], None, None
+            return None, c[0]["name"], c[0]["parameters"]
+        except KeyError:
+            logger.error("Can't parse glm output: %s", c)
+            return str(c), None, None
 
     @staticmethod
     def _eval_qwen_chat_arguments(c, tools):
@@ -668,7 +677,7 @@ Begin!"""
         family = model_family.model_family or model_family.model_name
         if family in ["gorilla-openfunctions-v1", "gorilla-openfunctions-v2"]:
             content, func, args = cls._eval_gorilla_openfunctions_arguments(c, tools)
-        elif family in ["chatglm3", "glm4-chat"]:
+        elif family in ["chatglm3"] + GLM4_TOOL_CALL_FAMILY:
             content, func, args = cls._eval_glm_chat_arguments(c, tools)
         elif family in QWEN_TOOL_CALL_FAMILY:
             content, func, args = cls._eval_qwen_chat_arguments(c, tools)
@@ -756,6 +765,16 @@ Begin!"""
             "usage": usage,
         }
 
+    @classmethod
+    def get_full_prompt(cls, model_family, prompt, system_prompt, chat_history, tools):
+        assert model_family.prompt_style is not None
+        prompt_style = model_family.prompt_style.copy()
+        if system_prompt:
+            prompt_style.system_prompt = system_prompt
+        chat_history = chat_history or []
+        full_prompt = cls.get_prompt(prompt, chat_history, prompt_style, tools=tools)
+        return full_prompt
+
 
 def get_file_location(
     llm_family: LLMFamilyV1, spec: LLMSpecV1, quantization: str
@@ -772,7 +791,7 @@ def get_file_location(
         is_cached = cache_status
     assert isinstance(is_cached, bool)
 
-    if spec.model_format in ["pytorch", "gptq", "awq"]:
+    if spec.model_format in ["pytorch", "gptq", "awq", "mlx"]:
         return cache_dir, is_cached
     elif spec.model_format in ["ggmlv3", "ggufv2"]:
         assert isinstance(spec, GgmlLLMSpecV1)

xinference/model/rerank/core.py

@@ -17,6 +17,7 @@ import logging
 import os
 import uuid
 from collections import defaultdict
+from collections.abc import Sequence
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
@@ -217,7 +218,11 @@ class RerankModel:
             if similarity_scores.dtype == torch.bfloat16:
                 similarity_scores = similarity_scores.float()
         else:
+            # Related issue: https://github.com/xorbitsai/inference/issues/1775
             similarity_scores = self._model.compute_score(sentence_combinations)
+            if not isinstance(similarity_scores, Sequence):
+                similarity_scores = [similarity_scores]
+
         sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))
         if top_n is not None:
             sim_scores_argsort = sim_scores_argsort[:top_n]

xinference/thirdparty/deepseek_vl/serve/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.