xinference 1.4.0__py3-none-any.whl → 1.4.1__py3-none-any.whl

xinference/model/llm/sglang/core.py

@@ -25,6 +25,7 @@ from xoscar.utils import get_next_port
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
+    ChatCompletionMessage,
     Completion,
     CompletionChoice,
     CompletionChunk,
@@ -95,7 +96,6 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "gemma-it",
     "gemma-2-it",
     "gemma-3-1b-it",
-    "gemma-3-it",
     "deepseek-v2.5",
     "deepseek-v2-chat",
     "deepseek-v2-chat-0628",
@@ -108,6 +108,12 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "deepseek-v3",
     "deepseek-r1",
 ]
+SGLANG_SUPPORTED_VISION_MODEL_LIST = [
+    "qwen2.5-vl-instruct",
+    "gemma-3-it",
+    "MiniCPM-V",
+    "llama-3.2-vision-instruct",
+]
 
 
 class SGLANGModel(LLM):
@@ -303,10 +309,6 @@ class SGLANGModel(LLM):
         if llm_spec.model_format == "pytorch":
             if quantization != "none" and not (quantization is None):
                 return False
-        if llm_spec.model_format in ["gptq", "awq"]:
-            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
-            if "4" not in quantization:
-                return False
         if isinstance(llm_family, CustomLLMFamilyV1):
             if llm_family.model_family not in SGLANG_SUPPORTED_MODELS:
                 return False
@@ -371,12 +373,18 @@ class SGLANGModel(LLM):
             sampling_params.pop("lora_name", None)
         return sampling_params
 
-    async def _stream_generate(self, prompt: str, **sampling_params):
+    async def _stream_generate(
+        self,
+        prompt: str,
+        image_data: Optional[Union[List[str], str]] = None,
+        **sampling_params,
+    ):
         import aiohttp
 
         sampling_params = self._filter_sampling_params(sampling_params)
         json_data = {
             "text": prompt,
+            "image_data": image_data,
             "sampling_params": sampling_params,
             "stream": True,
         }
@@ -404,12 +412,18 @@ class SGLANGModel(LLM):
                             if need_stop:
                                 break
 
-    async def _non_stream_generate(self, prompt: str, **sampling_params) -> dict:
+    async def _non_stream_generate(
+        self,
+        prompt: str,
+        image_data: Optional[Union[List[str], str]] = None,
+        **sampling_params,
+    ) -> dict:
         import aiohttp
 
         sampling_params = self._filter_sampling_params(sampling_params)
         json_data = {
             "text": prompt,
+            "image_data": image_data,
             "sampling_params": sampling_params,
         }
         async with aiohttp.ClientSession(trust_env=True) as session:
@@ -421,6 +435,7 @@ class SGLANGModel(LLM):
     async def async_generate(
         self,
         prompt: str,
+        image_data: Optional[Union[List[str], str]] = None,
         generate_config: Optional[SGLANGGenerateConfig] = None,
         request_id: Optional[str] = None,
     ) -> Union[Completion, AsyncGenerator[CompletionChunk, None]]:
@@ -439,7 +454,9 @@ class SGLANGModel(LLM):
         if not request_id:
             request_id = str(uuid.uuid1())
         if not stream:
-            state = await self._non_stream_generate(prompt, **sanitized_generate_config)
+            state = await self._non_stream_generate(
+                prompt, image_data, **sanitized_generate_config
+            )
             return self._convert_state_to_completion(
                 request_id,
                 model=self.model_uid,
@@ -452,7 +469,7 @@ class SGLANGModel(LLM):
                 prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
                 finish_reason = None
                 async for meta_info, out in self._stream_generate(
-                    prompt, **sanitized_generate_config
+                    prompt, image_data, **sanitized_generate_config
                 ):
                     chunk = self._convert_state_to_completion_chunk(
                         request_id, self.model_uid, output_text=out
@@ -515,10 +532,6 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
         if llm_spec.model_format == "pytorch":
             if quantization != "none" and not (quantization is None):
                 return False
-        if llm_spec.model_format in ["gptq", "awq"]:
-            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
-            if "4" not in quantization:
-                return False
         if isinstance(llm_family, CustomLLMFamilyV1):
             if llm_family.model_family not in SGLANG_SUPPORTED_CHAT_MODELS:
                 return False
@@ -559,3 +572,88 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
             c = await self.async_generate(full_prompt, generate_config)  # type: ignore
             assert not isinstance(c, AsyncGenerator)
             return self._to_chat_completion(c, self.reasoning_parser)
+
+
+class SGLANGVisionModel(SGLANGModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if not cls._has_cuda_device():
+            return False
+        if not cls._is_linux():
+            return False
+        if llm_spec.model_format not in ["pytorch", "gptq", "awq", "fp8"]:
+            return False
+        if llm_spec.model_format == "pytorch":
+            if quantization != "none" and not (quantization is None):
+                return False
+        if isinstance(llm_family, CustomLLMFamilyV1):
+            if llm_family.model_family not in SGLANG_SUPPORTED_VISION_MODEL_LIST:
+                return False
+        else:
+            if llm_family.model_name not in SGLANG_SUPPORTED_VISION_MODEL_LIST:
+                return False
+        if "vision" not in llm_family.model_ability:
+            return False
+        return SGLANG_INSTALLED
+
+    def _sanitize_chat_config(
+        self,
+        generate_config: Optional[Dict] = None,
+    ) -> Dict:
+        if not generate_config:
+            generate_config = {}
+        if self.model_family.stop:
+            if (not generate_config.get("stop")) and self.model_family.stop:
+                generate_config["stop"] = self.model_family.stop.copy()
+        return generate_config
+
+    async def async_chat(
+        self,
+        messages: List[ChatCompletionMessage],  # type: ignore
+        generate_config: Optional[Dict] = None,
+        request_id: Optional[str] = None,
+    ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
+        import base64
+        from io import BytesIO
+
+        from PIL import Image
+        from qwen_vl_utils import process_vision_info
+
+        messages = self._transform_messages(messages)
+
+        chat_template: str = (
+            self.model_family.chat_template if self.model_family.chat_template else ""
+        )
+
+        prompt = self.get_full_context(messages, chat_template)
+        images, video_inputs = process_vision_info(messages)
+        if video_inputs:
+            raise ValueError("Not support video input now.")
+
+        base64_images: Optional[List[str]] = None
+        if images:
+            base64_images = []
+            for image in images:
+                if isinstance(image, Image.Image):
+                    buffered = BytesIO()
+                    image.save(buffered, format="JPEG", quality=100)
+                    base64_images.append(base64.b64encode(buffered.getvalue()).decode())
+                elif isinstance(image, str):
+                    base64_images.append(image)
+                else:
+                    raise ValueError(
+                        f"Unsupported image type: {type(image)}, only support PIL.Image and base64 string"
+                    )
+
+        generate_config = self._sanitize_chat_config(generate_config)
+        stream = generate_config.get("stream", None)
+        if stream:
+            agen = await self.async_generate(prompt, base64_images, generate_config)  # type: ignore
+            assert isinstance(agen, AsyncGenerator)
+            return self._async_to_chat_completion_chunks(agen, self.reasoning_parser)
+        else:
+            c = await self.async_generate(prompt, base64_images, generate_config)  # type: ignore
+            assert not isinstance(c, AsyncGenerator)
+            return self._to_chat_completion(c, self.reasoning_parser)

xinference/model/llm/transformers/core.py

@@ -81,6 +81,7 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "cogagent",
     "gemma-3-1b-it",
     "gemma-3-it",
+    "deepseek-vl2",
 ]
 
 

xinference/model/llm/transformers/deepseek_vl.py

@@ -46,7 +46,7 @@ class DeepSeekVLChatModel(PytorchChatModel):
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         llm_family = model_family.model_family or model_family.model_name
-        if "deepseek-vl" in llm_family:
+        if "deepseek-vl" == llm_family.lower():
             return True
         return False
 

xinference/model/llm/transformers/deepseek_vl2.py

@@ -0,0 +1,287 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import base64
+import logging
+import os.path
+import tempfile
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from io import BytesIO
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+import requests
+import torch
+
+from ....model.utils import select_device
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
+from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class DeepSeekVL2ChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tokenizer = None
+        self._model = None
+        self._vl_chat_processor = None
+        self._type = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        llm_family = model_family.model_family or model_family.model_name
+        if "deepseek-vl2" == llm_family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoModelForCausalLM
+
+        from ....thirdparty.deepseek_vl2.models import (
+            DeepseekVLV2ForCausalLM,
+            DeepseekVLV2Processor,
+        )
+
+        self._device = self._pytorch_model_config.get("device", "auto")
+        self._device = select_device(self._device)
+        self._type = torch.float16 if self._device == "mps" else torch.bfloat16
+
+        # specify the path to the model
+        self._vl_chat_processor: DeepseekVLV2Processor = DeepseekVLV2Processor.from_pretrained(  # type: ignore
+            self.model_path
+        )
+        self._tokenizer = self._vl_chat_processor.tokenizer
+
+        vl_gpt: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(  # type: ignore
+            self.model_path, trust_remote_code=True, device_map=self._device
+        )
+        self._model = vl_gpt.to(torch.bfloat16).cuda().eval()
+
+    @staticmethod
+    def _message_content_to_deepseek(content) -> Tuple[str, List[str]]:
+        def _ensure_url(_url):
+            if _url.startswith("data:"):
+                logging.info("Parse url by base64 decoder.")
+                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+                # e.g. f"data:image/jpeg;base64,{base64_image}"
+                _type, data = _url.split(";")
+                _, ext = _type.split("/")
+                data = data[len("base64,") :]
+                data = base64.b64decode(data.encode("utf-8"))
+
+                with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as f:
+                    f.write(data)
+                logging.info("Dump base64 data to %s", f.name)
+                return f.name
+            else:
+                if len(_url) > 2048:
+                    raise Exception(f"Image url is too long, {len(_url)} > 2048.")
+
+                return _url
+
+        def _download(_images):
+            local_images = []
+
+            # To make requests.get works
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+            }
+            with ThreadPoolExecutor() as executor:
+                for url in images:
+                    try:
+                        if os.path.exists(url):
+                            local_images.append(url)
+                            continue
+                    except Exception as e:
+                        logger.debug("Image is remote: %s, e: %s", url, e)
+                        pass
+                    # Append a placeholder
+                    local_images.append(None)
+
+                    def _fill_placeholder(_url, _index):
+                        response = requests.get(url, headers=headers)
+                        local_images[_index] = BytesIO(response.content)
+
+                    executor.submit(_fill_placeholder, url, len(local_images) - 1)
+            return local_images
+
+        if not isinstance(content, str):
+            # TODO(codingl2k1): Optimize _ensure_url
+
+            images = []
+            new_content = []
+            for c in content:
+                c_type = c.get("type")
+                if c_type == "image_url":
+                    images.append(_ensure_url(c["image_url"]["url"]))
+                elif c_type == "text":
+                    new_content.append(c["text"])
+            if images:
+                new_content.insert(0, "<image_placeholder>")
+                images = _download(images)
+            return "".join(new_content), images
+        return content, []
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        if not generate_config:
+            generate_config = {}
+
+        stream = generate_config.get("stream", False)
+        stream_options = generate_config.pop("stream_options", None)
+        include_usage = (
+            stream_options["include_usage"]
+            if isinstance(stream_options, dict)
+            else False
+        )
+
+        prompt = ""
+        deepseek_messages = []
+        for i, message in enumerate(messages):
+            role = message["role"]
+            content = message["content"]
+            if role == "user":
+                if isinstance(content, str):
+                    deepseek_messages.append(
+                        {
+                            "role": "<|User|>",
+                            "content": "<image>\n<|ref|>" + content + "<|/ref|>",
+                        }
+                    )
+                else:
+                    content, images = self._message_content_to_deepseek(content)
+                    msg: Dict[str, Any] = {
+                        "role": "<|User|>",
+                        "content": "<image>\n<|ref|>" + content + "<|/ref|>",
+                    }
+                    if images:
+                        msg["images"] = images
+                    deepseek_messages.append(msg)
+                    deepseek_messages.append({"role": "<|Assistant|>", "content": ""})
+                if i == len(messages) - 1:
+                    prompt = "<image>\n<|ref|>" + content + "<|/ref|>"
+            elif role == "assistant":
+                deepseek_messages.append({"role": "<|Assistant|>", "content": content})
+            else:
+                logger.error(
+                    f"Unexpected message in messages: role: {role}, message: {message}"
+                )
+
+        from ....thirdparty.deepseek_vl2.utils.io import load_pil_images
+
+        # load images and prepare for inputs
+        pil_images = load_pil_images(deepseek_messages)
+        prepare_inputs = self._vl_chat_processor(
+            conversations=deepseek_messages,
+            images=pil_images,
+            force_batchify=True,
+            system_prompt="",
+        ).to(self._model.device, self._model.dtype)
+
+        # run image encoder to get the image embeddings
+        inputs_embeds = self._model.prepare_inputs_embeds(**prepare_inputs)
+
+        max_new_tokens = generate_config.get("max_tokens", 512)
+        conversation = self._vl_chat_processor.new_chat_template()
+        stop_str = conversation.sep2
+
+        streamer = self._model.language.generate(
+            inputs_embeds=inputs_embeds,
+            attention_mask=prepare_inputs.attention_mask,
+            pad_token_id=self._tokenizer.eos_token_id,
+            bos_token_id=self._tokenizer.bos_token_id,
+            eos_token_id=self._tokenizer.eos_token_id,
+            max_new_tokens=max_new_tokens,
+            do_sample=False,
+            use_cache=True,
+        )
+
+        if stream:
+            it = self._generate_stream(streamer, stop_str, include_usage, prompt)
+            return self._to_chat_completion_chunks(it)
+        else:
+            return self._generate(streamer, stop_str)
+
+    def _generate(self, streamer, stop_str) -> ChatCompletion:
+        generated_text = ""
+
+        for new_text in streamer:
+            if isinstance(new_text, torch.Tensor):
+                new_text = self._tokenizer.decode(
+                    new_text.cpu().tolist(), skip_special_tokens=True
+                )
+
+            if new_text.endswith(stop_str):
+                new_text = new_text[: -len(stop_str)]
+
+            generated_text += new_text
+
+        return generate_chat_completion(self.model_uid, generated_text)
+
+    def _generate_stream(
+        self, streamer, stop_str, include_usage, prompt
+    ) -> Iterator[CompletionChunk]:
+        completion_id = str(uuid.uuid1())
+        prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
+        input_ids = self._tokenizer(prompt).input_ids
+        prompt_tokens = len(input_ids)
+        for i, new_text in enumerate(streamer):
+            if new_text.endswith(stop_str):
+                new_text = new_text[: -len(stop_str)]
+            completion_tokens = i
+            total_tokens = prompt_tokens + completion_tokens
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                has_choice=True,
+                has_content=True,
+            )
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            has_choice=True,
+            has_content=False,
+        )
+
+        if include_usage:
+            yield generate_completion_chunk(
+                chunk_text=None,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                has_choice=False,
+                has_content=False,
+            )

xinference/model/llm/utils.py

@@ -255,19 +255,26 @@ class ChatModelMixin:
             and choices
             and "delta" in choices[0]
         ):
-            if reasoning_parser is not None:
-                # process parsing reasoning content
-                assert previous_texts is not None
+            if choices[0]["finish_reason"] is None:
+                if reasoning_parser is not None:
+                    # process parsing reasoning content
+                    assert previous_texts is not None
+                    delta = choices[0]["delta"]  # type: ignore
+                    if text := delta.get("content"):
+                        current_text = previous_texts[-1] + text
+                        delta = reasoning_parser.extract_reasoning_content_streaming(
+                            previous_text=previous_texts[-1],
+                            current_text=current_text,
+                            delta_text=text,
+                        )
+                        previous_texts[-1] = current_text
+                        choices[0]["delta"] = delta  # type: ignore
+            elif choices[0]["finish_reason"] is not None:
                 delta = choices[0]["delta"]  # type: ignore
-                if text := delta.get("content"):
-                    current_text = previous_texts[-1] + text
-                    delta = reasoning_parser.extract_reasoning_content_streaming(
-                        previous_text=previous_texts[-1],
-                        current_text=current_text,
-                        delta_text=text,
-                    )
-                    previous_texts[-1] = current_text
-                    choices[0]["delta"] = delta  # type: ignore
+                if "content" not in delta:
+                    delta["content"] = ""  # type: ignore
+                if reasoning_parser is not None:
+                    delta["reasoning_content"] = None  # type: ignore
             # Already a ChatCompletionChunk, we don't need to convert chunk.
             return cast(ChatCompletionChunk, chunk)
 
@@ -286,7 +293,11 @@ class ChatModelMixin:
                         delta_text=choice["text"],
                     )
                     previous_texts[-1] = current_text
-            if "tool_calls" in choice:
+            elif "text" in choice and choice["finish_reason"] is not None:
+                delta["content"] = choice["text"]
+                if reasoning_parser is not None:
+                    delta["reasoning_content"] = None
+            elif "tool_calls" in choice:
                 delta["tool_calls"] = choice["tool_calls"]
             choices_list.append(
                 {
@@ -319,8 +330,9 @@ class ChatModelMixin:
     ) -> ChatCompletionChunk:
         choices_list = []
         for i, choice in enumerate(chunk["choices"]):
-            delta = {"role": "assistant", "content": ""}
+            delta = ChatCompletionChunkDelta(role="assistant", content="")
             if reasoning_parser is not None:
+                delta["content"] = None
                 delta["reasoning_content"] = ""
             choices_list.append(
                 {