xinference 0.16.3__py3-none-any.whl → 1.2.1__py3-none-any.whl

xinference/model/llm/utils.py

@@ -52,6 +52,7 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen2-instruct",
     "qwen2-moe-instruct",
     "qwen2.5-instruct",
+    "qwen2.5-coder-instruct",
 ]
 
 GLM4_TOOL_CALL_FAMILY = [
@@ -96,13 +97,22 @@ class ChatModelMixin:
         return rendered
 
     def get_full_context(
-        self, messages: List, chat_template: str, tokenizer=None, **kwargs
-    ) -> str:
+        self,
+        messages: List,
+        chat_template: str,
+        tokenizer=None,
+        tokenize=False,
+        **kwargs,
+    ):
+        if "vision" not in self.model_family.model_ability:  # type: ignore
+            messages = self.convert_messages_with_content_list_to_str_conversion(
+                messages
+            )
         if tokenizer is not None:
             try:
                 full_context = tokenizer.apply_chat_template(
                     messages,
-                    tokenize=False,
+                    tokenize=tokenize,
                     chat_template=chat_template,
                     add_generation_prompt=True,
                     **kwargs,
@@ -118,6 +128,25 @@ class ChatModelMixin:
             # Compilation function uses a cache to avoid recompiling the same template
             return self._build_from_raw_template(messages, chat_template, **kwargs)
 
+    @staticmethod
+    def convert_messages_with_content_list_to_str_conversion(
+        messages: List[Dict],
+    ) -> List[Dict]:
+        """
+        Handles messages with content list conversion, in order to support Cline, see GH#2659 .
+        """
+        for message in messages:
+            texts = ""
+            msg_content = message.get("content")
+            if msg_content:
+                if isinstance(msg_content, str):
+                    texts = msg_content
+                elif isinstance(msg_content, list):
+                    texts = "\n".join(item.get("text", "") for item in msg_content)
+            if texts:
+                message["content"] = texts
+        return messages
+
     @staticmethod
     def get_specific_prompt(model_family: str, messages: List[ChatCompletionMessage]):
         """
@@ -324,7 +353,10 @@ class ChatModelMixin:
         """
         try:
             if isinstance(c, dict):
-                return [(None, c["name"], c["arguments"])]
+                try:
+                    return [(None, c["name"], json.loads(c["arguments"]))]
+                except Exception:
+                    return [(None, c["name"], c["arguments"])]
         except KeyError:
             logger.error("Can't parse glm output: %s", c)
             return [(str(c), None, None)]
@@ -569,6 +601,25 @@ def _decode_image(_url):
             return Image.open(BytesIO(response.content)).convert("RGB")
 
 
+def _decode_image_without_rgb(_url):
+    if _url.startswith("data:"):
+        logging.info("Parse url by base64 decoder.")
+        # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+        # e.g. f"data:image/jpeg;base64,{base64_image}"
+        _type, data = _url.split(";")
+        _, ext = _type.split("/")
+        data = data[len("base64,") :]
+        data = base64.b64decode(data.encode("utf-8"))
+        return Image.open(BytesIO(data))
+    else:
+        try:
+            response = requests.get(_url)
+        except requests.exceptions.MissingSchema:
+            return Image.open(_url)
+        else:
+            return Image.open(BytesIO(response.content))
+
+
 @typing.no_type_check
 def generate_completion_chunk(
     chunk_text: Optional[str],

xinference/model/llm/vllm/core.py

@@ -69,6 +69,8 @@ class VLLMModelConfig(TypedDict, total=False):
     quantization: Optional[str]
     max_model_len: Optional[int]
     limit_mm_per_prompt: Optional[Dict[str, int]]
+    guided_decoding_backend: Optional[str]
+    scheduling_policy: Optional[str]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -85,6 +87,15 @@ class VLLMGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     stream: bool  # non-sampling param, should not be passed to the engine.
     stream_options: Optional[Union[dict, None]]
+    skip_special_tokens: Optional[bool]
+    response_format: Optional[dict]
+    guided_json: Optional[Union[str, dict]]
+    guided_regex: Optional[str]
+    guided_choice: Optional[List[str]]
+    guided_grammar: Optional[str]
+    guided_json_object: Optional[bool]
+    guided_decoding_backend: Optional[str]
+    guided_whitespace_pattern: Optional[str]
 
 
 try:
@@ -144,7 +155,9 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-instruct")
     VLLM_SUPPORTED_MODELS.append("qwen2.5-coder")
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-coder-instruct")
-
+    VLLM_SUPPORTED_CHAT_MODELS.append("QwQ-32B-Preview")
+    VLLM_SUPPORTED_CHAT_MODELS.append("marco-o1")
+    VLLM_SUPPORTED_CHAT_MODELS.append("deepseek-r1-distill-qwen")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
     VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
@@ -171,14 +184,19 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.5.3":
 if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
+    VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.3-instruct")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.2":
+    VLLM_SUPPORTED_CHAT_MODELS.append("minicpm3-4b")
+
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.2-vision")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("llama-3.2-vision-instruct")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("QvQ-72B-Preview")
 
 
 class VLLMModel(LLM):
@@ -207,6 +225,10 @@ class VLLMModel(LLM):
         self._engine = None
         self.lora_modules = peft_model
         self.lora_requests: List[LoRARequest] = []
+        self._xavier_config = None
+
+    def set_xavier_config(self, value: Optional[Dict]):
+        self._xavier_config = value  # type: ignore
 
     def load(self):
         try:
@@ -232,7 +254,6 @@ class VLLMModel(LLM):
             multiprocessing.set_start_method("fork", force=True)
 
         self._model_config = self._sanitize_model_config(self._model_config)
-
         if self.lora_modules is None:
             self.lora_requests = []
         else:
@@ -253,13 +274,34 @@ class VLLMModel(LLM):
             f"Enable lora: {enable_lora}. Lora count: {max_loras}."
         )
 
-        engine_args = AsyncEngineArgs(
-            model=self.model_path,
-            enable_lora=enable_lora,
-            max_loras=max_loras,
-            **self._model_config,
-        )
-        self._engine = AsyncLLMEngine.from_engine_args(engine_args)
+        if self._xavier_config is not None:
+            from .xavier.engine import XavierEngine
+
+            # Enabling Xavier means that `enable_prefix_caching` is enabled by default.
+            self._model_config.setdefault("enable_prefix_caching", True)
+            xavier_transfer_block_num = self._model_config.pop(
+                "xavier_transfer_block_num", 512
+            )
+            self._xavier_config["transfer_block_num"] = xavier_transfer_block_num
+            engine_args = AsyncEngineArgs(
+                model=self.model_path,
+                enable_lora=enable_lora,
+                max_loras=max_loras,
+                **self._model_config,
+            )
+
+            logger.debug(f"Start xavier for vllm with config: {self._xavier_config}")
+            self._engine = XavierEngine.from_engine_args(
+                engine_args, xavier_config=self._xavier_config
+            )
+        else:
+            engine_args = AsyncEngineArgs(
+                model=self.model_path,
+                enable_lora=enable_lora,
+                max_loras=max_loras,
+                **self._model_config,
+            )
+            self._engine = AsyncLLMEngine.from_engine_args(engine_args)
 
         self._check_health_task = None
         if hasattr(self._engine, "check_health"):
@@ -277,6 +319,9 @@ class VLLMModel(LLM):
             model_executor.shutdown()
         self._engine = None
 
+    async def init_xavier(self):
+        await self._engine.init_xavier()
+
     async def _check_healthy(self, interval: int = 30):
         from vllm.engine.async_llm_engine import AsyncEngineDeadError
 
@@ -314,7 +359,10 @@ class VLLMModel(LLM):
         model_config.setdefault("max_num_seqs", 256)
         model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
-
+        model_config.setdefault("guided_decoding_backend", "outlines")
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
         return model_config
 
     @staticmethod
@@ -325,6 +373,22 @@ class VLLMModel(LLM):
             generate_config = {}
 
         sanitized = VLLMGenerateConfig()
+
+        response_format = generate_config.pop("response_format", None)
+        guided_decoding_backend = generate_config.get("guided_decoding_backend", None)
+        guided_json_object = None
+        guided_json = None
+
+        if response_format is not None:
+            if response_format.get("type") == "json_object":
+                guided_json_object = True
+            elif response_format.get("type") == "json_schema":
+                json_schema = response_format.get("json_schema")
+                assert json_schema is not None
+                guided_json = json_schema.get("json_schema")
+                if guided_decoding_backend is None:
+                    guided_decoding_backend = "outlines"
+
         sanitized.setdefault("lora_name", generate_config.get("lora_name", None))
         sanitized.setdefault("n", generate_config.get("n", 1))
         sanitized.setdefault("best_of", generate_config.get("best_of", None))
@@ -346,6 +410,31 @@ class VLLMModel(LLM):
         sanitized.setdefault(
             "stream_options", generate_config.get("stream_options", None)
         )
+        sanitized.setdefault(
+            "skip_special_tokens", generate_config.get("skip_special_tokens", True)
+        )
+        sanitized.setdefault(
+            "guided_json", generate_config.get("guided_json", guided_json)
+        )
+        sanitized.setdefault("guided_regex", generate_config.get("guided_regex", None))
+        sanitized.setdefault(
+            "guided_choice", generate_config.get("guided_choice", None)
+        )
+        sanitized.setdefault(
+            "guided_grammar", generate_config.get("guided_grammar", None)
+        )
+        sanitized.setdefault(
+            "guided_whitespace_pattern",
+            generate_config.get("guided_whitespace_pattern", None),
+        )
+        sanitized.setdefault(
+            "guided_json_object",
+            generate_config.get("guided_json_object", guided_json_object),
+        )
+        sanitized.setdefault(
+            "guided_decoding_backend",
+            generate_config.get("guided_decoding_backend", guided_decoding_backend),
+        )
 
         return sanitized
 
@@ -483,13 +572,46 @@ class VLLMModel(LLM):
             if isinstance(stream_options, dict)
             else False
         )
-        sampling_params = SamplingParams(**sanitized_generate_config)
+
+        if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
+            # guided decoding only available for vllm >= 0.6.3
+            from vllm.sampling_params import GuidedDecodingParams
+
+            guided_options = GuidedDecodingParams.from_optional(
+                json=sanitized_generate_config.pop("guided_json", None),
+                regex=sanitized_generate_config.pop("guided_regex", None),
+                choice=sanitized_generate_config.pop("guided_choice", None),
+                grammar=sanitized_generate_config.pop("guided_grammar", None),
+                json_object=sanitized_generate_config.pop("guided_json_object", None),
+                backend=sanitized_generate_config.pop("guided_decoding_backend", None),
+                whitespace_pattern=sanitized_generate_config.pop(
+                    "guided_whitespace_pattern", None
+                ),
+            )
+
+            sampling_params = SamplingParams(
+                guided_decoding=guided_options, **sanitized_generate_config
+            )
+        else:
+            # ignore generate configs
+            sanitized_generate_config.pop("guided_json", None)
+            sanitized_generate_config.pop("guided_regex", None)
+            sanitized_generate_config.pop("guided_choice", None)
+            sanitized_generate_config.pop("guided_grammar", None)
+            sanitized_generate_config.pop("guided_json_object", None)
+            sanitized_generate_config.pop("guided_decoding_backend", None)
+            sanitized_generate_config.pop("guided_whitespace_pattern", None)
+            sampling_params = SamplingParams(**sanitized_generate_config)
+
         if not request_id:
             request_id = str(uuid.uuid1())
 
         assert self._engine is not None
         results_generator = self._engine.generate(
-            prompt, sampling_params, request_id, lora_request=lora_request
+            prompt,
+            sampling_params,
+            request_id,
+            lora_request,
         )
 
         async def stream_results() -> AsyncGenerator[CompletionChunk, None]:
@@ -772,6 +894,9 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
                 "image": 2,  # default 2 images all chat
             }
         )
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
 
         return model_config
 

xinference/model/llm/vllm/xavier/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

xinference/model/llm/vllm/xavier/allocator.py

@@ -0,0 +1,74 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict, Optional
+
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
+from vllm.core.block.interfaces import DeviceAwareBlockAllocator
+from vllm.platforms import current_platform
+from vllm.utils import Device
+
+from .block import XavierPrefixCachingBlockAllocator
+
+
+class XavierCpuGpuBlockAllocator(CpuGpuBlockAllocator):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._xavier_config: Optional[Dict[str, Any]] = None
+
+    @property
+    def xavier_config(self):
+        return self._xavier_config
+
+    @xavier_config.setter
+    def xavier_config(self, v: Dict[str, Any]):
+        self._xavier_config = v
+        self._allocators[Device.GPU].xavier_config = v
+
+    @staticmethod
+    def create(
+        allocator_type: str,
+        num_gpu_blocks: int,
+        num_cpu_blocks: int,
+        block_size: int,
+    ) -> DeviceAwareBlockAllocator:
+        """Xinference Change!!!
+        1. The code is copied here because the `allocator` needs to be instantiated as a subclass.
+        2. Why not re-instantiate it externally?
+        Re-instantiating the `allocator` is costly because it requires initializing many tensors.
+        """
+
+        # For HPU, block id 0 is used only for padding
+        reserved_blocks = 1 if current_platform.is_hpu() else 0
+        block_ids = list(range(reserved_blocks, num_gpu_blocks + num_cpu_blocks))
+        num_gpu_blocks -= reserved_blocks
+        gpu_block_ids = block_ids[:num_gpu_blocks]
+        cpu_block_ids = block_ids[num_gpu_blocks:]
+
+        gpu_allocator = XavierPrefixCachingBlockAllocator(
+            run_isolation=True,
+            num_blocks=num_gpu_blocks,
+            block_size=block_size,
+            block_ids=gpu_block_ids,
+        )
+
+        cpu_allocator = XavierPrefixCachingBlockAllocator(
+            num_blocks=num_cpu_blocks,
+            block_size=block_size,
+            block_ids=cpu_block_ids,
+        )
+
+        return XavierCpuGpuBlockAllocator(
+            cpu_block_allocator=cpu_allocator,
+            gpu_block_allocator=gpu_allocator,
+        )

xinference/model/llm/vllm/xavier/block.py

@@ -0,0 +1,111 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+import xoscar as xo
+from vllm.core.block.interfaces import BlockId
+from vllm.core.block.prefix_caching_block import (
+    BlockTracker,
+    PrefixCachingBlockAllocator,
+)
+
+from .....isolation import Isolation
+
+logger = logging.getLogger(__name__)
+
+
+class XavierInnerBlockTracker(BlockTracker):
+    """Used to track the status of a block inside the prefix caching allocator"""
+
+    """
+    Here, two fixed attributes, `transferred` and `executed`,
+    have been added to the `BlockTracker` class to mark the status of the corresponding `block_id`.
+    We cannot directly set attributes on the `Block` object
+    because the `Block` objects are dynamically allocated with each scheduling.
+    The `Block` objects executed in two different scheduling steps may have the same `id`, `hash`, etc.,
+    but the instance objects may differ.
+    The BlockTracker object inside vllm is one-to-one with the block_id.
+    """
+    __slots__ = ("active", "last_accessed", "computed", "transferred", "executed")
+
+    def __init__(self):
+        super().__init__()
+        self.transferred = False
+        self.executed = False
+
+
+class XavierPrefixCachingBlockAllocator(PrefixCachingBlockAllocator):
+    def __init__(self, *args, run_isolation: bool = False, **kwargs):
+        super().__init__(*args, **kwargs)
+        for _id in self._block_tracker.keys():
+            self._block_tracker[_id] = XavierInnerBlockTracker()
+
+        self._xavier_config: Optional[Dict[str, Any]] = None
+        self._block_tracker_ref = None
+        if run_isolation:
+            self._isolation = Isolation(
+                asyncio.new_event_loop(), threaded=True, daemon=True
+            )
+            self._isolation.start()
+        else:
+            self._isolation = None  # type: ignore
+
+    def __del__(self):
+        if self._isolation is not None:
+            self._isolation.stop()
+
+    @property
+    def xavier_config(self):
+        return self._xavier_config
+
+    @xavier_config.setter
+    def xavier_config(self, v: Dict[str, Any]):
+        self._xavier_config = v
+
+    async def _get_block_tracker_ref(self):
+        if self._block_tracker_ref is None:
+            block_tracker_address = self.xavier_config.get("block_tracker_address")
+            block_tracker_uid = self.xavier_config.get("block_tracker_uid")
+            self._block_tracker_ref = await xo.actor_ref(
+                address=block_tracker_address, uid=block_tracker_uid
+            )
+        return self._block_tracker_ref
+
+    async def unregister_block(self, block_id: int):
+        assert self._xavier_config is not None
+        tracker_ref = await self._get_block_tracker_ref()
+        await tracker_ref.unregister_block(
+            self.xavier_config.get("virtual_engine"),
+            self.xavier_config.get("rank"),
+            block_id,
+        )
+
+    def _maybe_allocate_evicted_block_id(self) -> Optional[BlockId]:
+        """
+        This is the only entry point where the `block_id` is evicted from the cache.
+        Therefore, when the `block_id` is evicted, the tracker actor needs to unregister the block information.
+        At the same time, make sure to reset the attributes corresponding to that `block_id`.
+        """
+        evicted_block_id = super()._maybe_allocate_evicted_block_id()
+        logger.debug(f"block_id: {evicted_block_id} will be evicted from the cache.")
+        if evicted_block_id is not None and self._isolation is not None:
+            tracker = self._block_tracker[evicted_block_id]
+            assert isinstance(tracker, XavierInnerBlockTracker)
+            tracker.transferred = False
+            tracker.executed = False
+            self._isolation.call(self.unregister_block(evicted_block_id))
+            logger.debug(f"block_id: {evicted_block_id} will be used again.")
+        return evicted_block_id

xinference/model/llm/vllm/xavier/block_manager.py

@@ -0,0 +1,71 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import Any, Dict, Optional
+
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
+from vllm.core.block.interfaces import Block
+from vllm.core.block_manager import SelfAttnBlockSpaceManager
+from vllm.sequence import SequenceGroup, SequenceStatus
+from vllm.utils import Device
+
+from .allocator import XavierCpuGpuBlockAllocator
+
+logger = logging.getLogger(__name__)
+
+
+class XavierBlockManager(SelfAttnBlockSpaceManager):
+    def __init__(self, *args, **kwargs):
+        # Monkey patch
+        CpuGpuBlockAllocator.create = XavierCpuGpuBlockAllocator.create
+        super().__init__(*args, **kwargs)
+        self._xavier_config: Optional[Dict[str, Any]] = None
+        logger.debug("Init xavier block manager done.")
+
+    @property
+    def xavier_config(self):
+        return self._xavier_config
+
+    @xavier_config.setter
+    def xavier_config(self, value: Dict[str, Any]):
+        self._xavier_config = value
+        self.block_allocator.xavier_config = value
+
+    def get_block_by_block_id(self, seq_id: int, block_id: int) -> Block:
+        table = self.block_tables[seq_id]
+        for b in table.blocks:
+            if b.block_id == block_id:
+                return b
+
+    def get_block_status_by_block_id(self, status_name: str, block_id: int) -> bool:
+        tracker = self.block_allocator._allocators[Device.GPU]._block_tracker[block_id]
+        return getattr(tracker, status_name)
+
+    def set_block_status_by_block_id(
+        self, status_name: str, block_id: int, status: bool
+    ) -> None:
+        tracker = self.block_allocator._allocators[Device.GPU]._block_tracker[block_id]
+        assert getattr(tracker, status_name, None) is not None
+        setattr(tracker, status_name, status)
+
+    def allocate(self, seq_group: SequenceGroup) -> None:
+        """
+        If the `seq_group` has the `transferred` attribute,
+        it indicates that the `seq_group` has gone through the transfer process,
+        so the block allocation logic should not be executed again.
+        """
+        waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
+        if all([getattr(s, "transferred", False) for s in waiting_seqs]):
+            return
+        super().allocate(seq_group)