xinference 1.0.1__py3-none-any.whl → 1.2.1__py3-none-any.whl

xinference/model/llm/transformers/cogagent.py

@@ -0,0 +1,272 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import re
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import Dict, Iterator, List, Literal, Optional, Union
+
+import torch
+
+from ....model.utils import select_device
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    CogagentGenerateConfig,
+    CompletionChunk,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import (
+    _decode_image,
+    generate_chat_completion,
+    generate_completion_chunk,
+    parse_messages,
+)
+from .core import PytorchChatModel
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class CogAgentChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._torch_type = None
+        self._device = None
+        self._tokenizer = None
+        self._model = None
+        self._platform: Literal["Mac", "WIN", "Mobile"] | None = "Mac"
+        self._format: Literal[
+            "(Answer in Action-Operation-Sensitive format.)",
+            "(Answer in Status-Plan-Action-Operation format.)",
+            "(Answer in Status-Action-Operation-Sensitive format.)",
+            "(Answer in Status-Action-Operation format.)",
+            "(Answer in Action-Operation format.)",
+        ] | None = "(Answer in Action-Operation-Sensitive format.)"
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        family = model_family.model_family or model_family.model_name
+        if "cogagent" in family.lower():
+            return True
+        return False
+
+    def load(self, **kwargs):
+        from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+        device = self._pytorch_model_config.get("device", "auto")
+        self._device = select_device(device)
+
+        self._tokenizer = AutoTokenizer.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        if self.quantization == "4-bit":
+            quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+        elif self.quantization == "8-bit":
+            quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+        else:
+            quantization_config = None
+
+        self._model = AutoModelForCausalLM.from_pretrained(
+            self.model_path,
+            torch_dtype=torch.bfloat16,
+            trust_remote_code=True,
+            device_map=self._device,
+            quantization_config=quantization_config,
+        ).eval()
+
+    def _message_content_to_cogagent(self, content):
+        assert isinstance(content, list)
+        texts = []
+        image_urls = []
+        for c in content:
+            c_type = c.get("type")
+            if c_type == "text":
+                texts.append(c["text"])
+            elif c_type == "image_url":
+                image_urls.append(c["image_url"]["url"])
+        image_futures = []
+        with ThreadPoolExecutor() as executor:
+            for image_url in image_urls:
+                fut = executor.submit(_decode_image, image_url)
+                image_futures.append(fut)
+        images = [fut.result() for fut in image_futures]
+        text = " ".join(texts)
+        if len(images) == 0:
+            raise RuntimeError(
+                "CogAgent requires image input to perform GUI Agent tasks. Pure text-based interaction cannot execute such tasks."
+            )
+        elif len(images) == 1:
+            return text, images[-1]
+        else:
+            logger.warning(
+                "There are multiple images in the prompt, CogAgent will automatically use the most recently provided image as the input."
+            )
+            return text, images[-1]
+
+    def _history_content_to_cogagent(self, chat_history: List[Dict]):
+        grounded_pattern = r"Grounded Operation:\s*(.*)"
+        action_pattern = r"Action:\s*(.*)"
+
+        def extract_operations(_content: str):
+            """extract grounded operation and action operation"""
+            _history_step = []
+            _history_action = []
+
+            matches_history = re.search(grounded_pattern, _content)
+            matches_actions = re.search(action_pattern, _content)
+
+            if matches_history:
+                grounded_operation = matches_history.group(1)
+                _history_step.append(grounded_operation)
+            if matches_actions:
+                action_operation = matches_actions.group(1)
+                _history_action.append(action_operation)
+
+            return _history_step, _history_action
+
+        history_step = []
+        history_action = []
+
+        for i in range(0, len(chat_history) - 1, 2):
+            content = chat_history[i + 1].get("content")
+            if isinstance(content, str):  # 如果内容是字符串
+                steps, actions = extract_operations(content)
+                history_step.extend(steps)
+                history_action.extend(actions)
+
+            elif isinstance(content, list):  # 如果内容是列表
+                for c in content:
+                    c_content = c.get("content")
+                    if isinstance(c_content, str):  # 确保是字符串类型
+                        steps, actions = extract_operations(c_content)
+                        history_step.extend(steps)
+                        history_action.extend(actions)
+
+        return history_step, history_action
+
+    def get_query_and_history(
+        self,
+        prompt: Union[str, List[Dict]],
+        chat_history: Optional[List[Dict]] = None,
+    ):
+        task, image = self._message_content_to_cogagent(prompt)
+
+        history_step, history_action = [], []
+
+        if chat_history:
+            history_step, history_action = self._history_content_to_cogagent(
+                chat_history
+            )
+
+        # Verify history lengths match
+        if len(history_step) != len(history_action):
+            raise ValueError("Mismatch in lengths of history_step and history_action.")
+
+        # Format history steps for output
+        history_str = "\nHistory steps: "
+        for index, (step, action) in enumerate(zip(history_step, history_action)):
+            history_str += f"\n{index}. {step}\t{action}"
+
+        # Compose the query with task, platform, and selected format instructions
+        query = f"Task: {task}{history_str}\n{self._platform}{self._format}"
+        logger.info(f"query:{query}")
+        return query, image
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[CogagentGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        if generate_config is not None:
+            self._platform = generate_config.pop("platform", self._platform)
+            self._format = generate_config.pop("format", self._format)
+
+        sanitize_generate_config = self._sanitize_generate_config(generate_config)
+        stream = sanitize_generate_config.get("stream")
+        sanitized_config = {
+            "max_length": sanitize_generate_config.get("max_tokens", 512),
+            "top_k": sanitize_generate_config.get("top_k", 1),
+            "do_sample": True,
+        }
+        prompt, _, chat_history = parse_messages(messages)
+
+        query, image = self.get_query_and_history(prompt, chat_history)
+
+        full_context_kwargs = {
+            "return_tensors": "pt",
+            "return_dict": True,
+        }
+        assert self.model_family.chat_template is not None
+        inputs = self.get_full_context(
+            [{"role": "user", "image": image, "content": query}],
+            self.model_family.chat_template,
+            self._tokenizer,
+            tokenize=True,
+            **full_context_kwargs,
+        )
+        inputs.to(self._model.device)
+
+        if stream:
+            it = self._streaming_chat_response(inputs, sanitized_config)
+            return self._to_chat_completion_chunks(it)
+        else:
+            # Generate response
+            with torch.no_grad():
+                outputs = self._model.generate(**inputs, **sanitized_config)
+                outputs = outputs[:, inputs["input_ids"].shape[1] :]
+                response = self._tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+            return generate_chat_completion(self.model_uid, response)
+
+    def _streaming_chat_response(
+        self, inputs: Dict, config: Dict
+    ) -> Iterator[CompletionChunk]:
+        from threading import Thread
+
+        from transformers import TextIteratorStreamer
+
+        streamer = TextIteratorStreamer(
+            self._tokenizer, skip_prompt=True, skip_special_tokens=True
+        )
+        generation_kwargs = {**inputs, **config}
+
+        thread = Thread(target=self._model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=-1,
+                completion_tokens=-1,
+                total_tokens=-1,
+            )
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )

xinference/model/llm/transformers/core.py

@@ -69,6 +69,8 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "deepseek-v2.5",
     "deepseek-v2-chat-0628",
     "glm-edge-v",
+    "QvQ-72B-Preview",
+    "cogagent",
 ]
 
 

xinference/model/llm/transformers/qwen2_vl.py

@@ -17,6 +17,7 @@ import sys
 import uuid
 from typing import Iterator, List, Optional, Union
 
+from ....device_utils import is_npu_available
 from ....model.utils import select_device
 from ....types import (
     ChatCompletion,
@@ -47,6 +48,8 @@ class Qwen2VLChatModel(PytorchChatModel):
         llm_family = model_family.model_family or model_family.model_name
         if "qwen2-vl-instruct".lower() in llm_family.lower():
             return True
+        if "qvq-72b-preview".lower() in llm_family.lower():
+            return True
         return False
 
     def load(self):
@@ -71,6 +74,14 @@ class Qwen2VLChatModel(PytorchChatModel):
                 attn_implementation="flash_attention_2",
                 trust_remote_code=True,
             ).eval()
+        elif is_npu_available():
+            # Ascend do not support bf16
+            self._model = Qwen2VLForConditionalGeneration.from_pretrained(
+                self.model_path,
+                device_map="auto",
+                trust_remote_code=True,
+                torch_dtype="float16",
+            ).eval()
         else:
             self._model = Qwen2VLForConditionalGeneration.from_pretrained(
                 self.model_path, device_map=device, trust_remote_code=True
@@ -112,7 +123,7 @@ class Qwen2VLChatModel(PytorchChatModel):
             padding=True,
             return_tensors="pt",
         )
-        inputs = inputs.to("cuda")
+        inputs = inputs.to(self._device)
 
         # Inference: Generation of the output
         generated_ids = self._model.generate(

xinference/model/llm/transformers/utils.py

@@ -156,6 +156,7 @@ def _get_completion(
     finish_reason: Optional[str],
     model_uid: str,
     r: InferenceRequest,
+    completion_tokens: int,
 ):
     completion_choice = CompletionChoice(
         text=output, index=0, logprobs=None, finish_reason=finish_reason
@@ -170,8 +171,8 @@ def _get_completion(
     )
     completion_usage = CompletionUsage(
         prompt_tokens=len(r.prompt_tokens),
-        completion_tokens=len(r.new_tokens),
-        total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
+        completion_tokens=completion_tokens,
+        total_tokens=len(r.prompt_tokens) + completion_tokens,
     )
     completion = Completion(
         id=completion_chunk["id"],
@@ -371,7 +372,7 @@ def _batch_inference_one_step_internal(
                 r.stopped = stopped
                 r.finish_reason = finish_reason
 
-            if r.stopped and r not in stop_token_mapping and r not in output_mapping:
+            if r.stopped and r not in stop_token_mapping:
                 stop_token_mapping[r] = _i + 1
 
             if r.stream:
@@ -446,12 +447,14 @@ def _batch_inference_one_step_internal(
             else:
                 # last round, handle non-stream result
                 if r.stopped and _i == decode_round - 1:
-                    invalid_token_num = decode_round - stop_token_mapping[r]
+                    invalid_token_num = (
+                        (decode_round - stop_token_mapping[r] + 1)
+                        if r.finish_reason == "stop"
+                        else (decode_round - stop_token_mapping[r])
+                    )
                     outputs = (
                         tokenizer.decode(
-                            r.new_tokens[: -(invalid_token_num + 1)]
-                            if r.finish_reason == "stop"
-                            else r.new_tokens[:-invalid_token_num],
+                            r.new_tokens[:-invalid_token_num],
                             skip_special_tokens=True,
                             spaces_between_special_tokens=False,
                             clean_up_tokenization_spaces=True,
@@ -460,7 +463,12 @@ def _batch_inference_one_step_internal(
                         else output_mapping[r]
                     )
                     completion = _get_completion(
-                        outputs, r.chunk_id, r.finish_reason, model_uid, r
+                        outputs,
+                        r.chunk_id,
+                        r.finish_reason,
+                        model_uid,
+                        r,
+                        len(r.new_tokens) - invalid_token_num,
                     )
                     r.completion = [completion]
 

xinference/model/llm/utils.py

@@ -52,6 +52,7 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen2-instruct",
     "qwen2-moe-instruct",
     "qwen2.5-instruct",
+    "qwen2.5-coder-instruct",
 ]
 
 GLM4_TOOL_CALL_FAMILY = [
@@ -96,13 +97,22 @@ class ChatModelMixin:
         return rendered
 
     def get_full_context(
-        self, messages: List, chat_template: str, tokenizer=None, **kwargs
-    ) -> str:
+        self,
+        messages: List,
+        chat_template: str,
+        tokenizer=None,
+        tokenize=False,
+        **kwargs,
+    ):
+        if "vision" not in self.model_family.model_ability:  # type: ignore
+            messages = self.convert_messages_with_content_list_to_str_conversion(
+                messages
+            )
         if tokenizer is not None:
             try:
                 full_context = tokenizer.apply_chat_template(
                     messages,
-                    tokenize=False,
+                    tokenize=tokenize,
                     chat_template=chat_template,
                     add_generation_prompt=True,
                     **kwargs,
@@ -118,6 +128,25 @@ class ChatModelMixin:
             # Compilation function uses a cache to avoid recompiling the same template
             return self._build_from_raw_template(messages, chat_template, **kwargs)
 
+    @staticmethod
+    def convert_messages_with_content_list_to_str_conversion(
+        messages: List[Dict],
+    ) -> List[Dict]:
+        """
+        Handles messages with content list conversion, in order to support Cline, see GH#2659 .
+        """
+        for message in messages:
+            texts = ""
+            msg_content = message.get("content")
+            if msg_content:
+                if isinstance(msg_content, str):
+                    texts = msg_content
+                elif isinstance(msg_content, list):
+                    texts = "\n".join(item.get("text", "") for item in msg_content)
+            if texts:
+                message["content"] = texts
+        return messages
+
     @staticmethod
     def get_specific_prompt(model_family: str, messages: List[ChatCompletionMessage]):
         """
@@ -324,7 +353,10 @@ class ChatModelMixin:
         """
         try:
             if isinstance(c, dict):
-                return [(None, c["name"], c["arguments"])]
+                try:
+                    return [(None, c["name"], json.loads(c["arguments"]))]
+                except Exception:
+                    return [(None, c["name"], c["arguments"])]
         except KeyError:
             logger.error("Can't parse glm output: %s", c)
             return [(str(c), None, None)]

xinference/model/llm/vllm/core.py

@@ -70,6 +70,7 @@ class VLLMModelConfig(TypedDict, total=False):
     max_model_len: Optional[int]
     limit_mm_per_prompt: Optional[Dict[str, int]]
     guided_decoding_backend: Optional[str]
+    scheduling_policy: Optional[str]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -86,6 +87,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     stream: bool  # non-sampling param, should not be passed to the engine.
     stream_options: Optional[Union[dict, None]]
+    skip_special_tokens: Optional[bool]
     response_format: Optional[dict]
     guided_json: Optional[Union[str, dict]]
     guided_regex: Optional[str]
@@ -154,7 +156,8 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_MODELS.append("qwen2.5-coder")
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-coder-instruct")
     VLLM_SUPPORTED_CHAT_MODELS.append("QwQ-32B-Preview")
-
+    VLLM_SUPPORTED_CHAT_MODELS.append("marco-o1")
+    VLLM_SUPPORTED_CHAT_MODELS.append("deepseek-r1-distill-qwen")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
     VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
@@ -181,14 +184,19 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.5.3":
 if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
+    VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.3-instruct")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.2":
+    VLLM_SUPPORTED_CHAT_MODELS.append("minicpm3-4b")
+
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.2-vision")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("llama-3.2-vision-instruct")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("QvQ-72B-Preview")
 
 
 class VLLMModel(LLM):
@@ -217,6 +225,10 @@ class VLLMModel(LLM):
         self._engine = None
         self.lora_modules = peft_model
         self.lora_requests: List[LoRARequest] = []
+        self._xavier_config = None
+
+    def set_xavier_config(self, value: Optional[Dict]):
+        self._xavier_config = value  # type: ignore
 
     def load(self):
         try:
@@ -242,7 +254,6 @@ class VLLMModel(LLM):
             multiprocessing.set_start_method("fork", force=True)
 
         self._model_config = self._sanitize_model_config(self._model_config)
-
         if self.lora_modules is None:
             self.lora_requests = []
         else:
@@ -263,13 +274,34 @@ class VLLMModel(LLM):
             f"Enable lora: {enable_lora}. Lora count: {max_loras}."
         )
 
-        engine_args = AsyncEngineArgs(
-            model=self.model_path,
-            enable_lora=enable_lora,
-            max_loras=max_loras,
-            **self._model_config,
-        )
-        self._engine = AsyncLLMEngine.from_engine_args(engine_args)
+        if self._xavier_config is not None:
+            from .xavier.engine import XavierEngine
+
+            # Enabling Xavier means that `enable_prefix_caching` is enabled by default.
+            self._model_config.setdefault("enable_prefix_caching", True)
+            xavier_transfer_block_num = self._model_config.pop(
+                "xavier_transfer_block_num", 512
+            )
+            self._xavier_config["transfer_block_num"] = xavier_transfer_block_num
+            engine_args = AsyncEngineArgs(
+                model=self.model_path,
+                enable_lora=enable_lora,
+                max_loras=max_loras,
+                **self._model_config,
+            )
+
+            logger.debug(f"Start xavier for vllm with config: {self._xavier_config}")
+            self._engine = XavierEngine.from_engine_args(
+                engine_args, xavier_config=self._xavier_config
+            )
+        else:
+            engine_args = AsyncEngineArgs(
+                model=self.model_path,
+                enable_lora=enable_lora,
+                max_loras=max_loras,
+                **self._model_config,
+            )
+            self._engine = AsyncLLMEngine.from_engine_args(engine_args)
 
         self._check_health_task = None
         if hasattr(self._engine, "check_health"):
@@ -287,6 +319,9 @@ class VLLMModel(LLM):
             model_executor.shutdown()
         self._engine = None
 
+    async def init_xavier(self):
+        await self._engine.init_xavier()
+
     async def _check_healthy(self, interval: int = 30):
         from vllm.engine.async_llm_engine import AsyncEngineDeadError
 
@@ -325,7 +360,9 @@ class VLLMModel(LLM):
         model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
         model_config.setdefault("guided_decoding_backend", "outlines")
-
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
         return model_config
 
     @staticmethod
@@ -373,6 +410,9 @@ class VLLMModel(LLM):
         sanitized.setdefault(
             "stream_options", generate_config.get("stream_options", None)
         )
+        sanitized.setdefault(
+            "skip_special_tokens", generate_config.get("skip_special_tokens", True)
+        )
         sanitized.setdefault(
             "guided_json", generate_config.get("guided_json", guided_json)
         )
@@ -854,6 +894,9 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
                 "image": 2,  # default 2 images all chat
             }
         )
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
 
         return model_config
 

xinference/model/llm/vllm/xavier/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

xinference/model/llm/vllm/xavier/allocator.py

@@ -0,0 +1,74 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict, Optional
+
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
+from vllm.core.block.interfaces import DeviceAwareBlockAllocator
+from vllm.platforms import current_platform
+from vllm.utils import Device
+
+from .block import XavierPrefixCachingBlockAllocator
+
+
+class XavierCpuGpuBlockAllocator(CpuGpuBlockAllocator):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._xavier_config: Optional[Dict[str, Any]] = None
+
+    @property
+    def xavier_config(self):
+        return self._xavier_config
+
+    @xavier_config.setter
+    def xavier_config(self, v: Dict[str, Any]):
+        self._xavier_config = v
+        self._allocators[Device.GPU].xavier_config = v
+
+    @staticmethod
+    def create(
+        allocator_type: str,
+        num_gpu_blocks: int,
+        num_cpu_blocks: int,
+        block_size: int,
+    ) -> DeviceAwareBlockAllocator:
+        """Xinference Change!!!
+        1. The code is copied here because the `allocator` needs to be instantiated as a subclass.
+        2. Why not re-instantiate it externally?
+        Re-instantiating the `allocator` is costly because it requires initializing many tensors.
+        """
+
+        # For HPU, block id 0 is used only for padding
+        reserved_blocks = 1 if current_platform.is_hpu() else 0
+        block_ids = list(range(reserved_blocks, num_gpu_blocks + num_cpu_blocks))
+        num_gpu_blocks -= reserved_blocks
+        gpu_block_ids = block_ids[:num_gpu_blocks]
+        cpu_block_ids = block_ids[num_gpu_blocks:]
+
+        gpu_allocator = XavierPrefixCachingBlockAllocator(
+            run_isolation=True,
+            num_blocks=num_gpu_blocks,
+            block_size=block_size,
+            block_ids=gpu_block_ids,
+        )
+
+        cpu_allocator = XavierPrefixCachingBlockAllocator(
+            num_blocks=num_cpu_blocks,
+            block_size=block_size,
+            block_ids=cpu_block_ids,
+        )
+
+        return XavierCpuGpuBlockAllocator(
+            cpu_block_allocator=cpu_allocator,
+            gpu_block_allocator=gpu_allocator,
+        )