xinference 1.7.1.post1__py3-none-any.whl → 1.8.0__py3-none-any.whl

xinference/model/llm/lmdeploy/core.py

@@ -20,7 +20,7 @@ import torch
 
 from ....types import ChatCompletion, ChatCompletionChunk, Completion, LoRA
 from ..core import LLM
-from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..llm_family import LLMFamilyV2, LLMSpecV1
 from ..utils import ChatModelMixin, generate_chat_completion, generate_completion_chunk
 
 logger = logging.getLogger(__name__)
@@ -76,14 +76,12 @@ class LMDeployModel(LLM):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         model_config: Optional[LMDeployModelConfig] = None,
         peft_model: Optional[List[LoRA]] = None,
     ):
-        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        super().__init__(model_uid, model_family, model_path)
         self._model_config: LMDeployModelConfig = self._sanitize_model_config(
             model_config
         )
@@ -119,7 +117,7 @@ class LMDeployModel(LLM):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         return False
 
@@ -172,7 +170,7 @@ class LMDeployChatModel(LMDeployModel, ChatModelMixin):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format == "awq":
             # Currently, only 4-bit weight quantization is supported for AWQ, but got 8 bits.

xinference/model/llm/memory.py

@@ -214,16 +214,15 @@ def get_model_layers_info(
         logger.debug("get_model_layers_info by default size=%s", model_size_in_billions)
         size_in_billions = convert_model_size_to_float(model_size_in_billions)
         return _get_default_layers_from_size(size_in_billions)
-    match_result = match_llm(
+    llm_family = match_llm(
         model_name=model_name,
         model_format=model_format,
         model_size_in_billions=model_size_in_billions,
         quantization=quantization,
     )
-    if not match_result:
+    if not llm_family:
         return None
-    llm_family, llm_spec, _quant = match_result
-    config_path = cache_model_config(llm_family, llm_spec)
+    config_path = cache_model_config(llm_family)
     return load_model_config_json(config_path)
 
 

xinference/model/llm/mlx/core.py

@@ -48,7 +48,7 @@ from ....types import (
     LoRA,
 )
 from ..core import LLM, chat_context_var
-from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..llm_family import LLMFamilyV2, LLMSpecV1
 from ..utils import (
     DEEPSEEK_TOOL_CALL_FAMILY,
     QWEN_TOOL_CALL_FAMILY,
@@ -98,14 +98,12 @@ class MLXModel(LLM):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         model_config: Optional[MLXModelConfig] = None,
         peft_model: Optional[List[LoRA]] = None,
     ):
-        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        super().__init__(model_uid, model_family, model_path)
         self._use_fast_tokenizer = True
         self._model_config: MLXModelConfig = self._sanitize_model_config(model_config)
         # for distributed
@@ -370,7 +368,7 @@ class MLXModel(LLM):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["mlx"]:
             return False
@@ -670,7 +668,7 @@ class MLXChatModel(MLXModel, ChatModelMixin):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["mlx"]:
             return False
@@ -734,7 +732,7 @@ class MLXVisionModel(MLXModel, ChatModelMixin):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["mlx"]:
             return False

xinference/model/llm/reasoning_parser.py

@@ -165,7 +165,9 @@ class ReasoningParser:
         Returns:
             bool: True if reasoning content should be extracted, False otherwise
         """
-        return self.reasoning_content
+        if self.is_enable_thinking():
+            return self.reasoning_content
+        return False
 
     def _create_chat_completion_chunk(
         self, chunk: Union[Dict[str, Any], CompletionChunk], content: str

xinference/model/llm/sglang/core.py

@@ -32,10 +32,15 @@ from ....types import (
     CompletionChunk,
     CompletionUsage,
 )
-from .. import LLM, LLMFamilyV1, LLMSpecV1
+from .. import LLM, LLMFamilyV2, LLMSpecV1
 from ..core import chat_context_var
-from ..llm_family import CustomLLMFamilyV1
-from ..utils import ChatModelMixin, generate_completion_chunk
+from ..llm_family import CustomLLMFamilyV2
+from ..utils import (
+    DEEPSEEK_TOOL_CALL_FAMILY,
+    QWEN_TOOL_CALL_FAMILY,
+    ChatModelMixin,
+    generate_completion_chunk,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -131,13 +136,11 @@ class SGLANGModel(LLM):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         model_config: Optional[SGLANGModelConfig],
     ):
-        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        super().__init__(model_uid, model_family, model_path)
         self._model_config = model_config
         self._engine = None
         self._address = model_config.pop("address", None)  # type: ignore
@@ -319,7 +322,7 @@ class SGLANGModel(LLM):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if not cls._has_cuda_device():
             return False
@@ -330,7 +333,7 @@ class SGLANGModel(LLM):
         if llm_spec.model_format == "pytorch":
             if quantization != "none" and not (quantization is None):
                 return False
-        if isinstance(llm_family, CustomLLMFamilyV1):
+        if isinstance(llm_family, CustomLLMFamilyV2):
             if llm_family.model_family not in SGLANG_SUPPORTED_MODELS:
                 return False
         else:
@@ -547,14 +550,14 @@ class SGLANGModel(LLM):
 class SGLANGChatModel(SGLANGModel, ChatModelMixin):
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["pytorch", "gptq", "awq", "fp8"]:
             return False
         if llm_spec.model_format == "pytorch":
             if quantization != "none" and not (quantization is None):
                 return False
-        if isinstance(llm_family, CustomLLMFamilyV1):
+        if isinstance(llm_family, CustomLLMFamilyV2):
             if llm_family.model_family not in SGLANG_SUPPORTED_CHAT_MODELS:
                 return False
         else:
@@ -583,6 +586,9 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
         request_id: Optional[str] = None,
     ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
         assert self.model_family.chat_template is not None
+        # fix: Object of type list_iterator is not JSON serializable
+        tools = list(generate_config.pop("tools", [])) if generate_config else None
+        model_family = self.model_family.model_family or self.model_family.model_name
         chat_template_kwargs = (
             self._get_chat_template_kwargs_from_generate_config(
                 generate_config, self.reasoning_parser
@@ -591,6 +597,12 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
         )
         chat_context_var.set(chat_template_kwargs)
         full_context_kwargs = chat_template_kwargs.copy()
+        if tools:
+            if (
+                model_family in QWEN_TOOL_CALL_FAMILY
+                or model_family in DEEPSEEK_TOOL_CALL_FAMILY
+            ):
+                full_context_kwargs["tools"] = tools
         full_prompt = self.get_full_context(
             messages, self.model_family.chat_template, **full_context_kwargs
         )
@@ -599,17 +611,23 @@ class SGLANGChatModel(SGLANGModel, ChatModelMixin):
         if stream:
             agen = await self.async_generate(full_prompt, generate_config=generate_config)  # type: ignore
             assert isinstance(agen, AsyncGenerator)
-            return self._async_to_chat_completion_chunks(agen, self.reasoning_parser)
+            return self._async_to_chat_completion_chunks(
+                agen, self.reasoning_parser, chat_template_kwargs
+            )
         else:
             c = await self.async_generate(full_prompt, generate_config=generate_config)  # type: ignore
             assert not isinstance(c, AsyncGenerator)
+            if tools:
+                return self._post_process_completion(
+                    self.model_family, self.model_uid, c, self.reasoning_parser
+                )
             return self._to_chat_completion(c, self.reasoning_parser)
 
 
 class SGLANGVisionModel(SGLANGModel, ChatModelMixin):
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if not cls._has_cuda_device():
             return False
@@ -620,7 +638,7 @@ class SGLANGVisionModel(SGLANGModel, ChatModelMixin):
         if llm_spec.model_format == "pytorch":
             if quantization != "none" and not (quantization is None):
                 return False
-        if isinstance(llm_family, CustomLLMFamilyV1):
+        if isinstance(llm_family, CustomLLMFamilyV2):
             if llm_family.model_family not in SGLANG_SUPPORTED_VISION_MODEL_LIST:
                 return False
         else:

xinference/model/llm/transformers/chatglm.py

@@ -23,7 +23,7 @@ import torch
 from ....core.scheduler import InferenceRequest
 from ....types import ChatCompletion, ChatCompletionChunk, LoRA, PytorchGenerateConfig
 from ..core import chat_context_var
-from ..llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ..llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from ..utils import (
     GLM4_TOOL_CALL_FAMILY,
     generate_chat_completion,
@@ -40,9 +40,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         pytorch_model_config: Optional[PytorchModelConfig] = None,
         peft_model: Optional[List[LoRA]] = None,
@@ -50,8 +48,6 @@ class ChatglmPytorchChatModel(PytorchChatModel):
         super().__init__(
             model_uid,
             model_family,
-            model_spec,
-            quantization,
             model_path,
             pytorch_model_config=pytorch_model_config,
             peft_model=peft_model,
@@ -88,7 +84,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False

xinference/model/llm/transformers/core.py

@@ -38,7 +38,7 @@ from ....types import (
 )
 from ...utils import select_device
 from ..core import LLM, chat_context_var
-from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..llm_family import LLMFamilyV2, LLMSpecV1
 from ..utils import (
     DEEPSEEK_TOOL_CALL_FAMILY,
     LLAMA3_TOOL_CALL_FAMILY,
@@ -92,14 +92,12 @@ class PytorchModel(LLM):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         pytorch_model_config: Optional[PytorchModelConfig] = None,
         peft_model: Optional[List[LoRA]] = None,
     ):
-        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        super().__init__(model_uid, model_family, model_path)
         self._use_fast_tokenizer = True
         self._pytorch_model_config: PytorchModelConfig = self._sanitize_model_config(
             pytorch_model_config
@@ -345,7 +343,7 @@ class PytorchModel(LLM):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False
@@ -367,14 +365,26 @@ class PytorchModel(LLM):
         data = []
         for r in reqs:
             real_len = seq_length - r.padding_len
-            x = torch.cat(
-                [
-                    torch.full((r.padding_len,), 0, dtype=torch.long),
-                    torch.ones((real_len,), dtype=torch.long),
-                ]
-            )
-            data.append(x)
             r.extra_kwargs["attention_mask_seq_len"] = real_len
+
+            if self._tokenizer.padding_side == "left":
+                # [PAD][PAD]...[TOKEN]
+                x = torch.cat(
+                    [
+                        torch.full((r.padding_len,), 0, dtype=torch.long),
+                        torch.ones((real_len,), dtype=torch.long),
+                    ]
+                )
+            else:  # right padding
+                # [TOKEN]...[PAD][PAD]
+                x = torch.cat(
+                    [
+                        torch.ones((real_len,), dtype=torch.long),
+                        torch.full((r.padding_len,), 0, dtype=torch.long),
+                    ]
+                )
+            data.append(x)
+
         return torch.stack(data).to(self._device)
 
     def build_decode_attention_mask(
@@ -388,14 +398,30 @@ class PytorchModel(LLM):
         data = []
         for r in reqs:
             r.extra_kwargs["attention_mask_seq_len"] += 1
-            attention_mask_seq_len = r.extra_kwargs["attention_mask_seq_len"]
-            pad_len = seq_length - attention_mask_seq_len
-            x = torch.cat(
-                [
-                    torch.full((pad_len,), 0, dtype=torch.long),
-                    torch.ones((attention_mask_seq_len,), dtype=torch.long),
-                ]
-            )
+            if self._tokenizer.padding_side == "left":
+                attention_mask_seq_len = r.extra_kwargs["attention_mask_seq_len"]
+                pad_len = seq_length - attention_mask_seq_len
+                assert pad_len > 0, (
+                    f"pad_len must be greater than 0, got {pad_len} = "
+                    f"seq_length({seq_length}) - attention_mask_seq_len({attention_mask_seq_len})"
+                )
+                x = torch.cat(
+                    [
+                        torch.full((pad_len,), 0, dtype=torch.long),
+                        torch.ones((attention_mask_seq_len,), dtype=torch.long),
+                    ]
+                )
+            else:
+                max_len = max(r.extra_kwargs["attention_mask_seq_len"] for r in reqs)
+                real_len = r.extra_kwargs["attention_mask_seq_len"]
+                pad_len = max_len - real_len
+
+                x = torch.cat(
+                    [
+                        torch.ones((real_len,), dtype=torch.long),
+                        torch.full((pad_len,), 0, dtype=torch.long),
+                    ]
+                )
             data.append(x)
         return torch.stack(data).to(self._device)
 
@@ -668,9 +694,7 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
     def __init__(
         self,
         model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
+        model_family: "LLMFamilyV2",
         model_path: str,
         pytorch_model_config: Optional[PytorchModelConfig] = None,
         peft_model: Optional[List[LoRA]] = None,
@@ -678,8 +702,6 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
         super().__init__(
             model_uid,
             model_family,
-            model_spec,
-            quantization,
             model_path,
             pytorch_model_config,
             peft_model,
@@ -702,7 +724,7 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False

xinference/model/llm/transformers/deepseek_v2.py

@@ -15,7 +15,7 @@ import logging
 
 import torch
 
-from ..llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ..llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from .core import PytorchChatModel, register_non_default_model
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ class DeepSeekV2PytorchChatModel(PytorchChatModel):
 
     @classmethod
     def match_json(
-        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+        cls, llm_family: "LLMFamilyV2", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format != "pytorch":
             return False

xinference/model/llm/transformers/gemma3.py

@@ -15,7 +15,7 @@ import logging
 from typing import Dict, List, Set
 
 from ....core.scheduler import InferenceRequest
-from ..llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ..llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from .core import PytorchChatModel, register_non_default_model
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 class Gemma3TextChatModel(PytorchChatModel):
     @classmethod
     def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+        cls, model_family: "LLMFamilyV2", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False

xinference/model/llm/transformers/multimodal/cogagent.py

@@ -21,7 +21,7 @@ import torch
 
 from .....model.utils import select_device
 from ...core import chat_context_var
-from ...llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ...llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from ...utils import _decode_image, parse_messages
 from ..core import register_non_default_model
 from .core import PytorchMultiModalModel
@@ -47,7 +47,7 @@ class CogAgentChatModel(PytorchMultiModalModel):
 
     @classmethod
     def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+        cls, model_family: "LLMFamilyV2", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
         if "cogagent" in family.lower():

xinference/model/llm/transformers/multimodal/deepseek_vl2.py

@@ -23,7 +23,7 @@ import requests
 import torch
 
 from .....model.utils import select_device
-from ...llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ...llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from ..core import register_non_default_model
 from .core import PytorchMultiModalModel
 
@@ -39,7 +39,7 @@ class DeepSeekVL2ChatModel(PytorchMultiModalModel):
 
     @classmethod
     def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+        cls, model_family: "LLMFamilyV2", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         llm_family = model_family.model_family or model_family.model_name
         if "deepseek-vl2" == llm_family.lower():

xinference/model/llm/transformers/multimodal/gemma3.py

@@ -17,7 +17,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from .....model.utils import select_device
 from .....types import PytorchModelConfig
-from ...llm_family import LLMFamilyV1, LLMSpecV1, register_transformer
+from ...llm_family import LLMFamilyV2, LLMSpecV1, register_transformer
 from ..core import register_non_default_model
 from .core import PytorchMultiModalModel
 
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 class Gemma3ChatModel(PytorchMultiModalModel):
     @classmethod
     def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+        cls, model_family: "LLMFamilyV2", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False