xinference 1.5.0.post1__py3-none-any.whl → 1.5.1__py3-none-any.whl

xinference/model/llm/transformers/gemma3.py

@@ -36,7 +36,7 @@ logger = logging.getLogger(__name__)
 
 class Gemma3TextChatModel(PytorchChatModel):
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
@@ -56,7 +56,7 @@ class Gemma3ChatModel(PytorchChatModel):
         self._processor = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
@@ -85,6 +85,7 @@ class Gemma3ChatModel(PytorchChatModel):
         device = "auto" if device == "cuda" else device
         min_pixels = self._pytorch_model_config.get("min_pixels")
         max_pixels = self._pytorch_model_config.get("max_pixels")
+        kwargs = self.apply_bnb_quantization()
         self._processor = AutoProcessor.from_pretrained(
             self.model_path,
             min_pixels=min_pixels,
@@ -92,9 +93,7 @@ class Gemma3ChatModel(PytorchChatModel):
         )
         self._tokenizer = self._processor.tokenizer
         self._model = Gemma3ForConditionalGeneration.from_pretrained(
-            self.model_path,
-            device_map="auto",
-            torch_dtype="bfloat16",
+            self.model_path, device_map="auto", torch_dtype="bfloat16", **kwargs
         )
 
     @cache_clean
@@ -128,7 +127,12 @@ class Gemma3ChatModel(PytorchChatModel):
         ).to(self._device)
         input_len = inputs["input_ids"].shape[-1]
 
-        generation = self._model.generate(**inputs, do_sample=False)
+        generation = self._model.generate(
+            **inputs,
+            do_sample=False,
+            max_new_tokens=config.get("max_tokens", 512),
+            temperature=config.get("temperature", 1),
+        )
         generation = generation[0][input_len:]
 
         decoded = self._processor.decode(generation, skip_special_tokens=True)
@@ -159,7 +163,11 @@ class Gemma3ChatModel(PytorchChatModel):
 
         def model_generate():
             try:
-                return self._model.generate(**gen_kwargs)
+                return self._model.generate(
+                    **gen_kwargs,
+                    max_new_tokens=config.get("max_tokens", 512),
+                    temperature=config.get("temperature", 1),
+                )
             except Exception:
                 nonlocal error
                 error = sys.exc_info()

xinference/model/llm/transformers/glm4v.py

@@ -39,7 +39,7 @@ class Glm4VModel(PytorchChatModel):
         self._model = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
@@ -54,25 +54,7 @@ class Glm4VModel(PytorchChatModel):
         self._device = select_device(device)
 
         kwargs = {"device_map": self._device}
-        quantization = self.quantization
-
-        # referenced from PytorchModel.load
-        if quantization != "none":
-            if self._device == "cuda" and self._is_linux():
-                kwargs["device_map"] = "auto"
-                if quantization == "4-bit":
-                    kwargs["load_in_4bit"] = True
-                elif quantization == "8-bit":
-                    kwargs["load_in_8bit"] = True
-                else:
-                    raise ValueError(
-                        f"Quantization {quantization} is not supported in temporary"
-                    )
-            else:
-                if quantization != "8-bit":
-                    raise ValueError(
-                        f"Only 8-bit quantization is supported if it is not linux system or cuda device"
-                    )
+        kwargs = self.apply_bnb_quantization(kwargs)
 
         if self._check_tensorizer_integrity():
             self._model, self._tokenizer = self._load_tensorizer()

xinference/model/llm/transformers/glm_edge_v.py

@@ -42,7 +42,7 @@ class GlmEdgeVModel(PytorchChatModel):
         self._processor = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
@@ -57,25 +57,7 @@ class GlmEdgeVModel(PytorchChatModel):
         self._device = select_device(device)
 
         kwargs = {"device_map": self._device}
-        quantization = self.quantization
-
-        # referenced from PytorchModel.load
-        if quantization != "none":
-            if self._device == "cuda" and self._is_linux():
-                kwargs["device_map"] = "auto"
-                if quantization == "4-bit":
-                    kwargs["load_in_4bit"] = True
-                elif quantization == "8-bit":
-                    kwargs["load_in_8bit"] = True
-                else:
-                    raise ValueError(
-                        f"Quantization {quantization} is not supported in temporary"
-                    )
-            else:
-                if quantization != "8-bit":
-                    raise ValueError(
-                        f"Only 8-bit quantization is supported if it is not linux system or cuda device"
-                    )
+        kwargs = self.apply_bnb_quantization(kwargs)
 
         processor = AutoImageProcessor.from_pretrained(
             self.model_path, trust_remote_code=True
@@ -87,6 +69,7 @@ class GlmEdgeVModel(PytorchChatModel):
             trust_remote_code=True,
             torch_dtype=torch.bfloat16,
             device_map="auto",
+            **kwargs
         )
 
         self._model = model

xinference/model/llm/transformers/intern_vl.py

@@ -243,7 +243,7 @@ class InternVLChatModel(PytorchChatModel):
         self._model = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
@@ -311,7 +311,7 @@ class InternVLChatModel(PytorchChatModel):
         device_map[f"language_model.model.layers.{num_layers - 1}"] = 0
         return device_map
 
-    def load(self, **kwargs):
+    def load(self):
         from transformers import AutoModel, AutoTokenizer
 
         if self._check_tensorizer_integrity():
@@ -329,10 +329,7 @@ class InternVLChatModel(PytorchChatModel):
         if device is not None:
             kwargs["device_map"] = device
 
-        if "8-bit" in self.quantization.lower():
-            kwargs["load_in_8bit"] = True
-        elif "4-bit" in self.quantization.lower():
-            kwargs["load_in_4bit"] = True
+        kwargs = self.apply_bnb_quantization(kwargs)
 
         self._model = AutoModel.from_pretrained(self.model_path, **kwargs).eval()
 

xinference/model/llm/transformers/internlm2.py

@@ -71,7 +71,7 @@ class Internlm2PytorchChatModel(PytorchChatModel):
         return model, tokenizer
 
     @classmethod
-    def match(
+    def match_json(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         model_family = llm_family.model_family or llm_family.model_name

xinference/model/llm/transformers/minicpmv25.py

@@ -42,7 +42,7 @@ class MiniCPMV25Model(PytorchChatModel):
         self._model = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
@@ -55,7 +55,7 @@ class MiniCPMV25Model(PytorchChatModel):
 
         return AutoModel
 
-    def load(self, **kwargs):
+    def load(self):
         from transformers import AutoModel, AutoTokenizer
         from transformers.generation import GenerationConfig
 
@@ -76,11 +76,13 @@ class MiniCPMV25Model(PytorchChatModel):
         if "int4" in self.model_path:
             model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True)
         else:
+            kwargs = self.apply_bnb_quantization()
             model = AutoModel.from_pretrained(
                 self.model_path,
                 trust_remote_code=True,
                 torch_dtype=torch.float16,
                 device_map=self._device,
+                **kwargs
             )
         tokenizer = AutoTokenizer.from_pretrained(
             self.model_path, trust_remote_code=True

xinference/model/llm/transformers/minicpmv26.py

@@ -49,7 +49,7 @@ class MiniCPMV26Model(PytorchChatModel):
         self._processor = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         family = model_family.model_family or model_family.model_name
@@ -71,7 +71,7 @@ class MiniCPMV26Model(PytorchChatModel):
 
         return AutoModel
 
-    def load(self, **kwargs):
+    def load(self):
         from transformers import AutoModel, AutoProcessor, AutoTokenizer
         from transformers.generation import GenerationConfig
 
@@ -96,11 +96,13 @@ class MiniCPMV26Model(PytorchChatModel):
         if "int4" in self.model_path:
             model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True)
         else:
+            kwargs = self.apply_bnb_quantization()
             model = AutoModel.from_pretrained(
                 self.model_path,
                 trust_remote_code=True,
                 torch_dtype=torch.float16,
                 device_map=self._device,
+                **kwargs,
             )
         tokenizer = AutoTokenizer.from_pretrained(
             self.model_path, trust_remote_code=True
@@ -322,7 +324,7 @@ class MiniCPMV26Model(PytorchChatModel):
             "input_image": images,
         }
 
-    def _get_full_prompt(self, messages: List[Dict], tools):
+    def _get_full_prompt(self, messages: List[Dict], tools):  # type: ignore
         msgs, video_existed = self._convert_to_specific_style(messages)
         if video_existed:
             raise RuntimeError(

xinference/model/llm/transformers/omnilmm.py

@@ -35,7 +35,7 @@ class OmniLMMModel(PytorchChatModel):
         self._model = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         llm_family = model_family.model_family or model_family.model_name

xinference/model/llm/transformers/opt.py

@@ -42,7 +42,7 @@ class OptPytorchModel(PytorchModel):
         )
 
     @classmethod
-    def match(
+    def match_json(
         cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if llm_spec.model_format != "pytorch":

xinference/model/llm/transformers/ovis2.py

@@ -0,0 +1,302 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import uuid
+from typing import Dict, Iterator, List, Optional, Union
+
+import torch
+from PIL import Image
+
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    CompletionChunk,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
+from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class Ovis2ChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tokenizer = None
+        self._model = None
+        self._device = None
+        self._processor = None
+
+    @classmethod
+    def match_json(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        llm_family = model_family.model_family or model_family.model_name
+        if "ovis2".lower() in llm_family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoModelForCausalLM
+
+        # load model
+        self._model = AutoModelForCausalLM.from_pretrained(
+            self.model_path,
+            torch_dtype=torch.bfloat16,
+            multimodal_max_length=32768,
+            trust_remote_code=True,
+        ).cuda()
+        self._text_tokenizer = self._model.get_text_tokenizer()
+        self._visual_tokenizer = self._model.get_visual_tokenizer()
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[ChatCompletionMessage],  # type: ignore
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)
+
+        generate_config = generate_config if generate_config else {}
+
+        stream = generate_config.get("stream", False) if generate_config else False
+
+        if stream:
+            # raise NotImplementedError("Stream is not supported for Ovis2 model.")
+            it = self._generate_stream(messages, generate_config)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(messages, generate_config)
+            return c
+
+    def _generate(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> ChatCompletion:
+        input_ids, attention_mask, pixel_values, gen_kwargs = self._generate_chat_data(
+            messages, config
+        )
+
+        # generate output
+        with torch.inference_mode():
+            gen_kwargs.update(
+                dict(
+                    pixel_values=pixel_values,
+                    attention_mask=attention_mask,
+                )
+            )
+
+            output_ids = self._model.generate(
+                input_ids,
+                **gen_kwargs,
+            )[0]
+            output = self._text_tokenizer.decode(output_ids, skip_special_tokens=True)
+        return generate_chat_completion(self.model_uid, output)
+
+    def _generate_stream(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> Iterator[CompletionChunk]:
+        from threading import Thread
+
+        from transformers import TextIteratorStreamer
+
+        input_ids, attention_mask, pixel_values, gen_kwargs = self._generate_chat_data(
+            messages, config
+        )
+
+        _, inputs_embeds, _, attention_mask = self._model.merge_multimodal(
+            text_input_ids=input_ids,
+            text_attention_masks=attention_mask,
+            text_labels=None,
+            pixel_values=pixel_values,
+            left_padding=True,
+        )
+
+        streamer = TextIteratorStreamer(
+            self._text_tokenizer, timeout=60, skip_prompt=True, skip_special_tokens=True
+        )
+
+        gen_kwargs.update(
+            dict(
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                streamer=streamer,
+            )
+        )
+
+        inputs_embeds = inputs_embeds.detach()
+        torch.cuda.empty_cache()
+
+        thread = Thread(target=self._model.llm.generate, kwargs=gen_kwargs)
+        thread.start()
+
+        completion_id = str(uuid.uuid1())
+
+        for new_text in streamer:
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=-1,
+                completion_tokens=-1,
+                total_tokens=-1,
+                has_choice=True,
+                has_content=True,
+            )
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )
+
+    def parse_messages_ovis(self, messages: List[Dict]) -> List[Dict]:
+        ovis_msgs = []
+        for mess in messages:
+            contents = mess["content"]
+            role = mess["role"]
+            if role == "user":
+                role = "human"
+            elif role == "assistant":
+                role = "gpt"
+            elif role == "system":
+                role = "system"
+
+            for content in contents:
+                if content["type"] == "text":
+                    ovis_msgs.append({"from": role, "value": content["text"]})
+
+        return ovis_msgs
+
+    def _generate_chat_data(
+        self, messages: List[Dict], config: PytorchGenerateConfig = {}
+    ):
+        from qwen_vl_utils import process_vision_info
+
+        messages_ovis = self.parse_messages_ovis(messages)
+        max_partition = None
+        prompt = messages_ovis[-1]["value"]
+
+        # Preparation for inference
+        image_inputs, video_inputs = process_vision_info(messages)
+
+        image_inputs = image_inputs if image_inputs else []
+
+        if image_inputs and len(image_inputs) > 0:
+            if len(image_inputs) == 1:
+                max_partition = 9
+                prompt = f"<image>\n{prompt}"
+            else:
+                max_partition = len(image_inputs) + 1
+                prompt = (
+                    "\n".join(
+                        [f"Image {i+1}: <image>" for i in range(len(image_inputs))]
+                    )
+                    + "\n"
+                    + prompt
+                )
+        elif video_inputs and len(video_inputs) > 0:
+            if isinstance(video_inputs[0], torch.Tensor):
+                # Convert from list[Tensor] to list[Image]
+                pil_images = self._convert_video_tensors_to_pil(video_inputs)
+
+                video_inputs = pil_images  # Update video_inputs to PIL image list
+
+            max_partition = 1
+            image_inputs = video_inputs
+            prompt = "\n".join(["<image>"] * len(video_inputs)) + "\n" + prompt
+        else:
+            max_partition = 0
+            prompt = prompt
+
+        messages_ovis[-1]["value"] = prompt
+
+        # format conversation
+        prompt, input_ids, pixel_values = self._model.preprocess_inputs(
+            messages_ovis, image_inputs, max_partition=max_partition
+        )
+
+        attention_mask = torch.ne(input_ids, self._text_tokenizer.pad_token_id)
+        input_ids = input_ids.unsqueeze(0).to(device=self._model.device)
+        attention_mask = attention_mask.unsqueeze(0).to(device=self._model.device)
+        if pixel_values is not None:
+            pixel_values = pixel_values.to(
+                dtype=self._visual_tokenizer.dtype, device=self._visual_tokenizer.device
+            )
+        pixel_values = [pixel_values]
+
+        gen_kwargs = dict(
+            max_new_tokens=config.get("max_tokens", 1024),
+            do_sample=False,
+            top_p=None,
+            top_k=None,
+            temperature=config.get("temperature", None),
+            repetition_penalty=None,
+            eos_token_id=self._model.generation_config.eos_token_id,
+            pad_token_id=self._text_tokenizer.pad_token_id,
+            use_cache=True,
+        )
+
+        return input_ids, attention_mask, pixel_values, gen_kwargs
+
+    def _convert_video_tensors_to_pil(self, video_inputs: List) -> List[Image.Image]:
+        """Convert video tensors to a list of PIL images"""
+        from torchvision import transforms
+
+        to_pil = transforms.ToPILImage()
+        pil_images = []
+
+        for video_tensor_4d in video_inputs:
+            if isinstance(video_tensor_4d, torch.Tensor):
+                # Verify it's a 4D tensor
+                if video_tensor_4d.ndim == 4:
+                    # Iterate through the first dimension (frames) of 4D tensor
+                    for i in range(video_tensor_4d.size(0)):
+                        frame_tensor_3d = video_tensor_4d[
+                            i
+                        ]  # Get 3D frame tensor [C, H, W]
+                        # Ensure tensor is on CPU before conversion
+                        if frame_tensor_3d.is_cuda:
+                            frame_tensor_3d = frame_tensor_3d.cpu()
+                        try:
+                            pil_image = to_pil(frame_tensor_3d)
+                            pil_images.append(pil_image)
+                        except Exception as e:
+                            logger.error(
+                                f"Error converting frame {i} to PIL Image: {e}"
+                            )
+                            # Can choose to skip this frame or handle error differently
+                else:
+                    logger.warning(
+                        f"Expected 4D tensor in video_inputs, but got {video_tensor_4d.ndim}D. Skipping this tensor."
+                    )
+            elif isinstance(video_tensor_4d, Image.Image):
+                # If fetch_video returns Image list, add directly
+                pil_images.append(video_tensor_4d)
+            else:
+                logger.warning(
+                    f"Unexpected type in video_inputs: {type(video_tensor_4d)}. Skipping."
+                )
+
+        return pil_images

xinference/model/llm/transformers/qwen-omni.py

@@ -56,7 +56,7 @@ class Qwen2_5OmniChatModel(PytorchChatModel):
         self._processor = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
@@ -83,6 +83,7 @@ class Qwen2_5OmniChatModel(PytorchChatModel):
             if not flash_attn_installed
             else {"attn_implementation": "flash_attention_2"}
         )
+        kwargs = self.apply_bnb_quantization(kwargs)
         logger.debug("Loading model with extra kwargs: %s", kwargs)
 
         self._processor = Qwen2_5OmniProcessor.from_pretrained(

xinference/model/llm/transformers/qwen2_audio.py

@@ -42,7 +42,7 @@ class Qwen2AudioChatModel(PytorchChatModel):
         self._device = None
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         llm_family = model_family.model_family or model_family.model_name
@@ -58,6 +58,7 @@ class Qwen2AudioChatModel(PytorchChatModel):
         # for multiple GPU, set back to auto to make multiple devices work
         device = "auto" if device == "cuda" else device
         self._device = device
+        kwargs = self.apply_bnb_quantization()
 
         self._processor = AutoProcessor.from_pretrained(
             self.model_path,
@@ -70,6 +71,7 @@ class Qwen2AudioChatModel(PytorchChatModel):
             device_map=device,
             # trust_remote_code=True,
             revision=self.model_spec.model_revision,
+            **kwargs,
         )
 
     def _transform_messages(

xinference/model/llm/transformers/qwen2_vl.py

@@ -54,7 +54,7 @@ class Qwen2VLChatModel(PytorchChatModel):
         return pytorch_model_config
 
     @classmethod
-    def match(
+    def match_json(
         cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
     ) -> bool:
         if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
@@ -81,6 +81,8 @@ class Qwen2VLChatModel(PytorchChatModel):
         self._device = device
         # for multiple GPU, set back to auto to make multiple devices work
         device = "auto" if device == "cuda" else device
+        kwargs = self.apply_bnb_quantization()
+
         min_pixels = self._pytorch_model_config.get("min_pixels")
         max_pixels = self._pytorch_model_config.get("max_pixels")
         self._processor = AutoProcessor.from_pretrained(
@@ -106,6 +108,7 @@ class Qwen2VLChatModel(PytorchChatModel):
                 device_map=device,
                 attn_implementation="flash_attention_2",
                 trust_remote_code=True,
+                **kwargs,
             ).eval()
         elif is_npu_available():
             # Ascend do not support bf16
@@ -114,6 +117,7 @@ class Qwen2VLChatModel(PytorchChatModel):
                 device_map="auto",
                 trust_remote_code=True,
                 torch_dtype="float16",
+                **kwargs,
             ).eval()
         else:
             self._model = model_cls.from_pretrained(