sglang 0.4.6.post5__py3-none-any.whl → 0.4.7.post1__py3-none-any.whl

sglang/srt/managers/multimodal_processors/internvl.py

@@ -175,6 +175,10 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
         if not image_data:
             return None
 
+        # Ensure image_data is a list
+        if isinstance(image_data, str):
+            image_data = [image_data]
+
         base_output = self.load_mm_data(
             prompt=input_text,
             image_data=image_data,

sglang/srt/managers/multimodal_processors/kimi_vl.py

@@ -1,4 +1,7 @@
-from typing import List, Union
+import re
+from typing import Any, Dict, List, Optional, Union
+
+import torch
 
 from sglang.srt.managers.multimodal_processors.base_processor import (
     BaseMultimodalProcessor as SGLangBaseProcessor,
@@ -17,20 +20,12 @@ class KimiVLImageProcessor(SGLangBaseProcessor):
     def __init__(self, hf_config, server_args, _processor):
         super().__init__(hf_config, server_args, _processor)
         self.IMAGE_TOKEN = "<|media_pad|>"
-        self.im_token_id = _processor.tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN)
-
-        self.im_start = "<|media_start|>"
-        self.im_start_id = _processor.tokenizer.convert_tokens_to_ids(self.im_start)
-
-        self.im_end = "<|media_end|>"
-        self.im_end_id = _processor.tokenizer.convert_tokens_to_ids(self.im_end)
-
-        self.im_content = "<|media_content|>"
-        self.im_content_id = _processor.tokenizer.convert_tokens_to_ids(self.im_content)
+        self.IMAGE_TOKEN_REGEX = re.compile(r"(?:<\|media_pad\|>)+")
+        self.IM_TOKEN_ID = _processor.tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN)
 
     async def process_mm_data_async(
         self,
-        image_data: List[Union[str, bytes]],
+        image_data: List[Union[str, bytes, Dict]],
         input_text,
         request_obj,
         max_req_input_len,
@@ -45,30 +40,16 @@ class KimiVLImageProcessor(SGLangBaseProcessor):
         base_output = self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
-            multimodal_tokens=MultimodalSpecialTokens(image_token=self.IMAGE_TOKEN),
+            multimodal_tokens=MultimodalSpecialTokens(
+                image_token=self.IMAGE_TOKEN, image_token_regex=self.IMAGE_TOKEN_REGEX
+            ),
             max_req_input_len=max_req_input_len,
         )
-        ret = self.process_mm_data(
-            input_text=base_output.input_text,
-            images=base_output.images,
-        )
-        input_ids = ret["input_ids"].flatten()
-        image_offsets = self.get_mm_items_offset(
-            input_ids=input_ids,
-            mm_token_id=self.im_token_id,
-        )
+
+        combined_mm_item, input_ids = self.process_and_combine_mm_data(base_output)
+
         return {
             "input_ids": input_ids.tolist(),
-            "mm_items": [
-                MultimodalDataItem(
-                    pixel_values=ret["pixel_values"],
-                    image_grid_thws=ret["image_grid_hws"],
-                    modality=Modality.IMAGE,
-                    image_offsets=image_offsets,
-                )
-            ],
-            "im_token_id": self.im_token_id,
-            "im_start_id": self.im_start_id,
-            "im_end_id": self.im_end_id,
-            "im_content_id": self.im_content_id,
+            "mm_items": [combined_mm_item] if combined_mm_item is not None else [],
+            "im_token_id": self.IM_TOKEN_ID,
         }

sglang/srt/managers/multimodal_processors/minicpm.py

@@ -42,7 +42,8 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
             audio_data=audio_data,
             image_data=image_data,
             multimodal_tokens=MultimodalSpecialTokens(
-                image_token=self.image_token, audio_token=self.audio_token
+                image_token=self.image_token,
+                audio_token=self.audio_token,
             ),
         )
         if base_output is None:

sglang/srt/managers/multimodal_processors/phi4mm.py

@@ -0,0 +1,87 @@
+import logging
+from typing import List, Union
+
+from sglang.srt.managers.multimodal_processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.models.phi4mm import Phi4MMForCausalLM
+
+logger = logging.getLogger(__name__)
+
+_IMAGE_SPECIAL_TOKEN = "<|endoftext10|>"
+_IMAGE_SPECIAL_TOKEN_ID = 200010
+
+
+class Phi4MMImageProcessor(BaseMultimodalProcessor):
+    models = [Phi4MMForCausalLM]
+
+    def __init__(self, hf_config, server_args, _processor):
+        super().__init__(hf_config, server_args, _processor)
+        self.multimodal_tokens = MultimodalSpecialTokens(
+            image_token=_IMAGE_SPECIAL_TOKEN,
+        )
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        max_req_input_len,
+        **kwargs,
+    ):
+        audio_data = request_obj.audio_data
+
+        if not image_data and not audio_data:
+            return None
+
+        if not isinstance(image_data, list):
+            image_data = [image_data]
+
+        if not isinstance(audio_data, list):
+            audio_data = [audio_data]
+
+        if audio_data:
+            logger.warning(
+                "Currently SGLang does not support audio data for Phi4MM. We are working on it. You can file an issue to help us prioritize."
+            )
+            audio_data = []
+
+        base_output = self.load_mm_data(
+            prompt=input_text,
+            max_req_input_len=max_req_input_len,
+            audio_data=audio_data,
+            image_data=image_data,
+            multimodal_tokens=self.multimodal_tokens,
+        )
+        if base_output is None:
+            return None
+
+        res = self.process_mm_data(
+            input_text=base_output.input_text,
+            images=base_output.images,
+            audios=base_output.audios,
+        )
+
+        input_ids = res["input_ids"].flatten()
+        image_offsets = self.get_mm_items_offset(
+            input_ids=input_ids,
+            mm_token_id=_IMAGE_SPECIAL_TOKEN_ID,
+        )
+
+        items = [
+            MultimodalDataItem(
+                pixel_values=res["input_image_embeds"],
+                image_sizes=res["image_sizes"],
+                image_emb_mask=res["image_attention_mask"],
+                image_offsets=image_offsets,
+                modality=Modality.IMAGE,
+            )
+        ]
+
+        return {
+            "mm_items": items,
+            "input_ids": input_ids.tolist(),
+            "im_token_id": _IMAGE_SPECIAL_TOKEN_ID,
+        }

sglang/srt/managers/multimodal_processors/qwen_vl.py

@@ -32,8 +32,8 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
         )
         self.IM_START_TOKEN_ID = hf_config.vision_start_token_id
         self.IM_END_TOKEN_ID = hf_config.vision_end_token_id
-        self.image_token_id = hf_config.image_token_id
-        self.video_token_id = hf_config.video_token_id
+        self.IM_TOKEN_ID = hf_config.image_token_id
+        self.VIDEO_TOKEN_ID = hf_config.video_token_id
         self.vision_start_token_id = hf_config.vision_start_token_id
         self.vision_end_token_id = hf_config.vision_end_token_id
         self.NUM_TOKEN_PER_FRAME = 770
@@ -125,87 +125,45 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
         async def resize_image_async(image):
             return resize_image(image)
 
-        images_are_preprocessed = self.mm_inputs_are_preprocessed(base_output.images)
-        if base_output.images and not images_are_preprocessed:
+        # Qwen-specific: resize images if they are raw Image objects
+        if base_output.images and isinstance(base_output.images[0], Image.Image):
             resize_tasks = [resize_image_async(image) for image in base_output.images]
             base_output.images = await asyncio.gather(*resize_tasks)
 
-        ret = self.process_mm_data(
-            input_text=base_output.input_text,
-            images=None if images_are_preprocessed else base_output.images,
-        )
-        input_ids = ret["input_ids"].flatten().tolist()
-        image_offsets = self.get_mm_items_offset(
-            input_ids=ret["input_ids"].flatten(), mm_token_id=self.image_token_id
-        )
-        image_grid_thw = None
         video_grid_thw = None  # TODO
-        items = []
-
-        if base_output.images:
-            if images_are_preprocessed:
-                image_grid_thw = torch.concat(
-                    [
-                        torch.as_tensor(item.image_grid_thws)
-                        for item in base_output.images
-                    ]
-                )
-                all_pixel_values = [
-                    item.pixel_values
-                    for item in base_output.images
-                    if item.pixel_values is not None
-                ]
-                all_precomputed_features = [
-                    item.precomputed_features
-                    for item in base_output.images
-                    if item.precomputed_features is not None
-                ]
-                pixel_values = (
-                    torch.concat(all_pixel_values) if all_pixel_values else None
-                )
-                precomputed_features = (
-                    torch.concat(all_precomputed_features)
-                    if all_precomputed_features
-                    else None
-                )
-            else:
-                image_grid_thw = ret["image_grid_thw"]
-                pixel_values = ret["pixel_values"]
-                precomputed_features = None
-            items += [
-                MultimodalDataItem(
-                    pixel_values=pixel_values,
-                    image_grid_thws=image_grid_thw,
-                    video_grid_thws=video_grid_thw,
-                    precomputed_features=precomputed_features,
-                    image_offsets=image_offsets,
-                    modality=Modality.IMAGE,
-                )
-            ]
+
+        combined_mm_item, input_ids = self.process_and_combine_mm_data(base_output)
+
+        if combined_mm_item is None:
+            # Note(Xinyuan): This is the case where image loading fails.
+            return None
+
+        video_grid_thw = None  # TODO
+        second_per_grid_ts = getattr(combined_mm_item, "second_per_grid_ts", None)
 
         mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
             spatial_merge_size=self.hf_config.vision_config.spatial_merge_size,
-            image_token_id=self.image_token_id,
-            video_token_id=self.video_token_id,
+            image_token_id=self.IM_TOKEN_ID,
+            video_token_id=self.VIDEO_TOKEN_ID,
             vision_start_token_id=self.vision_start_token_id,
             model_type=self.hf_config.model_type,
             tokens_per_second=getattr(
                 self.hf_config.vision_config, "tokens_per_second", None
             ),
-            input_ids=torch.tensor(input_ids).unsqueeze(0),
-            image_grid_thw=image_grid_thw,
+            input_ids=input_ids.unsqueeze(0),
+            image_grid_thw=combined_mm_item.image_grid_thw,
             video_grid_thw=video_grid_thw,
-            second_per_grid_ts=ret.get("second_per_grid_ts", None),
+            second_per_grid_ts=second_per_grid_ts,
         )
         mrope_positions = mrope_positions.squeeze(1)
 
         return {
-            "input_ids": input_ids,
-            "mm_items": items,
+            "input_ids": input_ids.tolist(),
+            "mm_items": [combined_mm_item],
             "im_start_id": self.IM_START_TOKEN_ID,
             "im_end_id": self.IM_END_TOKEN_ID,
-            "im_token_id": self.image_token_id,
-            "video_token_id": self.video_token_id,
+            "im_token_id": self.IM_TOKEN_ID,
+            "video_token_id": self.VIDEO_TOKEN_ID,
             "mrope_positions": mrope_positions,
             "mrope_position_delta": mrope_position_delta,
         }

sglang/srt/managers/multimodal_processors/vila.py

@@ -0,0 +1,85 @@
+from typing import Any, Dict, List, Optional, Type, cast
+
+import torch.nn as nn
+from transformers.configuration_utils import PretrainedConfig
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+from sglang.srt.managers.io_struct import (
+    EmbeddingReqInput,
+    GenerateReqInput,
+    ImageDataItem,
+)
+from sglang.srt.managers.multimodal_processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.models.vila import VILAForConditionalGeneration
+from sglang.srt.server_args import ServerArgs
+
+
+class VILAProcessor(ProcessorMixin):
+    """A stub class for the VILA processor."""
+
+    tokenizer: PreTrainedTokenizerBase
+
+
+class VILAMultimodalProcessor(BaseMultimodalProcessor):
+    models: List[Type[nn.Module]] = [VILAForConditionalGeneration]
+
+    _processor: VILAProcessor
+
+    def __init__(
+        self,
+        hf_config: PretrainedConfig,
+        server_args: ServerArgs,
+        _processor: VILAProcessor,
+    ) -> None:
+        super().__init__(hf_config, server_args, _processor)
+
+    async def process_mm_data_async(
+        self,
+        image_data: Optional[ImageDataItem | List[ImageDataItem]],
+        input_text: str | List[int],
+        request_obj: GenerateReqInput | EmbeddingReqInput,
+        max_req_input_len: int,
+        **kwargs,
+    ) -> Optional[Dict[str, Any]]:
+        if not image_data:
+            return None
+
+        if not isinstance(image_data, list):
+            image_data = [image_data]
+
+        mm_data = self.load_mm_data(
+            prompt=input_text,
+            multimodal_tokens=MultimodalSpecialTokens(
+                image_token=self._processor.tokenizer.image_token
+            ),
+            max_req_input_len=max_req_input_len,
+            image_data=image_data,
+        )
+
+        inputs = self.process_mm_data(
+            input_text=mm_data.input_text,
+            images=mm_data.images,
+        )
+
+        image_offsets = self.get_mm_items_offset(
+            input_ids=inputs.input_ids[0],
+            mm_token_id=cast(int, self._processor.tokenizer.image_token_id),
+        )
+
+        mm_items: List[MultimodalDataItem] = [
+            MultimodalDataItem(
+                modality=Modality.IMAGE,
+                image_offsets=image_offsets,
+                pixel_values=inputs.pixel_values,
+            )
+        ]
+
+        return dict(
+            input_ids=inputs.input_ids[0].tolist(),
+            mm_items=mm_items,
+        )