sglang 0.4.9.post2__py3-none-any.whl → 0.4.9.post3__py3-none-any.whl

sglang/srt/multimodal/processors/qwen_audio.py

@@ -0,0 +1,65 @@
+import re
+
+from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.models.qwen2_audio import Qwen2AudioForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+
+
+class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
+    models = [Qwen2AudioForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor):
+        super().__init__(hf_config, server_args, _processor)
+        self.AUDIO_TOKEN = "<|audio_bos|><|AUDIO|><|audio_eos|>"
+        self.AUDIO_TOKEN_REGEX = re.compile(
+            r"<\|audio_bos\|>(?:<\|AUDIO\|>)+<\|audio_eos\|>"
+        )
+        # Collect special token ids
+        tokenizer = self._processor.tokenizer
+        self.audio_start_id = tokenizer.convert_tokens_to_ids("<|audio_bos|>")
+        self.audio_token_id = tokenizer.convert_tokens_to_ids("<|AUDIO|>")
+        self.audio_end_id = tokenizer.convert_tokens_to_ids("<|audio_eos|>")
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            audio_token=self.AUDIO_TOKEN,
+            audio_token_regex=self.AUDIO_TOKEN_REGEX,
+            audio_token_id=self.audio_token_id,
+        ).build(_processor)
+
+    async def process_mm_data_async(
+        self,
+        audio_data,
+        input_text,
+        **kwargs,
+    ):
+        base_output = self.load_mm_data(
+            prompt=input_text,
+            audio_data=audio_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+        if base_output is None:
+            return None
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        assert (
+            "feature_attention_mask" in ret
+        ), "feature_attention_mask not found in processor output"
+        input_lengths = ret["feature_attention_mask"].sum(dim=-1)
+        input_lengths = (input_lengths - 1) // 2 + 1
+        output_lengths = (input_lengths - 2) // 2 + 1
+
+        mm_items[0].model_specific_data["audio_feature_lens"] = output_lengths
+
+        return {
+            "mm_items": mm_items,
+            "input_ids": input_ids.tolist(),
+            "audio_start_id": self.audio_start_id,
+            "audio_token_id": self.audio_token_id,
+            "audio_end_id": self.audio_end_id,
+        }

sglang/srt/multimodal/processors/qwen_vl.py

@@ -203,16 +203,9 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
 
     def __init__(self, hf_config, server_args, _processor):
         super().__init__(hf_config, server_args, _processor)
-        # The single, pre-expanded image token.
-        self.IMAGE_TOKEN = "<|vision_start|><|image_pad|><|vision_end|>"
         # The regex that matches expanded image tokens.
-        self.IMAGE_TOKEN_REGEX = re.compile(
-            r"<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>"
-        )
         self.IM_START_TOKEN_ID = hf_config.vision_start_token_id
         self.IM_END_TOKEN_ID = hf_config.vision_end_token_id
-        self.IM_TOKEN_ID = hf_config.image_token_id
-        self.VIDEO_TOKEN_ID = hf_config.video_token_id
         self.vision_start_token_id = hf_config.vision_start_token_id
         self.vision_end_token_id = hf_config.vision_end_token_id
         self.NUM_TOKEN_PER_FRAME = 770
@@ -220,19 +213,20 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
         self.MIN_PIXELS = 4 * 28 * 28
         self.MAX_PIXELS = 16384 * 28 * 28
         self.MAX_RATIO = 200
-        # TODO(mick): move all MultimodalSpecialTokens initializations into processor init
-        self.mm_special_tokens = MultimodalSpecialTokens(
-            image_token=self.IMAGE_TOKEN,
-            image_token_regex=self.IMAGE_TOKEN_REGEX,
-            video_token=self.VIDEO_TOKEN_ID,
-        )
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token="<|vision_start|><|image_pad|><|vision_end|>",
+            image_token_id=hf_config.image_token_id,
+            image_token_regex=re.compile(
+                r"<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>"
+            ),
+            video_token_id=hf_config.video_token_id,
+        ).build(_processor)
 
     async def process_mm_data_async(
         self,
         image_data: List[Union[str, bytes]],
         input_text,
         request_obj,
-        max_req_input_len,
         *args,
         **kwargs,
     ):
@@ -241,8 +235,7 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,
-            multimodal_tokens=self.mm_special_tokens,
-            max_req_input_len=max_req_input_len,
+            multimodal_tokens=self.mm_tokens,
         )
 
         # Qwen-specific: resize images if they are raw Image objects
@@ -255,13 +248,15 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
                 await preprocess_video(video) for video in base_output.videos
             ]
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(base_output)
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
 
         input_ids = input_ids.flatten()
         mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index(
             spatial_merge_size=self.hf_config.vision_config.spatial_merge_size,
-            image_token_id=self.IM_TOKEN_ID,
-            video_token_id=self.VIDEO_TOKEN_ID,
+            image_token_id=self.mm_tokens.image_token_id,
+            video_token_id=self.mm_tokens.video_token_id,
             vision_start_token_id=self.vision_start_token_id,
             model_type=self.hf_config.model_type,
             tokens_per_second=getattr(
@@ -279,8 +274,8 @@ class Qwen2_5VLImageProcessor(SGLangBaseProcessor):
             "mm_items": mm_items,
             "im_start_id": self.IM_START_TOKEN_ID,
             "im_end_id": self.IM_END_TOKEN_ID,
-            "im_token_id": self.IM_TOKEN_ID,
-            "video_token_id": self.VIDEO_TOKEN_ID,
+            "im_token_id": self.mm_tokens.image_token_id,
+            "video_token_id": self.mm_tokens.video_token_id,
             "mrope_positions": mrope_positions,
             "mrope_position_delta": mrope_position_delta,
         }

sglang/srt/multimodal/processors/vila.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Type, cast
+from typing import Any, Dict, List, Optional, Type
 
 import torch.nn as nn
 from transformers.configuration_utils import PretrainedConfig
@@ -8,9 +8,8 @@ from transformers.tokenization_utils_base import PreTrainedTokenizerBase
 from sglang.srt.managers.io_struct import (
     EmbeddingReqInput,
     GenerateReqInput,
-    ImageDataItem,
+    ImageDataInputItem,
 )
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
 from sglang.srt.models.vila import VILAForConditionalGeneration
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor,
@@ -37,31 +36,32 @@ class VILAMultimodalProcessor(BaseMultimodalProcessor):
         _processor: VILAProcessor,
     ) -> None:
         super().__init__(hf_config, server_args, _processor)
-        self.IM_TOKEN_ID = hf_config.image_token_id
-        self.VIDEO_TOKEN_ID = hf_config.video_token_id
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self._processor.tokenizer.image_token,
+            image_token_id=hf_config.image_token_id,
+            video_token_id=hf_config.video_token_id,
+        ).build(_processor)
 
     async def process_mm_data_async(
         self,
-        image_data: Optional[ImageDataItem | List[ImageDataItem]],
+        image_data: Optional[ImageDataInputItem | List[ImageDataInputItem]],
         input_text: str | List[int],
         request_obj: GenerateReqInput | EmbeddingReqInput,
-        max_req_input_len: int,
         **kwargs,
     ) -> Optional[Dict[str, Any]]:
         base_output = self.load_mm_data(
             prompt=input_text,
-            multimodal_tokens=MultimodalSpecialTokens(
-                image_token=self._processor.tokenizer.image_token
-            ),
-            max_req_input_len=max_req_input_len,
+            multimodal_tokens=self.mm_tokens,
             image_data=image_data,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(base_output)
+        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
 
         return {
             "input_ids": input_ids.tolist(),
             "mm_items": mm_items,
-            "im_token_id": self.IM_TOKEN_ID,
-            "video_token_id": self.VIDEO_TOKEN_ID,
+            "im_token_id": self.mm_tokens.image_token_id,
+            "video_token_id": self.mm_tokens.video_token_id,
         }

sglang/srt/sampling/sampling_params.py

@@ -89,7 +89,7 @@ class SamplingParams:
         if self.top_k == -1:
             self.top_k = TOP_K_ALL  # whole vocabulary
 
-    def verify(self):
+    def verify(self, vocab_size):
         if self.temperature < 0.0:
             raise ValueError(
                 f"temperature must be non-negative, got {self.temperature}."
@@ -131,6 +131,13 @@ class SamplingParams:
                     f"min_new_tokens must be in [0, max_new_tokens({self.max_new_tokens})], got "
                     f"{self.min_new_tokens}."
                 )
+        if self.logit_bias is not None:
+            for token_id in self.logit_bias:
+                if not 0 <= int(token_id) < vocab_size:
+                    raise ValueError(
+                        f"logit_bias must has keys in [0, {vocab_size - 1}], got "
+                        f"{token_id}."
+                    )
         grammars = [
             self.json_schema,
             self.regex,