sglang 0.5.0rc0__py3-none-any.whl → 0.5.0rc2__py3-none-any.whl

sglang/srt/multimodal/processors/glm4v.py

@@ -0,0 +1,132 @@
+import re
+from typing import List, Union
+
+from decord import VideoReader
+from transformers.video_utils import VideoMetadata
+
+from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
+from sglang.srt.models.glm4v import Glm4vForConditionalGeneration
+from sglang.srt.models.glm4v_moe import Glm4vMoeForConditionalGeneration
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor as SGLangBaseProcessor,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultiModalProcessorOutput,
+    MultimodalSpecialTokens,
+)
+
+
+class Glm4vImageProcessor(SGLangBaseProcessor):
+    models = [Glm4vForConditionalGeneration, Glm4vMoeForConditionalGeneration]
+
+    def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+        super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+
+        # GLM-4.1V and GLM-4.5V specific tokens
+        self.IMAGE_TOKEN = "<|image|>"
+        self.VIDEO_TOKEN = "<|video|>"
+        self.IMAGE_START_TOKEN = "<|begin_of_image|>"
+        self.IMAGE_END_TOKEN = "<|end_of_image|>"
+        self.VIDEO_START_TOKEN = "<|begin_of_video|>"
+        self.VIDEO_END_TOKEN = "<|end_of_video|>"
+
+        # Token IDs
+        self.IM_TOKEN_ID = hf_config.image_token_id
+        self.VIDEO_TOKEN_ID = hf_config.video_token_id
+        self.IMAGE_START_TOKEN_ID = hf_config.image_start_token_id
+        self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id
+        self.VIDEO_START_TOKEN_ID = hf_config.video_start_token_id
+        self.VIDEO_END_TOKEN_ID = hf_config.video_end_token_id
+
+        # Vision config
+        self.IMAGE_FACTOR = 28
+        self.MIN_PIXELS = 112 * 112
+        self.MAX_PIXELS = 30000 * 28 * 28 * 2
+
+        self.mm_tokens = MultimodalSpecialTokens(
+            image_token=self.IMAGE_TOKEN,
+            image_token_id=self.IM_TOKEN_ID,
+            video_token=self.VIDEO_TOKEN,
+            # Note: For GLM4v videos, it uses the video token before tokenization but uses image token after tokenization
+            video_token_id=self.IM_TOKEN_ID,
+        ).build(_processor)
+
+    # adapted from https://github.com/huggingface/transformers/blob/369c99d0cea403b77bd0aef818527106453fd9fc/src/transformers/video_utils.py#L312
+    async def preprocess_video(self, vr: VideoReader):
+        """
+        Preprocess video using VideoReader from Decord backend.
+
+        Args:
+            vr (VideoReader): VideoReader object from decord
+
+        Returns:
+            tuple: A tuple containing processed frames and metadata
+        """
+        video_fps = vr.get_avg_fps()
+        total_num_frames = len(vr)
+        duration = total_num_frames / video_fps if video_fps else 0
+
+        metadata = VideoMetadata(
+            total_num_frames=int(total_num_frames),
+            fps=float(video_fps),
+            duration=float(duration),
+            video_backend="decord",
+        )
+
+        # Extract all frames
+        indices = list(range(total_num_frames))
+        frames = vr.get_batch(indices).asnumpy()
+        metadata.frames_indices = indices
+
+        return frames, metadata
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        base_output = self.load_mm_data(
+            prompt=input_text,
+            image_data=image_data,
+            video_data=request_obj.video_data,
+            multimodal_tokens=self.mm_tokens,
+        )
+
+        video_metadata = None
+
+        if base_output.videos:
+            videos_processed = [
+                await self.preprocess_video(video) for video in base_output.videos
+            ]
+            base_output.videos, video_metadata = map(list, zip(*videos_processed))
+            # transformer requires the video inputs to be under this format
+            base_output.videos = [base_output.videos]
+            video_metadata = [video_metadata]
+
+        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens, video_metadata=video_metadata
+        )
+
+        input_ids = input_ids.flatten()
+        mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v(
+            input_ids=input_ids.unsqueeze(0),
+            hf_config=self.hf_config,
+            image_grid_thw=getattr(ret, "image_grid_thw", None),
+            video_grid_thw=getattr(ret, "video_grid_thw", None),
+            attention_mask=getattr(ret, "attention_mask", None),
+        )
+        mrope_positions = mrope_positions.squeeze(1)
+
+        mm_inputs = {
+            "input_ids": input_ids.tolist(),
+            "mm_items": mm_items,
+            "im_token_id": self.mm_tokens.image_token_id,
+            "video_token_id": self.mm_tokens.video_token_id,
+            "mrope_positions": mrope_positions,
+            "mrope_position_delta": mrope_position_delta,
+        }
+
+        return mm_inputs

sglang/srt/multimodal/processors/qwen_audio.py

@@ -1,6 +1,6 @@
 import re
 
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.managers.schedule_batch import Modality
 from sglang.srt.models.qwen2_audio import Qwen2AudioForConditionalGeneration
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor,
@@ -29,6 +29,8 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
             audio_token_id=self.audio_token_id,
         ).build(_processor)
 
+        self.ATTR_NAME_TO_MODALITY.update({"feature_attention_mask": Modality.AUDIO})
+
     async def process_mm_data_async(
         self,
         audio_data,
@@ -54,7 +56,7 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
         input_lengths = (input_lengths - 1) // 2 + 1
         output_lengths = (input_lengths - 2) // 2 + 1
 
-        mm_items[0].model_specific_data["audio_feature_lens"] = output_lengths
+        mm_items[0].audio_feature_lens = output_lengths
 
         return {
             "mm_items": mm_items,

sglang/srt/operations.py

@@ -1,10 +1,17 @@
+from __future__ import annotations
+
 import os
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, Generator, List, Sequence, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Sequence, Union
 
 import torch
 
+from sglang.srt.layers.dp_attention import set_dp_buffer_len
+
+if TYPE_CHECKING:
+    from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+
 _ENABLE_PROFILE = bool(int(os.environ.get("SGLANG_OPERATIONS_ENABLE_PROFILE", "0")))
 
 if _ENABLE_PROFILE:
@@ -66,18 +73,26 @@ Stage = List[ExecutionOperation]
 
 
 class _StageExecutor:
-    def __init__(self, debug_name: str, stages: List[Stage], inputs):
+    def __init__(self, debug_name: str, stages: List[Stage], inputs: dict):
         self._debug_name = debug_name
         self._stages = stages
         self._index = 0
         self._stage_state = _StateDict()
         self._stage_output = inputs
 
+        # handling DP attention
+        forward_batch: ForwardBatch = inputs["forward_batch"]
+        self._global_dp_buffer_len = forward_batch.global_dp_buffer_len
+        self._local_dp_buffer_len = forward_batch.input_ids.shape[0]
+
     def next(self):
         assert not self.done
 
         stage = self._stages[self._index]
 
+        if self._global_dp_buffer_len is not None:
+            set_dp_buffer_len(self._global_dp_buffer_len, self._local_dp_buffer_len)
+
         with _annotate_region(debug_name=f"{self._debug_name}{self._index}"):
             for op in stage:
                 with _annotate_region(debug_name=op.debug_name):

sglang/srt/reasoning_parser.py

@@ -1,3 +1,4 @@
+import re
 from typing import Dict, Optional, Tuple, Type
 
 
@@ -185,6 +186,320 @@ class KimiDetector(BaseReasoningFormatDetector):
         )
 
 
+class GptOssDetector(BaseReasoningFormatDetector):
+    """
+    Detector for T4-style reasoning format.
+
+    Assumes reasoning format with two channels:
+      <|channel|>analysis<|message|>...reasoning content...<|end|>
+      <|start|>assistant<|channel|>final<|message|>...final answer...<|return|>
+
+    Returns content from 'analysis' channel as reasoning_text
+    and content from 'final' channel as normal_text.
+
+    Args:
+        stream_reasoning (bool): If False, accumulates reasoning content until complete.
+            If True, streams reasoning content as it arrives.
+    """
+
+    def __init__(self, stream_reasoning: bool = True, force_reasoning: bool = True):
+        # TypeScript uses channel tokens instead of simple start/end tokens
+        super().__init__(
+            "<|channel|>analysis<|message|>",
+            "<|end|>",
+            force_reasoning=True,
+            stream_reasoning=stream_reasoning,
+        )
+        self.final_channel_start = "<|start|>assistant<|channel|>final<|message|>"
+        self.final_channel_end = "<|return|>"
+        self._in_final_channel = False
+        self._analysis_complete = False
+        self._in_reasoning = True
+
+    def detect_and_parse(self, text: str) -> StreamingParseResult:
+        """
+        One-time parsing: Detects and parses both analysis and final channels.
+        Tool call channels are preserved in normal_text for downstream processing.
+
+        HACK: Also handles simplified format where text starts with "analysis" and transitions
+        to "assistantfinal" without full channel markers.
+        """
+        # HACK: Handle simplified format (analysis...assistantfinal) without channel markers
+        if (
+            text.startswith("analysis")
+            and "assistantfinal" in text
+            and "<|channel|>" not in text
+        ):
+            # Split on "assistantfinal"
+            parts = text.split("assistantfinal", 1)
+            self._in_reasoning = False
+            if len(parts) == 2:
+                reasoning_text = parts[0][
+                    len("analysis") :
+                ].strip()  # Remove "analysis" prefix
+                normal_text = parts[1].strip()
+                return StreamingParseResult(
+                    normal_text=normal_text, reasoning_text=reasoning_text
+                )
+
+        reasoning_parts = []
+        normal_parts = []
+        current_pos = 0
+
+        # Process text sequentially to preserve tool calls between analysis sections
+        while current_pos < len(text):
+            # Look for next analysis channel
+            analysis_start_idx = text.find(self.think_start_token, current_pos)
+
+            if analysis_start_idx == -1:
+                # No more analysis channels, rest goes to remaining
+                break
+
+            # Preserve any content before this analysis channel (could include tool calls)
+            if analysis_start_idx > current_pos:
+                between_content = text[current_pos:analysis_start_idx]
+                # This content will be added to normal_parts later
+                normal_parts.append(between_content)
+
+            # Extract analysis content
+            analysis_content_start = analysis_start_idx + len(self.think_start_token)
+            analysis_end_idx = text.find(self.think_end_token, analysis_content_start)
+
+            if analysis_end_idx != -1:
+                reasoning_parts.append(
+                    text[analysis_content_start:analysis_end_idx].strip()
+                )
+                current_pos = analysis_end_idx + len(self.think_end_token)
+            else:
+                # Analysis not complete
+                reasoning_parts.append(text[analysis_content_start:].strip())
+                reasoning_text = "".join(reasoning_parts)
+                return StreamingParseResult(reasoning_text=reasoning_text)
+
+        # Add any remaining text after all analysis sections
+        if current_pos < len(text):
+            remaining = text[current_pos:]
+            normal_parts.append(remaining)
+
+        # Process non-analysis content for commentary sections
+        full_normal_text = "".join(normal_parts)
+
+        # Extract reasoning from non-tool-call commentary sections
+        # Tool calls have "to=" in their header, regular commentary does not
+        commentary_pattern = re.compile(
+            r"<\|start\|>assistant<\|channel\|>commentary<\|message\|>(.*?)(?:<\|end\|>|<\|call\|>)",
+            re.DOTALL,
+        )
+
+        cleaned_text = full_normal_text
+        for match in reversed(list(commentary_pattern.finditer(full_normal_text))):
+            # Check if this commentary is a tool call by looking at the text before <|message|>
+            match_start = match.start()
+            # Find where "<|channel|>commentary" starts within the matched pattern
+            # The pattern starts with "<|start|>assistant<|channel|>commentary"
+            # So we look for the text between "commentary" and "<|message|>" in the match
+            match_text = full_normal_text[match_start : match.end()]
+            commentary_idx = match_text.find("<|channel|>commentary")
+            if commentary_idx != -1:
+                message_idx = match_text.find("<|message|>", commentary_idx)
+                if message_idx != -1:
+                    between_text = match_text[commentary_idx:message_idx]
+                    # If no "to=" found, this is regular commentary (reasoning content)
+                    if " to=" not in between_text:
+                        content = match.group(1).strip()
+                        reasoning_parts.append(content)
+                        # Remove this commentary section from normal text
+                        cleaned_text = (
+                            cleaned_text[: match.start()] + cleaned_text[match.end() :]
+                        )
+
+        full_normal_text = cleaned_text
+
+        # Combine all reasoning parts
+        reasoning_text = "".join(reasoning_parts)
+
+        # Process full_normal_text for final output
+        normal_text = ""
+        if self.final_channel_start in full_normal_text:
+            final_start = full_normal_text.find(self.final_channel_start)
+            final_content_start = final_start + len(self.final_channel_start)
+            final_end = full_normal_text.find(
+                self.final_channel_end, final_content_start
+            )
+
+            if final_end != -1:
+                # Extract content before final channel (includes tool calls)
+                before_final = full_normal_text[:final_start].strip()
+                # Extract ONLY the final channel content (not the channel markers)
+                final_text = full_normal_text[final_content_start:final_end].strip()
+                # Extract content after final channel
+                after_final = full_normal_text[
+                    final_end + len(self.final_channel_end) :
+                ].strip()
+
+                # For tool calls + final answer: concatenate tool calls with final text
+                parts = []
+                if before_final:
+                    parts.append(before_final)
+                if final_text:
+                    parts.append(final_text)
+                if after_final:
+                    parts.append(after_final)
+                normal_text = " ".join(parts)
+            else:
+                # Final channel not complete - extract what we have
+                # Look for just <|channel|>final<|message|> without <|return|>
+                alt_final_start = full_normal_text.find("<|channel|>final<|message|>")
+                if alt_final_start != -1:
+                    before_alt_final = full_normal_text[:alt_final_start].strip()
+                    alt_final_content = full_normal_text[
+                        alt_final_start + len("<|channel|>final<|message|>") :
+                    ].strip()
+
+                    parts = []
+                    if before_alt_final:
+                        parts.append(before_alt_final)
+                    if alt_final_content:
+                        parts.append(alt_final_content)
+                    normal_text = " ".join(parts)
+                else:
+                    normal_text = full_normal_text.strip()
+        else:
+            # No final channel, treat all as normal text (includes tool calls)
+            normal_text = full_normal_text.strip()
+
+        return StreamingParseResult(
+            normal_text=normal_text, reasoning_text=reasoning_text
+        )
+
+    def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
+        """
+        Streaming incremental parsing for GPT-OSS format.
+
+        This is a simplified streaming implementation that accumulates content
+        and delegates to the non-streaming parser for complex multi-channel parsing.
+        TODO: Implement proper incremental parsing for better streaming performance.
+        """
+        self._buffer += new_text
+
+        if not self._in_reasoning:
+            return StreamingParseResult(normal_text=new_text)
+
+        # Check if we have complete sections to process
+        # For GPT-OSS, we need to wait for complete channel sections
+        # HACK: For now, use simplified approach - wait for key markers before processing
+        key_markers = ["<|end|>", "<|call|>", "<|return|>", "assistantfinal"]
+        has_complete_section = any(marker in self._buffer for marker in key_markers)
+
+        if not has_complete_section:
+            # Still accumulating, don't process yet
+            return StreamingParseResult()
+
+        # Handle simplified format (analysis...assistantfinal) with true incremental streaming
+        if (
+            "<|channel|>" not in self._buffer
+        ):  # Simplified format without channel markers
+            if self._buffer.startswith("analysis"):
+                # Check if we have the transition to assistantfinal
+                if "assistantfinal" in self._buffer:
+                    self._in_reasoning = False
+                    # Complete reasoning section - extract and stream it
+                    parts = self._buffer.split("assistantfinal", 1)
+                    reasoning_text = parts[0][len("analysis") :].strip()
+                    final_content = parts[1].strip()
+
+                    # Clear buffer and return both reasoning and final content
+                    self._buffer = ""
+                    return StreamingParseResult(
+                        reasoning_text=reasoning_text if self.stream_reasoning else "",
+                        normal_text=final_content,
+                    )
+                elif self.stream_reasoning:
+                    # Stream reasoning content incrementally as it arrives
+                    current_reasoning = self._buffer[len("analysis") :].strip()
+                    self._buffer = ""
+                    return StreamingParseResult(reasoning_text=current_reasoning)
+                else:
+                    # Wait for assistantfinal
+                    return StreamingParseResult()
+            elif self._buffer.startswith("assistantfinal"):
+                # Direct final content without analysis
+                final_content = self._buffer[len("assistantfinal") :].strip()
+                self._buffer = ""
+                return StreamingParseResult(normal_text=final_content)
+
+        # For full channel format, process sections as they complete
+        result = StreamingParseResult()
+
+        # Process complete analysis sections
+        while (
+            self.think_start_token in self._buffer
+            and self.think_end_token in self._buffer
+        ):
+            start_idx = self._buffer.find(self.think_start_token)
+            start_pos = start_idx + len(self.think_start_token)
+            end_pos = self._buffer.find(self.think_end_token, start_pos)
+
+            if end_pos != -1:
+                reasoning_content = self._buffer[start_pos:end_pos].strip()
+                if self.stream_reasoning and reasoning_content:
+                    result.reasoning_text += reasoning_content
+
+                # Remove processed analysis section
+                self._buffer = (
+                    self._buffer[:start_idx]
+                    + self._buffer[end_pos + len(self.think_end_token) :]
+                )
+            else:
+                break
+
+        # Process complete commentary sections
+        commentary_pattern = re.compile(
+            r"<\|start\|>assistant<\|channel\|>commentary<\|message\|>(.*?)(?:<\|end\|>|<\|call\|>)",
+            re.DOTALL,
+        )
+
+        for match in reversed(list(commentary_pattern.finditer(self._buffer))):
+            # Check if this is a tool call
+            start_pos = match.start()
+            commentary_content = match.group(1).strip()
+            if self.stream_reasoning and commentary_content:
+                result.reasoning_text += commentary_content
+
+            # Remove this commentary section
+            self._buffer = self._buffer[: match.start()] + self._buffer[match.end() :]
+            # Clean up any standalone <|start|>assistant
+            self._buffer = re.sub(
+                r"<\|start\|>assistant(?=<\|start\|>assistant)", "", self._buffer
+            )
+
+        # Handle final channel completion
+        if self.final_channel_start in self._buffer:
+            final_start = self._buffer.find(self.final_channel_start)
+            final_content_start = final_start + len(self.final_channel_start)
+
+            # Check if final channel is complete
+            final_end = self._buffer.find(self.final_channel_end, final_content_start)
+            if final_end != -1:
+                # Complete final channel - process everything
+                final_result = self.detect_and_parse(self._buffer)
+                self._buffer = ""
+                return StreamingParseResult(
+                    normal_text=final_result.normal_text,
+                    reasoning_text=result.reasoning_text + final_result.reasoning_text,
+                )
+            else:
+                # Extract content before final channel (e.g. tool calls)
+                before_final = self._buffer[:final_start]
+                if before_final:
+                    # Output tool calls for processing
+                    result.normal_text += before_final
+                    # Keep the final channel part in buffer
+                    self._buffer = self._buffer[final_start:]
+
+        return result
+
+
 class ReasoningParser:
     """
     Parser that handles both streaming and non-streaming scenarios for extracting
@@ -203,6 +518,7 @@ class ReasoningParser:
         "glm45": Qwen3Detector,
         "kimi": KimiDetector,
         "step3": DeepSeekR1Detector,
+        "gpt-oss": GptOssDetector,
     }
 
     def __init__(

sglang/srt/sampling/sampling_batch_info.py

@@ -68,6 +68,8 @@ class SamplingBatchInfo:
 
     @classmethod
     def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int):
+        from sglang.srt.managers.schedule_batch import global_server_args_dict
+
         reqs = batch.reqs
         device = batch.device
         temperatures = (
@@ -97,10 +99,11 @@ class SamplingBatchInfo:
                         logit_bias[i, int(key)] = value
 
         # Check if any request has custom logit processor
-        has_custom_logit_processor = (
-            batch.enable_custom_logit_processor  # check the flag first.
-            and any(r.custom_logit_processor for r in reqs)  # then check the requests.
-        )
+        has_custom_logit_processor = global_server_args_dict[
+            "enable_custom_logit_processor"
+        ] and any(  # check the flag first.
+            r.custom_logit_processor for r in reqs
+        )  # then check the requests.
 
         if has_custom_logit_processor:
             # Merge the same type of custom logit processors together