sglang 0.4.4.post1__py3-none-any.whl → 0.4.4.post2__py3-none-any.whl

sglang/srt/managers/image_processor.py

@@ -1,55 +0,0 @@
-# TODO: also move pad_input_ids into this module
-import importlib
-import logging
-import pkgutil
-from functools import lru_cache
-
-from transformers import IMAGE_PROCESSOR_MAPPING
-
-from sglang.srt.managers.image_processors.base_image_processor import (
-    BaseImageProcessor,
-    DummyImageProcessor,
-)
-from sglang.srt.server_args import ServerArgs
-
-logger = logging.getLogger(__name__)
-
-
-IMAGE_PROCESSOR_MAPPING = {}
-
-
-def get_image_processor(
-    hf_config, server_args: ServerArgs, processor
-) -> BaseImageProcessor:
-    for model_cls, processor_cls in IMAGE_PROCESSOR_MAPPING.items():
-        if model_cls.__name__ in hf_config.architectures:
-            return processor_cls(hf_config, server_args, processor)
-    raise ValueError(
-        f"No image processor found for architecture: {hf_config.architectures}"
-    )
-
-
-def get_dummy_image_processor():
-    return DummyImageProcessor()
-
-
-@lru_cache()
-def import_image_processors():
-    package_name = "sglang.srt.managers.image_processors"
-    package = importlib.import_module(package_name)
-    for _, name, ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
-        if not ispkg:
-            try:
-                module = importlib.import_module(name)
-            except Exception as e:
-                logger.warning(f"Ignore import error when loading {name}: " f"{e}")
-                continue
-            if hasattr(module, "ImageProcessorMapping"):
-                entry = module.ImageProcessorMapping
-                if isinstance(entry, dict):
-                    for processor_name, cls in entry.items():
-                        IMAGE_PROCESSOR_MAPPING[processor_name] = cls
-
-
-# also register processors
-import_image_processors()

sglang/srt/managers/image_processors/base_image_processor.py

@@ -1,219 +0,0 @@
-import concurrent
-import concurrent.futures
-import dataclasses
-import multiprocessing as mp
-import os
-from abc import ABC, abstractmethod
-from typing import Optional
-
-import PIL
-import transformers
-from decord import VideoReader, cpu
-from PIL import Image
-
-from sglang.srt.server_args import ServerArgs
-from sglang.srt.utils import load_image
-from sglang.utils import logger
-
-global global_processor
-
-
-def get_global_processor():
-    global global_processor
-    return global_processor
-
-
-def init_global_processor(sglang_image_processor, server_args: ServerArgs):
-    """Init the global processor for multi-modal models."""
-    global global_processor
-    transformers.logging.set_verbosity_error()
-    global_processor = sglang_image_processor._build_processor(server_args=server_args)
-
-
-@dataclasses.dataclass
-class BaseImageProcessorOutput:
-    image_hashes: list[int]
-    image_sizes: list[tuple[int, int]]
-    all_frames: [PIL.Image]
-    # input_text, with each frame of video/image represented as an image_token
-    input_text: str
-
-
-class BaseImageProcessor(ABC):
-    def __init__(self, hf_config, server_args, _processor):
-        self.hf_config = hf_config
-        self._processor = _processor
-        self.server_args = server_args
-        # FIXME: not accurate, model and image specific
-        self.NUM_TOKEN_PER_FRAME = 330
-
-        self.executor = concurrent.futures.ProcessPoolExecutor(
-            initializer=init_global_processor,
-            mp_context=mp.get_context("fork"),
-            initargs=(
-                self,
-                server_args,
-            ),
-            max_workers=int(os.environ.get("SGLANG_CPU_COUNT", os.cpu_count())),
-        )
-
-    def _build_processor(self, server_args):
-        """Init the global processor for multi modal models."""
-        from sglang.srt.hf_transformers_utils import get_processor
-
-        return get_processor(
-            server_args.tokenizer_path,
-            tokenizer_mode=server_args.tokenizer_mode,
-            trust_remote_code=server_args.trust_remote_code,
-        )
-
-    @abstractmethod
-    async def process_images_async(
-        self, image_data, input_text, max_req_input_len, **kwargs
-    ):
-        pass
-
-    def get_estimated_frames_list(self, image_data):
-        """
-        estimate the total frame count from all visual input
-        """
-        # Before processing inputs
-        estimated_frames_list = []
-        for image in image_data:
-            if isinstance(image, str) and image.startswith("video:"):
-                path = image[len("video:") :]
-                # Estimate frames for the video
-                vr = VideoReader(path, ctx=cpu(0))
-                num_frames = len(vr)
-            else:
-                # For images, each contributes one frame
-                num_frames = 1
-            estimated_frames_list.append(num_frames)
-
-        return estimated_frames_list
-
-    @staticmethod
-    def encode_video(video_path, frame_count_limit=None):
-        if not os.path.exists(video_path):
-            logger.error(f"Video {video_path} does not exist")
-            return []
-
-        if frame_count_limit == 0:
-            return []
-
-        def uniform_sample(l, n):
-            gap = len(l) / n
-            idxs = [int(i * gap + gap / 2) for i in range(n)]
-            return [l[i] for i in idxs]
-
-        vr = VideoReader(video_path, ctx=cpu(0))
-        sample_fps = round(vr.get_avg_fps() / 1)  # FPS
-        frame_indices = [i for i in range(0, len(vr), sample_fps)]
-        if frame_count_limit is not None and len(frame_indices) > frame_count_limit:
-            frame_indices = uniform_sample(frame_indices, frame_count_limit)
-
-        frames = vr.get_batch(frame_indices).asnumpy()
-        frames = [Image.fromarray(v.astype("uint8")) for v in frames]
-        return frames
-
-    def load_images(
-        self,
-        input_ids: list,
-        image_data,
-        image_token: str,
-        max_req_input_len: int,
-        return_text: Optional[bool] = True,
-        discard_alpha_channel: bool = True,
-    ) -> BaseImageProcessorOutput:
-        """
-        Each frame of video/image will be replaced by a single image token
-
-        Args:
-
-            discard_alpha_channel: if True, discards the alpha channel in the returned images
-
-        """
-        image_hashes, image_sizes = [], []
-        all_frames = []
-        new_text_parts = []
-
-        if isinstance(input_ids, list) and return_text:
-            assert len(input_ids) and isinstance(input_ids[0], int)
-            input_text = self._processor.tokenizer.decode(input_ids)
-        else:
-            input_text = input_ids
-
-        if return_text:
-            text_parts = input_text.split(image_token)
-
-        # TODO(mick): load from server_args, env, or sampling_params
-        MAX_NUM_FRAMES = 30
-        estimated_frames_list = self.get_estimated_frames_list(image_data=image_data)
-        total_frame_count = sum(estimated_frames_list)
-        # a heuristic value, suggesting the maximum fraction of frames to embed from all visual inputs.
-        # e.g., 0.1 suggests that 1 frame out of 10 input frames should be used
-        scaling_factor = min(1.0, MAX_NUM_FRAMES / total_frame_count)
-
-        assert len(image_data) == len(estimated_frames_list)
-
-        # Process each input with allocated frames
-        for image_index, (image, estimated_frames) in enumerate(
-            zip(image_data, estimated_frames_list)
-        ):
-            if len(all_frames) >= MAX_NUM_FRAMES:
-                max_frames_to_process = 0
-            else:
-                max_frames_to_process = max(1, int(estimated_frames * scaling_factor))
-
-            if max_frames_to_process == 0:
-                frames = []
-            else:
-                try:
-                    if isinstance(image, str) and image.startswith("video:"):
-                        path = image[len("video:") :]
-                        frames = BaseImageProcessor.encode_video(
-                            path, frame_count_limit=max_frames_to_process
-                        )
-                    else:
-                        raw_image, _size = load_image(image)
-                        if discard_alpha_channel:
-                            raw_image = raw_image.convert("RGB")
-                        frames = [raw_image]
-                    assert len(frames) != 0
-                except FileNotFoundError as e:
-                    print(e)
-                    return None
-
-                image_sizes += [frames[0].size] * len(frames)
-                image_hashes += [hash(image)] * len(frames)
-                all_frames += frames
-
-            if return_text:
-                new_text_parts.append(text_parts[image_index])
-            if max_frames_to_process != 0:
-                new_text_parts.append(image_token * len(frames))
-            assert max_frames_to_process >= len(frames)
-        if return_text:
-            new_text_parts.append(text_parts[-1])
-
-        input_text = "".join(new_text_parts)
-        return BaseImageProcessorOutput(
-            image_hashes, image_sizes, all_frames, input_text
-        )
-
-
-class DummyImageProcessor(BaseImageProcessor):
-    def __init__(self):
-        pass
-
-    async def process_images_async(self, *args, **kwargs):
-        return None
-
-
-def init_global_processor(
-    sglang_image_processor: BaseImageProcessor, server_args: ServerArgs
-):
-    """Init the global processor for multi-modal models."""
-    global global_processor
-    transformers.logging.set_verbosity_error()
-    global_processor = sglang_image_processor._build_processor(server_args=server_args)

sglang/srt/managers/image_processors/minicpmv.py

@@ -1,86 +0,0 @@
-import asyncio
-from typing import List, Union
-
-from sglang.srt.managers.image_processor import BaseImageProcessor
-from sglang.srt.managers.image_processors.base_image_processor import (
-    get_global_processor,
-)
-from sglang.srt.models.minicpmv import MiniCPMV
-
-
-class MiniCPMVImageProcessor(BaseImageProcessor):
-    def __init__(self, hf_config, server_args, _processor):
-        super().__init__(hf_config, server_args, _processor)
-        self.IMAGE_TOKEN = "(<image>./</image>)"
-
-    @staticmethod
-    def _process_images_task(images, input_text):
-        processor = get_global_processor()
-        result = processor.__call__(text=input_text, images=images, return_tensors="pt")
-        return {
-            "input_ids": result.input_ids,
-            "pixel_values": result.pixel_values,
-            "tgt_sizes": result.tgt_sizes,
-        }
-
-    async def _process_images(self, images, input_text):
-        if self.executor is not None:
-            loop = asyncio.get_event_loop()
-            image_inputs = await loop.run_in_executor(
-                self.executor,
-                MiniCPMVImageProcessor._process_images_task,
-                images,
-                input_text,
-            )
-        else:
-            image_inputs = self._processor(
-                images=images, text=input_text, return_tensors="pt"
-            )
-
-        return image_inputs
-
-    async def process_images_async(
-        self,
-        image_data: List[Union[str, bytes]],
-        input_ids,
-        request_obj,
-        max_req_input_len,
-    ):
-        if not image_data:
-            return None
-        if not isinstance(image_data, list):
-            image_data = [image_data]
-
-        base_output = self.load_images(
-            input_ids, image_data, self.IMAGE_TOKEN, max_req_input_len
-        )
-        if base_output is None:
-            return None
-
-        if len(base_output.all_frames) == 0:
-            return None
-        res = await self._process_images(
-            images=base_output.all_frames, input_text=base_output.input_text
-        )
-
-        # Collect special token ids
-        tokenizer = self._processor.tokenizer
-        im_start_id = tokenizer.im_start_id
-        im_end_id = tokenizer.im_end_id
-        if tokenizer.slice_start_id:
-            slice_start_id = tokenizer.slice_start_id
-            slice_end_id = tokenizer.slice_end_id
-        return {
-            "input_ids": res["input_ids"].flatten().tolist(),
-            "pixel_values": res["pixel_values"],
-            "tgt_sizes": res["tgt_sizes"],
-            "image_hashes": base_output.image_hashes,
-            "modalities": request_obj.modalities or ["image"],
-            "im_start_id": im_start_id,
-            "im_end_id": im_end_id,
-            "slice_start_id": slice_start_id,
-            "slice_end_id": slice_end_id,
-        }
-
-
-ImageProcessorMapping = {MiniCPMV: MiniCPMVImageProcessor}

sglang/srt/managers/multi_modality_padding.py

@@ -1,134 +0,0 @@
-from abc import abstractmethod
-from typing import Callable, List, Optional, Tuple
-
-from sglang.srt.managers.schedule_batch import ImageInputs
-from sglang.utils import logger
-
-
-class MultiModalityDataPaddingPattern:
-    """
-    Data tokens (like image tokens) often need special handling during padding
-    to maintain model compatibility. This class provides the interface for
-    implementing different padding strategies for data tokens
-    """
-
-    @abstractmethod
-    def pad_input_tokens(
-        self, input_ids: List[int], image_inputs: ImageInputs
-    ) -> List[int]:
-        """
-        Pad the input ids sequence containing data tokens, and replace them with pad_values
-        """
-        pass
-
-
-class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern):
-    """In this pattern, data tokens should be enclosed by special token pairs (e.g. <image>...</image>, data_token_pairs)
-
-    This strategy should be applied when data content is marked by start/end token pairs in the input sequence.
-    """
-
-    def __init__(self, data_token_pairs: Optional[List[Tuple[int, int]]]) -> None:
-        self.data_token_id_pairs = data_token_pairs
-
-    def pad_input_tokens(
-        self, input_ids: List[int], image_inputs: ImageInputs
-    ) -> List[int]:
-        """
-        This function will replace the data-tokens inbetween with pad_values accordingly
-        """
-        pad_values = image_inputs.pad_values
-        data_token_pairs = self.data_token_id_pairs
-        image_inputs.image_offsets = []
-        if data_token_pairs is None:
-            data_token_pairs = [image_inputs.im_start_id, image_inputs.im_end_id]
-        if data_token_pairs is None:
-            logger.warning(
-                "No data_token_pairs provided, RadixAttention might be influenced."
-            )
-            return input_ids
-        start_token_ids = [s for s, _e in data_token_pairs]
-        end_tokens_ids = [e for _s, e in data_token_pairs]
-        # First start token marks new data
-        data_start_token = start_token_ids[0]
-
-        padded_ids = []
-        last_idx = 0
-        data_idx = -1
-
-        start_indices = [i for i, x in enumerate(input_ids) if x in start_token_ids]
-        end_indices = [i for i, x in enumerate(input_ids) if x in end_tokens_ids]
-
-        if len(start_indices) != len(end_indices):
-            return input_ids
-
-        for start_idx, end_idx in zip(start_indices, end_indices):
-            padded_ids.extend(input_ids[last_idx : start_idx + 1])
-
-            if input_ids[start_idx] == data_start_token:
-                data_idx += 1
-                image_inputs.image_offsets += [start_idx]
-
-            num_tokens = end_idx - start_idx - 1
-            pad_value = pad_values[data_idx]
-            padded_ids.extend([pad_value] * num_tokens)
-
-            last_idx = end_idx
-
-        padded_ids.extend(input_ids[last_idx:])
-
-        assert len(input_ids) == len(padded_ids)
-        return padded_ids
-
-
-class MultModalityDataPaddingPatternSingleToken(MultiModalityDataPaddingPattern):
-    """In this pattern, data is represented with a special token_id ( image_inputs.im_token_id ),
-         which needs first to be expanded to multiple tokens, then replaced with their padding values
-
-    This strategy should be used when a single data token represents content that should
-    be expanded to multiple tokens during processing.
-    """
-
-    def __init__(
-        self, num_data_token_calc_func: Callable[[Tuple[int, int, int]], int]
-    ) -> None:
-        self.num_data_token_calc_func = num_data_token_calc_func
-
-    def pad_input_tokens(
-        self, input_ids: List[int], image_inputs: ImageInputs
-    ) -> List[int]:
-        """
-        This function will follow the procedure of:
-            1. the data token will be expanded, of which the final number will be calculated by `num_data_token_calc_func`
-            2. the padded data tokens will be replaced with their pad_values
-        """
-        image_grid_thws = image_inputs.image_grid_thws
-        pad_values = image_inputs.pad_values
-
-        image_indices = [
-            idx
-            for idx, token in enumerate(input_ids)
-            if token == image_inputs.im_token_id
-        ]
-
-        image_inputs.image_offsets = []
-
-        input_ids_with_image = []
-        for image_cnt, _ in enumerate(image_grid_thws):
-            print(f"image_cnt {image_cnt}")
-            num_image_tokens = self.num_data_token_calc_func(image_grid_thws[image_cnt])
-            if image_cnt == 0:
-                non_image_tokens = input_ids[: image_indices[image_cnt]]
-            else:
-                non_image_tokens = input_ids[
-                    image_indices[image_cnt - 1] + 1 : image_indices[image_cnt]
-                ]
-            input_ids_with_image.extend(non_image_tokens)
-            image_inputs.image_offsets.append(len(input_ids_with_image))
-            pad_ids = pad_values * (
-                (num_image_tokens + len(pad_values)) // len(pad_values)
-            )
-            input_ids_with_image.extend(pad_ids[:num_image_tokens])
-        input_ids_with_image.extend(input_ids[image_indices[-1] + 1 :])
-
-        return input_ids_with_image