sglang 0.4.9.post2__py3-none-any.whl → 0.4.9.post3__py3-none-any.whl

sglang/srt/layers/quantization/quant_utils.py

@@ -1,166 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/utils/quant_utils.py
-
-from typing import Optional
-
-import numpy
-import torch
-from sgl_kernel.scalar_type import ScalarType
-
-
-def get_pack_factor(num_bits):
-    assert 32 % num_bits == 0, f"Unsupported num_bits = {num_bits}"
-    return 32 // num_bits
-
-
-def pack_cols(
-    q_w: torch.Tensor,
-    num_bits: int,
-    size_k: int,
-    size_n: int,
-):
-    assert q_w.shape == (size_k, size_n)
-
-    pack_factor = get_pack_factor(num_bits)
-    assert size_n % pack_factor == 0
-
-    orig_device = q_w.device
-
-    q_w = q_w.cpu().numpy().astype(numpy.uint32)
-
-    q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32)
-
-    for i in range(pack_factor):
-        q_res |= q_w[:, i::pack_factor] << num_bits * i
-
-    q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
-    q_res = q_res.contiguous()
-
-    return q_res
-
-
-def unpack_cols(
-    packed_q_w: torch.Tensor,
-    num_bits: int,
-    size_k: int,
-    size_n: int,
-):
-    pack_factor = get_pack_factor(num_bits)
-    assert size_n % pack_factor == 0
-    assert packed_q_w.shape == (
-        size_k,
-        size_n // pack_factor,
-    ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
-        packed_q_w.shape, size_k, size_n, pack_factor
-    )
-
-    orig_device = packed_q_w.device
-
-    packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32)
-    q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32)
-
-    mask = (1 << num_bits) - 1
-    for i in range(pack_factor):
-        vals = packed_q_w_cpu & mask
-        packed_q_w_cpu >>= num_bits
-        q_res[:, i::pack_factor] = vals
-
-    q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
-    q_res = q_res.contiguous()
-
-    return q_res
-
-
-def quantize_weights(
-    w: torch.Tensor,
-    quant_type: ScalarType,
-    group_size: Optional[int],
-    zero_points: bool = False,
-    ref_zero_points_after_scales: bool = False,
-):
-    assert (
-        quant_type.is_integer()
-    ), "Floating point quantization may work but has not been tested"
-    assert not zero_points or group_size is not None, (
-        "to have group zero points, group_size must be provided "
-        "(-1 group_size is channelwise)"
-    )
-
-    orig_device = w.device
-    orig_type = w.dtype
-    size_k, size_n = w.shape
-
-    assert w.is_floating_point(), "w must be float"
-
-    if group_size == -1:
-        group_size = size_k
-
-    # Reshape to [groupsize, -1]
-    if group_size is not None and group_size < size_k:
-        w = w.reshape((-1, group_size, size_n))
-        w = w.permute(1, 0, 2)
-        w = w.reshape((group_size, -1))
-
-    # Compute scale for each group
-    max_val = torch.max(w, 0, keepdim=True).values
-    min_val = torch.min(w, 0, keepdim=True).values
-
-    max_q_val = quant_type.max()
-    min_q_val = quant_type.min()
-
-    w_s = torch.Tensor([1.0]).to(w.device)  # unscaled case
-    maybe_w_zp = None
-    if group_size is not None:
-        if zero_points:
-            assert not quant_type.is_signed() and quant_type.max() > 0
-            w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
-            maybe_w_zp = (
-                torch.round(torch.abs(min_val / w_s)).clamp(min_q_val, max_q_val).int()
-            )
-        else:
-            # If the bias is such that there are no possible negative/positive
-            #  values, set the max value to inf to avoid divide by 0
-            w_s = torch.max(
-                abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
-                abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)),
-            )
-
-    # Quantize
-    w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
-    w_q = torch.clamp(w_q, min_q_val, max_q_val)
-
-    # Compute ref (dequantized)
-    # For some kernels (namely Machete) the zero-points are applied after the
-    # scales are applied, for this case computing the reference in similar way
-    # allows us to use tighter error tolerances in our unit tests.
-    if ref_zero_points_after_scales and maybe_w_zp is not None:
-        w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s
-    else:
-        w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s
-
-    if quant_type.has_bias():
-        w_q += quant_type.bias
-
-    # Restore original shapes
-    if group_size is not None and group_size < size_k:
-
-        def reshape_w(w):
-            w = w.reshape((group_size, -1, size_n))
-            w = w.permute(1, 0, 2)
-            w = w.reshape((size_k, size_n)).contiguous()
-            return w
-
-        w_q = reshape_w(w_q)
-        w_ref = reshape_w(w_ref)
-        w_s = w_s.reshape((-1, size_n)).contiguous()
-
-    if maybe_w_zp is not None:
-        maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous()
-        maybe_w_zp = maybe_w_zp.to(device=orig_device)
-
-    return (
-        w_ref.to(device=orig_device),
-        w_q.to(device=orig_device),
-        w_s if group_size is not None else None,
-        maybe_w_zp,
-    )

sglang/srt/managers/multimodal_processors/qwen_audio.py

@@ -1,94 +0,0 @@
-import re
-from typing import List, Union
-
-import torch
-
-from sglang.srt.managers.multimodal_processors.base_processor import (
-    BaseMultimodalProcessor,
-    MultimodalSpecialTokens,
-)
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
-from sglang.srt.models.qwen2_audio import Qwen2AudioForConditionalGeneration
-
-
-class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
-    models = [Qwen2AudioForConditionalGeneration]
-
-    def __init__(self, hf_config, server_args, _processor):
-        super().__init__(hf_config, server_args, _processor)
-        self.AUDIO_TOKEN = "<|audio_bos|><|AUDIO|><|audio_eos|>"
-        self.AUDIO_TOKEN_REGEX = re.compile(
-            r"<\|audio_bos\|>(?:<\|AUDIO\|>)+<\|audio_eos\|>"
-        )
-
-    async def process_mm_data_async(
-        self,
-        image_data: List[Union[str, bytes]],
-        input_text,
-        request_obj,
-        max_req_input_len,
-        **kwargs,
-    ):
-        audio_data = request_obj.audio_data
-        if not isinstance(audio_data, list):
-            audio_data = [audio_data]
-
-        base_output = self.load_mm_data(
-            prompt=input_text,
-            max_req_input_len=max_req_input_len,
-            audio_data=audio_data,
-            multimodal_tokens=MultimodalSpecialTokens(
-                audio_token=self.AUDIO_TOKEN,
-                audio_token_regex=self.AUDIO_TOKEN_REGEX,
-            ),
-        )
-        if base_output is None:
-            return None
-
-        res = self.process_mm_data(
-            input_text=base_output.input_text,
-            audio=base_output.audios,
-        )
-
-        # Collect special token ids
-        tokenizer = self._processor.tokenizer
-        audio_start_id = tokenizer.convert_tokens_to_ids("<|audio_bos|>")
-        audio_token_id = tokenizer.convert_tokens_to_ids("<|AUDIO|>")
-        audio_end_id = tokenizer.convert_tokens_to_ids("<|audio_eos|>")
-
-        items = []
-        input_ids = res["input_ids"].flatten()
-
-        if (
-            "input_features" in res
-            and res["input_features"] is not None
-            and len(res["input_features"]) != 0
-        ):
-            if audio_start_id is not None and audio_end_id is not None:
-                audio_offsets = self.get_mm_items_offset_by_pair(
-                    input_ids=input_ids,
-                    mm_start_id=audio_start_id,
-                    mm_end_id=audio_end_id,
-                )
-            else:
-                audio_offsets = None
-
-            input_lengths = res["feature_attention_mask"].sum(dim=-1)
-            input_lengths = (input_lengths - 1) // 2 + 1
-            output_lengths = (input_lengths - 2) // 2 + 1
-
-            item = MultimodalDataItem(
-                audio_features=res["input_features"],
-                audio_feature_lens=output_lengths,
-                audio_offsets=audio_offsets,
-                modality=Modality.AUDIO,
-            )
-            items += [item]
-
-        return {
-            "mm_items": items,
-            "input_ids": input_ids.tolist(),
-            "audio_start_id": audio_start_id,
-            "audio_token_id": audio_token_id,
-            "audio_end_id": audio_end_id,
-        }