sglang 0.5.0rc0__py3-none-any.whl → 0.5.0rc2__py3-none-any.whl

sglang/srt/function_call/harmony_tool_parser.py

@@ -1,130 +0,0 @@
-# Copyright 2023-2024 SGLang Team
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Harmony tool call parser for processing tool calls in harmony models."""
-
-import uuid
-from typing import List, Optional, Tuple
-
-from sglang.srt.entrypoints.openai.protocol import (
-    ChatMessage,
-    FunctionResponse,
-    ToolCall,
-)
-
-
-class HarmonyToolCallParser:
-    """Parser for extracting tool calls from harmony model outputs."""
-
-    def extract_tool_calls_from_message(self, msg) -> Optional[ToolCall]:
-        """
-        Extract tool call from a single message if it's a tool call.
-
-        Args:
-            msg: The harmony message
-
-        Returns:
-            ToolCall if the message is a tool call, None otherwise
-        """
-        if (
-            msg.channel == "commentary"
-            and msg.recipient
-            and msg.recipient.startswith("functions.")
-        ):
-            function_name = msg.recipient.split(".")[-1]
-            arguments = msg.content[0].text if msg.content else "{}"
-
-            return ToolCall(
-                id=f"call_{uuid.uuid4().hex[:24]}",
-                function=FunctionResponse(
-                    name=function_name,
-                    arguments=arguments,
-                ),
-            )
-        return None
-
-    def process_streaming_chunk(
-        self,
-        harmony_parser,
-        index: int,
-        tool_call_trackers: dict,
-        stream_buffers: dict,
-    ) -> Tuple[Optional[dict], bool, Optional[str]]:
-        """
-        Process a streaming chunk for tool calls.
-
-        Args:
-            harmony_parser: The harmony parser instance
-            index: The choice index
-            tool_call_trackers: Dict tracking tool calls per choice
-            stream_buffers: Dict for buffering content
-
-        Returns:
-            Tuple of (tool_call_data, is_tool_call, delta)
-        """
-        # Check if we're in a tool call
-        is_tool_call = (
-            harmony_parser.current_channel == "commentary"
-            and harmony_parser.current_recipient
-            and harmony_parser.current_recipient.startswith("functions.")
-        )
-
-        delta = harmony_parser.last_content_delta or ""
-        tool_call_data = None
-
-        if is_tool_call:
-            # Handle tool call streaming
-            function_name = harmony_parser.current_recipient.split(".")[-1]
-
-            # Track tool call indices per choice
-            if index not in tool_call_trackers:
-                tool_call_trackers[index] = {"count": 0, "current_function": None}
-
-            # Check if we just started a new tool call
-            tool_call_tracker = tool_call_trackers[index]
-            if tool_call_tracker["current_function"] != function_name:
-                # New tool call started
-                tool_call_tracker["current_function"] = function_name
-                tool_call_index = tool_call_tracker["count"]
-                tool_call_tracker["count"] += 1
-
-                # Store the tool call index for this function
-                tool_call_key = f"{index}_{function_name}"
-                stream_buffers[tool_call_key] = {
-                    "index": tool_call_index,
-                    "content": "",
-                }
-
-                tool_call_data = {
-                    "id": f"call_{uuid.uuid4().hex[:24]}",
-                    "index": tool_call_index,
-                    "function_name": function_name,
-                    "arguments": delta,
-                    "is_first_chunk": True,
-                }
-            else:
-                # Subsequent chunks for the same tool call
-                tool_call_key = f"{index}_{function_name}"
-                tool_call_index = stream_buffers[tool_call_key]["index"]
-
-                tool_call_data = {
-                    "id": None,
-                    "index": tool_call_index,
-                    "function_name": None,
-                    "arguments": delta,
-                    "is_first_chunk": False,
-                }
-
-            stream_buffers[tool_call_key]["content"] += delta
-
-        return tool_call_data, is_tool_call, delta

sglang/srt/layers/quantization/scalar_type.py

@@ -1,352 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import functools
-import struct
-from dataclasses import dataclass
-from enum import Enum
-from typing import Optional, Union
-
-_SCALAR_TYPES_ID_MAP = {}
-
-
-# Mirrors enum in `core/scalar_type.hpp`
-class NanRepr(Enum):
-    NONE = 0  # nans are not supported
-    IEEE_754 = 1  # nans are: Exp all 1s, mantissa not all 0s
-    EXTD_RANGE_MAX_MIN = 2  # nans are: Exp all 1s, mantissa all 1s
-
-
-# This ScalarType class is a parallel implementation of the C++ ScalarType
-# class found in csrc/core/scalar_type.hpp.  These two classes should be kept
-# in sync until the inductor fully supports custom C++ classes.
-@dataclass(frozen=True)
-class ScalarType:
-    """
-    ScalarType can represent a wide range of floating point and integer
-    types, in particular it can be used to represent sub-byte data types
-    (something that torch.dtype currently does not support). It is also
-    capable of  representing types with a bias, i.e.:
-      `stored_value = value + bias`,
-    this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
-    of 8). The implementation for this class can be found in
-    csrc/core/scalar_type.hpp, these type signatures should be kept in sync
-    with that file.
-    """
-
-    exponent: int
-    """
-    Number of bits in the exponent if this is a floating point type
-    (zero if this an integer type)
-    """
-
-    mantissa: int
-    """
-    Number of bits in the mantissa if this is a floating point type,
-    or the number bits representing an integer excluding the sign bit if
-    this an integer type.
-    """
-
-    signed: bool
-    "If the type is signed (i.e. has a sign bit)"
-
-    bias: int
-    """
-    bias used to encode the values in this scalar type
-    (value = stored_value - bias, default 0) for example if we store the
-    type as an unsigned integer with a bias of 128 then the value 0 will be
-    stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
-    """
-
-    _finite_values_only: bool = False
-    """
-    Private: if infs are supported, used `has_infs()` instead.
-    """
-
-    nan_repr: NanRepr = NanRepr.IEEE_754
-    """
-    How NaNs are represent in this scalar type, returns NanRepr value.
-    (not applicable for integer types)
-    """
-
-    def _floating_point_max_int(self) -> int:
-        assert (
-            self.mantissa <= 52 and self.exponent <= 11
-        ), f"Cannot represent max/min as a double for type {self.__str__()}"
-
-        max_mantissa = (1 << self.mantissa) - 1
-        if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN:
-            max_mantissa = max_mantissa - 1
-
-        max_exponent = (1 << self.exponent) - 2
-        if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN or self.nan_repr == NanRepr.NONE:
-            assert (
-                self.exponent < 11
-            ), f"Cannot represent max/min as a double for type {self.__str__()}"
-            max_exponent = max_exponent + 1
-
-        # adjust the exponent to match that of a double
-        # for now we assume the exponent bias is the standard 2^(e-1) -1, (where
-        # e is the exponent bits), there is some precedent for non-standard
-        # biases, example `float8_e4m3b11fnuz` here:
-        # https://github.com/jax-ml/ml_dtypes but to avoid premature over
-        # complication we are just assuming the standard exponent bias until
-        # there is a need to support non-standard biases
-        exponent_bias = (1 << (self.exponent - 1)) - 1
-        exponent_bias_double = (1 << 10) - 1  # double e = 11
-
-        max_exponent_double = max_exponent - exponent_bias + exponent_bias_double
-
-        # shift the mantissa and exponent into the proper positions for an
-        # IEEE double and bitwise-or them together.
-        return (max_mantissa << (52 - self.mantissa)) | (max_exponent_double << 52)
-
-    def _floating_point_max(self) -> float:
-        double_raw = self._floating_point_max_int()
-        return struct.unpack("!d", struct.pack("!Q", double_raw))[0]
-
-    def _raw_max(self) -> Union[int, float]:
-        if self.is_floating_point():
-            return self._floating_point_max()
-        else:
-            assert (
-                self.size_bits < 64 or self.size_bits == 64 and self.is_signed()
-            ), "Cannot represent max as an int"
-            return (1 << self.mantissa) - 1
-
-    def _raw_min(self) -> Union[int, float]:
-        if self.is_floating_point():
-            assert (
-                self.is_signed()
-            ), "We currently assume all floating point types are signed"
-            sign_bit_double = 1 << 63
-
-            max_raw = self._floating_point_max_int()
-            min_raw = max_raw | sign_bit_double
-            return struct.unpack("!d", struct.pack("!Q", min_raw))[0]
-        else:
-            assert (
-                not self.is_signed() or self.size_bits <= 64
-            ), "Cannot represent min as a int64_t"
-
-            if self.is_signed():
-                return -(1 << (self.size_bits - 1))
-            else:
-                return 0
-
-    @functools.cached_property
-    def id(self) -> int:
-        """
-        Convert the ScalarType to an int which can be passed to pytorch custom
-        ops. This layout of the int must be kept in sync with the C++
-        ScalarType's from_id method.
-        """
-        val = 0
-        offset = 0
-
-        def or_and_advance(member, bit_width):
-            nonlocal val
-            nonlocal offset
-            bit_mask = (1 << bit_width) - 1
-            val = val | (int(member) & bit_mask) << offset
-            offset = offset + bit_width
-
-        or_and_advance(self.exponent, 8)
-        or_and_advance(self.mantissa, 8)
-        or_and_advance(self.signed, 1)
-        or_and_advance(self.bias, 32)
-        or_and_advance(self._finite_values_only, 1)
-        or_and_advance(self.nan_repr.value, 8)
-
-        assert offset <= 64, f"ScalarType fields too big {offset} to fit into an int64"
-
-        _SCALAR_TYPES_ID_MAP[val] = self
-
-        return val
-
-    @property
-    def size_bits(self) -> int:
-        return self.exponent + self.mantissa + int(self.signed)
-
-    def min(self) -> Union[int, float]:
-        """
-        Min representable value for this scalar type.
-        (accounting for bias if there is one)
-        """
-        return self._raw_min() - self.bias
-
-    def max(self) -> Union[int, float]:
-        """
-        Max representable value for this scalar type.
-        (accounting for bias if there is one)
-        """
-        return self._raw_max() - self.bias
-
-    def is_signed(self) -> bool:
-        """
-        If the type is signed (i.e. has a sign bit), same as `signed`
-        added for consistency with:
-        https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
-        """
-        return self.signed
-
-    def is_floating_point(self) -> bool:
-        "If the type is a floating point type"
-        return self.exponent != 0
-
-    def is_integer(self) -> bool:
-        "If the type is an integer type"
-        return self.exponent == 0
-
-    def has_bias(self) -> bool:
-        "If the type has a non-zero bias"
-        return self.bias != 0
-
-    def has_infs(self) -> bool:
-        "If the type is floating point and supports infinity"
-        return not self._finite_values_only
-
-    def has_nans(self) -> bool:
-        return self.nan_repr != NanRepr.NONE.value
-
-    def is_ieee_754(self) -> bool:
-        """
-        If the type is a floating point type that follows IEEE 754
-        conventions
-        """
-        return self.nan_repr == NanRepr.IEEE_754.value and not self._finite_values_only
-
-    def __str__(self) -> str:
-        """
-        naming generally follows: https://github.com/jax-ml/ml_dtypes
-        for floating point types (leading f) the scheme is:
-        `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
-        flags:
-          - no-flags: means it follows IEEE 754 conventions
-          - f: means finite values only (no infinities)
-          - n: means nans are supported (non-standard encoding)
-        for integer types the scheme is:
-          `[u]int<size_bits>[b<bias>]`
-          - if bias is not present it means its zero
-        """
-        if self.is_floating_point():
-            ret = (
-                "float"
-                + str(self.size_bits)
-                + "_e"
-                + str(self.exponent)
-                + "m"
-                + str(self.mantissa)
-            )
-
-            if not self.is_ieee_754():
-                if self._finite_values_only:
-                    ret = ret + "f"
-                if self.nan_repr != NanRepr.NONE:
-                    ret = ret + "n"
-
-            return ret
-        else:
-            ret = ("int" if self.is_signed() else "uint") + str(self.size_bits)
-            if self.has_bias():
-                ret = ret + "b" + str(self.bias)
-            return ret
-
-    def __repr__(self) -> str:
-        return "ScalarType." + self.__str__()
-
-    # __len__ needs to be defined (and has to throw TypeError) for pytorch's
-    # opcheck to work.
-    def __len__(self) -> int:
-        raise TypeError
-
-    #
-    # Convenience Constructors
-    #
-
-    @classmethod
-    def int_(cls, size_bits: int, bias: Optional[int]) -> "ScalarType":
-        "Create a signed integer scalar type (size_bits includes sign-bit)."
-        ret = cls(0, size_bits - 1, True, bias if bias else 0)
-        ret.id  # noqa B018: make sure the id is cached
-        return ret
-
-    @classmethod
-    def uint(cls, size_bits: int, bias: Optional[int]) -> "ScalarType":
-        """Create a unsigned integer scalar type."""
-        ret = cls(0, size_bits, False, bias if bias else 0)
-        ret.id  # noqa B018: make sure the id is cached
-        return ret
-
-    @classmethod
-    def float_IEEE754(cls, exponent: int, mantissa: int) -> "ScalarType":
-        """
-        Create a standard floating point type
-        (i.e. follows IEEE 754 conventions).
-        """
-        assert mantissa > 0 and exponent > 0
-        ret = cls(exponent, mantissa, True, 0)
-        ret.id  # noqa B018: make sure the id is cached
-        return ret
-
-    @classmethod
-    def float_(
-        cls, exponent: int, mantissa: int, finite_values_only: bool, nan_repr: NanRepr
-    ) -> "ScalarType":
-        """
-        Create a non-standard floating point type
-        (i.e. does not follow IEEE 754 conventions).
-        """
-        assert mantissa > 0 and exponent > 0
-        assert nan_repr != NanRepr.IEEE_754, (
-            "use `float_IEEE754` constructor for floating point types that "
-            "follow IEEE 754 conventions"
-        )
-        ret = cls(exponent, mantissa, True, 0, finite_values_only, nan_repr)
-        ret.id  # noqa B018: make sure the id is cached
-        return ret
-
-    @classmethod
-    def from_id(cls, scalar_type_id: int):
-        if scalar_type_id not in _SCALAR_TYPES_ID_MAP:
-            raise ValueError(f"scalar_type_id {scalar_type_id} doesn't exists.")
-        return _SCALAR_TYPES_ID_MAP[scalar_type_id]
-
-
-# naming generally follows: https://github.com/jax-ml/ml_dtypes
-# for floating point types (leading f) the scheme is:
-#  `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
-#  flags:
-#  - no-flags: means it follows IEEE 754 conventions
-#  - f: means finite values only (no infinities)
-#  - n: means nans are supported (non-standard encoding)
-# for integer types the scheme is:
-#  `[u]int<size_bits>[b<bias>]`
-#  - if bias is not present it means its zero
-
-
-class scalar_types:
-    int4 = ScalarType.int_(4, None)
-    uint4 = ScalarType.uint(4, None)
-    int8 = ScalarType.int_(8, None)
-    uint8 = ScalarType.uint(8, None)
-    float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)
-    float8_e5m2 = ScalarType.float_IEEE754(5, 2)
-    float16_e8m7 = ScalarType.float_IEEE754(8, 7)
-    float16_e5m10 = ScalarType.float_IEEE754(5, 10)
-
-    # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
-    float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE)
-
-    # fp4, https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
-    float4_e2m1f = ScalarType.float_(2, 1, True, NanRepr.NONE)
-
-    # "gptq" types
-    uint2b2 = ScalarType.uint(2, 2)
-    uint3b4 = ScalarType.uint(3, 4)
-    uint4b8 = ScalarType.uint(4, 8)
-    uint8b128 = ScalarType.uint(8, 128)
-
-    # colloquial names
-    bfloat16 = float16_e8m7
-    float16 = float16_e5m10

sglang/srt/lora/backend/flashinfer_backend.py

@@ -1,131 +0,0 @@
-from typing import Tuple
-
-import torch
-
-from sglang.srt.lora.backend.base_backend import BaseLoRABackend
-from sglang.srt.lora.utils import LoRABatchInfo
-from sglang.srt.utils import is_flashinfer_available
-
-if is_flashinfer_available():
-    from flashinfer import SegmentGEMMWrapper
-
-
-class FlashInferLoRABackend(BaseLoRABackend):
-
-    def __init__(self, name: str, batch_info: LoRABatchInfo = None):
-        super().__init__(name, batch_info)
-
-        # Set up SGemm Wrapper from flashinfer
-        # FIXME wait for flashinfer segment gemm update
-        workspace_buffer = torch.empty(1 * 1024 * 1024, dtype=torch.int8, device="cuda")
-        self.segment_gemm = SegmentGEMMWrapper(workspace_buffer)
-
-    def run_lora_a_sgemm(
-        self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs
-    ) -> torch.Tensor:
-
-        return self.segment_gemm.run(
-            x=x,
-            weights=weights,
-            batch_size=self.batch_info.bs,
-            weight_column_major=True,
-            seg_indptr=self.batch_info.seg_indptr,
-            weight_indices=self.batch_info.weight_indices,
-        )
-
-    def run_lora_b_sgemm(
-        self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs
-    ) -> torch.Tensor:
-
-        return (
-            self.segment_gemm.run(
-                x=x,
-                weights=weights,
-                batch_size=self.batch_info.bs,
-                weight_column_major=True,
-                seg_indptr=self.batch_info.seg_indptr,
-                weight_indices=self.batch_info.weight_indices,
-            )
-            * self.batch_info.scalings[0]
-        )
-
-    def run_qkv_lora(
-        self,
-        x: torch.Tensor,
-        qkv_lora_a: torch.Tensor,
-        qkv_lora_b: Tuple[torch.Tensor],
-        *args,
-        **kwargs,
-    ) -> torch.Tensor:
-
-        assert isinstance(qkv_lora_b, tuple) and len(qkv_lora_b) == 2
-
-        # Shape of lora_a_output: (s, 3 * r)
-        lora_a_output = self.run_lora_a_sgemm(x=x, weights=qkv_lora_a)
-
-        q_lora_b, kv_lora_b = qkv_lora_b
-        lora_rank = kv_lora_b.shape[-1]
-        output_dim_q = q_lora_b.shape[-2]
-        output_dim_kv = kv_lora_b.shape[-2]
-        lora_output = torch.empty(
-            (x.shape[0], output_dim_q + 2 * output_dim_kv),
-            device=x.device,
-            dtype=x.dtype,
-        )
-
-        # q
-        lora_output[:, :output_dim_q] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, :lora_rank].contiguous(), weights=q_lora_b[0]
-        )
-
-        # kv
-        lora_output[:, output_dim_q : output_dim_q + output_dim_kv] = (
-            self.run_lora_b_sgemm(
-                x=lora_a_output[:, lora_rank : 2 * lora_rank].contiguous(),
-                weights=kv_lora_b[0],
-            )
-        )
-
-        lora_output[
-            :, output_dim_q + output_dim_kv : output_dim_q + 2 * output_dim_kv
-        ] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, 2 * lora_rank : 3 * lora_rank].contiguous(),
-            weights=kv_lora_b[1],
-        )
-
-        return lora_output * self.batch_info.scalings[0]
-
-    def run_gate_up_lora(
-        self,
-        x: torch.Tensor,
-        gate_up_lora_a: torch.Tensor,
-        gate_up_lora_b: Tuple[torch.Tensor],
-        *args,
-        **kwargs,
-    ) -> torch.Tensor:
-
-        assert isinstance(gate_up_lora_b, tuple) and len(gate_up_lora_b) == 2
-        lora_rank = gate_up_lora_b[0].shape[-1]
-        output_dim = gate_up_lora_b[0].shape[-2]
-
-        # Shape of lora_a_output: (s, 2 * r)
-        lora_a_output = self.run_lora_a_sgemm(x=x, weights=gate_up_lora_a)
-
-        lora_output = torch.empty(
-            (x.shape[0], 2 * output_dim),
-            device=x.device,
-            dtype=x.dtype,
-        )
-
-        # Compute lora for gate and up proj respectively
-        lora_output[:, :output_dim] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, :lora_rank].contiguous(),
-            weights=gate_up_lora_b[0],
-        )
-
-        lora_output[:, output_dim:] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, lora_rank:].contiguous(),
-            weights=gate_up_lora_b[1],
-        )
-
-        return lora_output * self.batch_info.scalings[0]