sglang 0.5.0rc0__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

sglang/srt/function_call/harmony_tool_parser.py

@@ -1,130 +0,0 @@
-# Copyright 2023-2024 SGLang Team
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Harmony tool call parser for processing tool calls in harmony models."""
-
-import uuid
-from typing import List, Optional, Tuple
-
-from sglang.srt.entrypoints.openai.protocol import (
-    ChatMessage,
-    FunctionResponse,
-    ToolCall,
-)
-
-
-class HarmonyToolCallParser:
-    """Parser for extracting tool calls from harmony model outputs."""
-
-    def extract_tool_calls_from_message(self, msg) -> Optional[ToolCall]:
-        """
-        Extract tool call from a single message if it's a tool call.
-
-        Args:
-            msg: The harmony message
-
-        Returns:
-            ToolCall if the message is a tool call, None otherwise
-        """
-        if (
-            msg.channel == "commentary"
-            and msg.recipient
-            and msg.recipient.startswith("functions.")
-        ):
-            function_name = msg.recipient.split(".")[-1]
-            arguments = msg.content[0].text if msg.content else "{}"
-
-            return ToolCall(
-                id=f"call_{uuid.uuid4().hex[:24]}",
-                function=FunctionResponse(
-                    name=function_name,
-                    arguments=arguments,
-                ),
-            )
-        return None
-
-    def process_streaming_chunk(
-        self,
-        harmony_parser,
-        index: int,
-        tool_call_trackers: dict,
-        stream_buffers: dict,
-    ) -> Tuple[Optional[dict], bool, Optional[str]]:
-        """
-        Process a streaming chunk for tool calls.
-
-        Args:
-            harmony_parser: The harmony parser instance
-            index: The choice index
-            tool_call_trackers: Dict tracking tool calls per choice
-            stream_buffers: Dict for buffering content
-
-        Returns:
-            Tuple of (tool_call_data, is_tool_call, delta)
-        """
-        # Check if we're in a tool call
-        is_tool_call = (
-            harmony_parser.current_channel == "commentary"
-            and harmony_parser.current_recipient
-            and harmony_parser.current_recipient.startswith("functions.")
-        )
-
-        delta = harmony_parser.last_content_delta or ""
-        tool_call_data = None
-
-        if is_tool_call:
-            # Handle tool call streaming
-            function_name = harmony_parser.current_recipient.split(".")[-1]
-
-            # Track tool call indices per choice
-            if index not in tool_call_trackers:
-                tool_call_trackers[index] = {"count": 0, "current_function": None}
-
-            # Check if we just started a new tool call
-            tool_call_tracker = tool_call_trackers[index]
-            if tool_call_tracker["current_function"] != function_name:
-                # New tool call started
-                tool_call_tracker["current_function"] = function_name
-                tool_call_index = tool_call_tracker["count"]
-                tool_call_tracker["count"] += 1
-
-                # Store the tool call index for this function
-                tool_call_key = f"{index}_{function_name}"
-                stream_buffers[tool_call_key] = {
-                    "index": tool_call_index,
-                    "content": "",
-                }
-
-                tool_call_data = {
-                    "id": f"call_{uuid.uuid4().hex[:24]}",
-                    "index": tool_call_index,
-                    "function_name": function_name,
-                    "arguments": delta,
-                    "is_first_chunk": True,
-                }
-            else:
-                # Subsequent chunks for the same tool call
-                tool_call_key = f"{index}_{function_name}"
-                tool_call_index = stream_buffers[tool_call_key]["index"]
-
-                tool_call_data = {
-                    "id": None,
-                    "index": tool_call_index,
-                    "function_name": None,
-                    "arguments": delta,
-                    "is_first_chunk": False,
-                }
-
-            stream_buffers[tool_call_key]["content"] += delta
-
-        return tool_call_data, is_tool_call, delta

sglang/srt/lora/backend/flashinfer_backend.py

@@ -1,131 +0,0 @@
-from typing import Tuple
-
-import torch
-
-from sglang.srt.lora.backend.base_backend import BaseLoRABackend
-from sglang.srt.lora.utils import LoRABatchInfo
-from sglang.srt.utils import is_flashinfer_available
-
-if is_flashinfer_available():
-    from flashinfer import SegmentGEMMWrapper
-
-
-class FlashInferLoRABackend(BaseLoRABackend):
-
-    def __init__(self, name: str, batch_info: LoRABatchInfo = None):
-        super().__init__(name, batch_info)
-
-        # Set up SGemm Wrapper from flashinfer
-        # FIXME wait for flashinfer segment gemm update
-        workspace_buffer = torch.empty(1 * 1024 * 1024, dtype=torch.int8, device="cuda")
-        self.segment_gemm = SegmentGEMMWrapper(workspace_buffer)
-
-    def run_lora_a_sgemm(
-        self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs
-    ) -> torch.Tensor:
-
-        return self.segment_gemm.run(
-            x=x,
-            weights=weights,
-            batch_size=self.batch_info.bs,
-            weight_column_major=True,
-            seg_indptr=self.batch_info.seg_indptr,
-            weight_indices=self.batch_info.weight_indices,
-        )
-
-    def run_lora_b_sgemm(
-        self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs
-    ) -> torch.Tensor:
-
-        return (
-            self.segment_gemm.run(
-                x=x,
-                weights=weights,
-                batch_size=self.batch_info.bs,
-                weight_column_major=True,
-                seg_indptr=self.batch_info.seg_indptr,
-                weight_indices=self.batch_info.weight_indices,
-            )
-            * self.batch_info.scalings[0]
-        )
-
-    def run_qkv_lora(
-        self,
-        x: torch.Tensor,
-        qkv_lora_a: torch.Tensor,
-        qkv_lora_b: Tuple[torch.Tensor],
-        *args,
-        **kwargs,
-    ) -> torch.Tensor:
-
-        assert isinstance(qkv_lora_b, tuple) and len(qkv_lora_b) == 2
-
-        # Shape of lora_a_output: (s, 3 * r)
-        lora_a_output = self.run_lora_a_sgemm(x=x, weights=qkv_lora_a)
-
-        q_lora_b, kv_lora_b = qkv_lora_b
-        lora_rank = kv_lora_b.shape[-1]
-        output_dim_q = q_lora_b.shape[-2]
-        output_dim_kv = kv_lora_b.shape[-2]
-        lora_output = torch.empty(
-            (x.shape[0], output_dim_q + 2 * output_dim_kv),
-            device=x.device,
-            dtype=x.dtype,
-        )
-
-        # q
-        lora_output[:, :output_dim_q] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, :lora_rank].contiguous(), weights=q_lora_b[0]
-        )
-
-        # kv
-        lora_output[:, output_dim_q : output_dim_q + output_dim_kv] = (
-            self.run_lora_b_sgemm(
-                x=lora_a_output[:, lora_rank : 2 * lora_rank].contiguous(),
-                weights=kv_lora_b[0],
-            )
-        )
-
-        lora_output[
-            :, output_dim_q + output_dim_kv : output_dim_q + 2 * output_dim_kv
-        ] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, 2 * lora_rank : 3 * lora_rank].contiguous(),
-            weights=kv_lora_b[1],
-        )
-
-        return lora_output * self.batch_info.scalings[0]
-
-    def run_gate_up_lora(
-        self,
-        x: torch.Tensor,
-        gate_up_lora_a: torch.Tensor,
-        gate_up_lora_b: Tuple[torch.Tensor],
-        *args,
-        **kwargs,
-    ) -> torch.Tensor:
-
-        assert isinstance(gate_up_lora_b, tuple) and len(gate_up_lora_b) == 2
-        lora_rank = gate_up_lora_b[0].shape[-1]
-        output_dim = gate_up_lora_b[0].shape[-2]
-
-        # Shape of lora_a_output: (s, 2 * r)
-        lora_a_output = self.run_lora_a_sgemm(x=x, weights=gate_up_lora_a)
-
-        lora_output = torch.empty(
-            (x.shape[0], 2 * output_dim),
-            device=x.device,
-            dtype=x.dtype,
-        )
-
-        # Compute lora for gate and up proj respectively
-        lora_output[:, :output_dim] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, :lora_rank].contiguous(),
-            weights=gate_up_lora_b[0],
-        )
-
-        lora_output[:, output_dim:] = self.run_lora_b_sgemm(
-            x=lora_a_output[:, lora_rank:].contiguous(),
-            weights=gate_up_lora_b[1],
-        )
-
-        return lora_output * self.batch_info.scalings[0]