sglang 0.1.14__py3-none-any.whl → 0.1.21__py3-none-any.whl

sglang/srt/models/stablelm.py

@@ -1,40 +1,38 @@
-# This code is based on:
-# https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/stablelm.py
+# Adapted from:
+# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/stablelm.py#L1
 """Inference-only StableLM-2 (https://huggingface.co/stabilityai/stablelm-2-1_6b)
 model compatible with HuggingFace weights."""
-from typing import Optional, Tuple
+from typing import Iterable, Optional, Tuple
 
 import torch
 from torch import nn
 from transformers import PretrainedConfig
-
-from sglang.srt.layers.logits_processor import LogitsProcessor
-from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.managers.router.model_runner import InputMetadata
+from vllm.config import CacheConfig
+from vllm.distributed import get_tensor_model_parallel_world_size
 from vllm.model_executor.layers.activation import SiluAndMul
 from vllm.model_executor.layers.linear import (
-    LinearMethodBase,
     MergedColumnParallelLinear,
     QKVParallelLinear,
     RowParallelLinear,
 )
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import get_rope
 from vllm.model_executor.layers.vocab_parallel_embedding import (
-    VocabParallelEmbedding,
     ParallelLMHead,
+    VocabParallelEmbedding,
 )
-from vllm.model_executor.parallel_utils.parallel_state import (
-    get_tensor_model_parallel_world_size,
-)
-from vllm.model_executor.weight_utils import (
-    default_weight_loader,
-    hf_model_weights_iterator,
-)
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
+from sglang.srt.layers.logits_processor import LogitsProcessor
+from sglang.srt.layers.radix_attention import RadixAttention
+from sglang.srt.managers.controller.model_runner import InputMetadata
 
 
 class StablelmMLP(nn.Module):
     def __init__(
-        self, config: PretrainedConfig, linear_method: Optional[LinearMethodBase] = None
+        self,
+        config: PretrainedConfig,
+        quant_config: Optional[QuantizationConfig] = None,
     ) -> None:
         super().__init__()
         self.config = config
@@ -44,10 +42,13 @@ class StablelmMLP(nn.Module):
             config.hidden_size,
             [config.intermediate_size] * 2,
             bias=False,
-            linear_method=linear_method,
+            quant_config=quant_config,
         )
         self.down_proj = RowParallelLinear(
-            config.intermediate_size, config.hidden_size, bias=False
+            config.intermediate_size,
+            config.hidden_size,
+            bias=False,
+            quant_config=quant_config,
         )
         self.act_fn = SiluAndMul()
 
@@ -63,7 +64,7 @@ class StablelmAttention(nn.Module):
         self,
         config: PretrainedConfig,
         layer_id: int = 0,
-        linear_method: Optional[LinearMethodBase] = None,
+        quant_config: Optional[QuantizationConfig] = None,
     ) -> None:
         super().__init__()
         self.config = config
@@ -105,13 +106,11 @@ class StablelmAttention(nn.Module):
             self.total_num_heads,
             self.total_num_key_value_heads,
             self.qkv_bias,
-            linear_method=linear_method,
         )
         self.o_proj = RowParallelLinear(
             self.total_num_heads * self.head_dim,
             self.hidden_size,
             bias=False,
-            linear_method=linear_method,
         )
         self.rotary_emb = get_rope(
             self.head_dim,
@@ -146,11 +145,11 @@ class StablelmDecoderLayer(nn.Module):
         self,
         config: PretrainedConfig,
         layer_id: int = 0,
-        linear_method: Optional[LinearMethodBase] = None,
+        quant_config: Optional[QuantizationConfig] = None,
     ) -> None:
         super().__init__()
         self.self_attn = StablelmAttention(config, layer_id=layer_id)
-        self.mlp = StablelmMLP(config, linear_method)
+        self.mlp = StablelmMLP(config, quant_config=quant_config)
         norm_eps = getattr(config, "norm_eps", getattr(config, "layer_norm_eps", 1e-05))
         self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
         self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
@@ -182,7 +181,9 @@ class StablelmDecoderLayer(nn.Module):
 
 class StableLMEpochModel(nn.Module):
     def __init__(
-        self, config: PretrainedConfig, linear_method: Optional[LinearMethodBase] = None
+        self,
+        config: PretrainedConfig,
+        quant_config: Optional[QuantizationConfig] = None,
     ) -> None:
         super().__init__()
         self.embed_tokens = VocabParallelEmbedding(
@@ -191,7 +192,7 @@ class StableLMEpochModel(nn.Module):
         )
         self.layers = nn.ModuleList(
             [
-                StablelmDecoderLayer(config, i, linear_method)
+                StablelmDecoderLayer(config, i, quant_config=quant_config)
                 for i in range(config.num_hidden_layers)
             ]
         )
@@ -224,12 +225,13 @@ class StableLmForCausalLM(nn.Module):
     def __init__(
         self,
         config: PretrainedConfig,
-        linear_method: Optional[LinearMethodBase] = None,
+        quant_config: Optional[QuantizationConfig] = None,
+        cache_config: Optional[CacheConfig] = None,
     ) -> None:
         super().__init__()
         self.config = config
-        self.linear_method = linear_method
-        self.model = StableLMEpochModel(config, linear_method)
+        self.quant_config = quant_config
+        self.model = StableLMEpochModel(config, quant_config=quant_config)
         self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
         self.logits_processor = LogitsProcessor(config)
 
@@ -245,13 +247,7 @@ class StableLmForCausalLM(nn.Module):
             input_ids, hidden_states, self.lm_head.weight, input_metadata
         )
 
-    def load_weights(
-        self,
-        model_name_or_path: str,
-        cache_dir: Optional[str] = None,
-        load_format: str = "auto",
-        revision: Optional[str] = None,
-    ):
+    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
         stacked_params_mapping = [
             # (param_name, shard_name, shard_id)
             ("qkv_proj", "q_proj", "q"),
@@ -261,9 +257,7 @@ class StableLmForCausalLM(nn.Module):
             ("gate_up_proj", "up_proj", 1),
         ]
         params_dict = dict(self.named_parameters())
-        for name, loaded_weight in hf_model_weights_iterator(
-            model_name_or_path, cache_dir, load_format, revision
-        ):
+        for name, loaded_weight in weights:
             if "rotary_emb.inv_freq" in name:
                 continue
             if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:

sglang/srt/models/yivl.py

@@ -1,42 +1,38 @@
 """Inference-only Yi-VL model."""
 
-import os
-from typing import List, Optional
+from typing import Iterable, Optional, Tuple
 
 import torch
 import torch.nn as nn
+from transformers import CLIPVisionModel, LlavaConfig
+from vllm.config import CacheConfig
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
 from sglang.srt.models.llava import (
     LlavaLlamaForCausalLM,
-    clip_vision_embed_forward,
     monkey_path_clip_vision_embed_forward,
 )
-from transformers import CLIPVisionModel, LlavaConfig
-from vllm.model_executor.weight_utils import (
-    default_weight_loader,
-    hf_model_weights_iterator,
-)
 
 
 class YiVLForCausalLM(LlavaLlamaForCausalLM):
-    def __init__(self, *args, **kwargs):
-        self.config = kwargs["config"]
-        super().__init__(self.config)
+    def __init__(
+        self,
+        config: LlavaConfig,
+        quant_config: Optional[QuantizationConfig] = None,
+        cache_config: Optional[CacheConfig] = None,
+    ) -> None:
+        super().__init__(config, quant_config, cache_config)
 
         self.multi_modal_projector = YiVLMultiModalProjector(self.config)
         self.vision_tower_subfolder = self.config.mm_vision_tower.replace(
             "./", ""
         )  # Everything after "./"
 
-    def load_weights(
-        self,
-        model_name_or_path: str,
-        cache_dir: Optional[str] = None,
-        load_format: str = "auto",
-        revision: Optional[str] = None,
-    ):
+    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
         # We have to use the subfolder of the main model directory (e.g. 01-ai/Yi-VL-6B)
         self.vision_tower = CLIPVisionModel.from_pretrained(
-            model_name_or_path,
+            self.config._name_or_path,
             torch_dtype=torch.float16,
             subfolder=self.vision_tower_subfolder,
         ).cuda()
@@ -70,9 +66,8 @@ class YiVLForCausalLM(LlavaLlamaForCausalLM):
             "model.vision_tower.vision_tower": "vision_tower",  # Update the vision tower weights if we find them in the checkpoint (it may be finetuned).
         }
         params_dict = dict(self.named_parameters())
-        for name, loaded_weight in hf_model_weights_iterator(
-            model_name_or_path, cache_dir, load_format, revision
-        ):
+        weights = list(weights)
+        for name, loaded_weight in weights:
             if "projector" in name or "vision_tower" in name:
                 for weight_name, param_name in projector_weights.items():
                     if weight_name in name:
@@ -82,9 +77,7 @@ class YiVLForCausalLM(LlavaLlamaForCausalLM):
                 weight_loader(param, loaded_weight)
 
         # load language model
-        self.language_model.load_weights(
-            model_name_or_path, cache_dir, load_format, revision
-        )
+        self.language_model.load_weights(weights)
 
         monkey_path_clip_vision_embed_forward()
 
@@ -105,7 +98,7 @@ class YiVLMultiModalProjector(nn.Module):
 
     def forward(self, image_features):
         hidden_states = self.linear_1(image_features)
-        hidden_state = self.ln_1(hidden_states)
+        hidden_states = self.ln_1(hidden_states)
         hidden_states = self.act(hidden_states)
         hidden_states = self.linear_2(hidden_states)
         hidden_states = self.ln_2(hidden_states)

sglang/srt/openai_api_adapter.py

@@ -0,0 +1,411 @@
+"""Conversion between OpenAI APIs and native SRT APIs"""
+
+import asyncio
+import json
+import os
+from http import HTTPStatus
+
+from fastapi import Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from sglang.srt.conversation import (
+    Conversation,
+    SeparatorStyle,
+    chat_template_exists,
+    generate_chat_conv,
+    register_conv_template,
+)
+from sglang.srt.managers.io_struct import GenerateReqInput
+from sglang.srt.openai_protocol import (
+    ChatCompletionRequest,
+    ChatCompletionResponse,
+    ChatCompletionResponseChoice,
+    ChatCompletionResponseStreamChoice,
+    ChatCompletionStreamResponse,
+    ChatMessage,
+    CompletionRequest,
+    CompletionResponse,
+    CompletionResponseChoice,
+    CompletionResponseStreamChoice,
+    CompletionStreamResponse,
+    DeltaMessage,
+    ErrorResponse,
+    LogProbs,
+    UsageInfo,
+)
+
+chat_template_name = None
+
+
+def create_error_response(
+    message: str,
+    err_type: str = "BadRequestError",
+    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+):
+    error = ErrorResponse(message=message, type=err_type, code=status_code.value)
+    return JSONResponse(content=error.model_dump(), status_code=error.code)
+
+
+def create_streaming_error_response(
+    message: str,
+    err_type: str = "BadRequestError",
+    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+) -> str:
+    error = ErrorResponse(message=message, type=err_type, code=status_code.value)
+    json_str = json.dumps({"error": error.model_dump()})
+    return json_str
+
+
+def load_chat_template_for_openai_api(chat_template_arg):
+    global chat_template_name
+
+    print(f"Use chat template: {chat_template_arg}")
+    if not chat_template_exists(chat_template_arg):
+        if not os.path.exists(chat_template_arg):
+            raise RuntimeError(
+                f"Chat template {chat_template_arg} is not a built-in template name "
+                "or a valid chat template file path."
+            )
+        with open(chat_template_arg, "r") as filep:
+            template = json.load(filep)
+            try:
+                sep_style = SeparatorStyle[template["sep_style"]]
+            except KeyError:
+                raise ValueError(
+                    f"Unknown separator style: {template['sep_style']}"
+                ) from None
+            register_conv_template(
+                Conversation(
+                    name=template["name"],
+                    system_template=template["system"] + "\n{system_message}",
+                    system_message=template.get("system_message", ""),
+                    roles=(template["user"], template["assistant"]),
+                    sep_style=sep_style,
+                    sep=template.get("sep", "\n"),
+                    stop_str=template["stop_str"],
+                ),
+                override=True,
+            )
+        chat_template_name = template["name"]
+    else:
+        chat_template_name = chat_template_arg
+
+
+async def v1_completions(tokenizer_manager, raw_request: Request):
+    request_json = await raw_request.json()
+    request = CompletionRequest(**request_json)
+
+    if request.n != 1:
+        return create_error_response("n != 1 is not supported")
+
+    adapted_request = GenerateReqInput(
+        text=request.prompt,
+        sampling_params={
+            "temperature": request.temperature,
+            "max_new_tokens": request.max_tokens,
+            "stop": request.stop,
+            "top_p": request.top_p,
+            "presence_penalty": request.presence_penalty,
+            "frequency_penalty": request.frequency_penalty,
+            "regex": request.regex,
+        },
+        return_logprob=request.logprobs is not None and request.logprobs > 0,
+        top_logprobs_num=request.logprobs if request.logprobs is not None else 0,
+        return_text_in_logprobs=True,
+        stream=request.stream,
+    )
+
+    if adapted_request.stream:
+
+        async def generate_stream_resp():
+            stream_buffer = ""
+            n_prev_token = 0
+            try:
+                async for content in tokenizer_manager.generate_request(
+                    adapted_request, raw_request
+                ):
+                    text = content["text"]
+                    prompt_tokens = content["meta_info"]["prompt_tokens"]
+                    completion_tokens = content["meta_info"]["completion_tokens"]
+
+                    if not stream_buffer:  # The first chunk
+                        if request.echo:
+                            # Prepend prompt in response text.
+                            text = request.prompt + text
+
+                    if request.logprobs:
+                        # The first chunk and echo is enabled.
+                        if not stream_buffer and request.echo:
+                            prefill_token_logprobs = content["meta_info"][
+                                "prefill_token_logprobs"
+                            ]
+                            prefill_top_logprobs = content["meta_info"][
+                                "prefill_top_logprobs"
+                            ]
+                        else:
+                            prefill_token_logprobs = None
+                            prefill_top_logprobs = None
+
+                        logprobs = to_openai_style_logprobs(
+                            prefill_token_logprobs=prefill_token_logprobs,
+                            prefill_top_logprobs=prefill_top_logprobs,
+                            decode_token_logprobs=content["meta_info"][
+                                "decode_token_logprobs"
+                            ][n_prev_token:],
+                            decode_top_logprobs=content["meta_info"][
+                                "decode_top_logprobs"
+                            ][n_prev_token:],
+                        )
+
+                        n_prev_token = len(
+                            content["meta_info"]["decode_token_logprobs"]
+                        )
+                    else:
+                        logprobs = None
+
+                    delta = text[len(stream_buffer) :]
+                    stream_buffer = stream_buffer + delta
+                    choice_data = CompletionResponseStreamChoice(
+                        index=0,
+                        text=delta,
+                        logprobs=logprobs,
+                        finish_reason=content["meta_info"]["finish_reason"],
+                    )
+                    chunk = CompletionStreamResponse(
+                        id=content["meta_info"]["id"],
+                        object="text_completion",
+                        choices=[choice_data],
+                        model=request.model,
+                        usage=UsageInfo(
+                            prompt_tokens=prompt_tokens,
+                            completion_tokens=completion_tokens,
+                            total_tokens=prompt_tokens + completion_tokens,
+                        ),
+                    )
+                    yield f"data: {chunk.model_dump_json()}\n\n"
+            except ValueError as e:
+                error = create_streaming_error_response(str(e))
+                yield f"data: {error}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(
+            generate_stream_resp(),
+            media_type="text/event-stream",
+            background=tokenizer_manager.create_abort_task(adapted_request),
+        )
+
+    # Non-streaming response.
+    try:
+        ret = await tokenizer_manager.generate_request(
+            adapted_request, raw_request
+        ).__anext__()
+    except ValueError as e:
+        return create_error_response(str(e))
+
+    ret = ret[0] if isinstance(ret, list) else ret
+    prompt_tokens = ret["meta_info"]["prompt_tokens"]
+    completion_tokens = ret["meta_info"]["completion_tokens"]
+    text = ret["text"]
+    if request.echo:
+        text = request.prompt + text
+
+    if request.logprobs:
+        if request.echo:
+            prefill_token_logprobs = ret["meta_info"]["prefill_token_logprobs"]
+            prefill_top_logprobs = ret["meta_info"]["prefill_top_logprobs"]
+        else:
+            prefill_token_logprobs = None
+            prefill_top_logprobs = None
+
+        logprobs = to_openai_style_logprobs(
+            prefill_token_logprobs=prefill_token_logprobs,
+            prefill_top_logprobs=prefill_top_logprobs,
+            decode_token_logprobs=ret["meta_info"]["decode_token_logprobs"],
+            decode_top_logprobs=ret["meta_info"]["decode_top_logprobs"],
+        )
+    else:
+        logprobs = None
+
+    choice_data = CompletionResponseChoice(
+        index=0,
+        text=text,
+        logprobs=logprobs,
+        finish_reason=ret["meta_info"]["finish_reason"],
+    )
+    response = CompletionResponse(
+        id=ret["meta_info"]["id"],
+        model=request.model,
+        choices=[choice_data],
+        usage=UsageInfo(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        ),
+    )
+    return response
+
+
+async def v1_chat_completions(tokenizer_manager, raw_request: Request):
+    request_json = await raw_request.json()
+    request = ChatCompletionRequest(**request_json)
+
+    if request.n != 1:
+        return create_error_response("n != 1 is not supported")
+
+    # Prep the data needed for the underlying GenerateReqInput:
+    #  - prompt: The full prompt string.
+    #  - stop: Custom stop tokens.
+    #  - image_data: None or a list of image strings (URLs or base64 strings).
+    #    None skips any image processing in GenerateReqInput.
+    if not isinstance(request.messages, str):
+        # Apply chat template and its stop strings.
+        if chat_template_name is None:
+            prompt = tokenizer_manager.tokenizer.apply_chat_template(
+                request.messages, tokenize=False, add_generation_prompt=True
+            )
+            stop = request.stop
+            image_data = None
+        else:
+            conv = generate_chat_conv(request, chat_template_name)
+            prompt = conv.get_prompt()
+            image_data = conv.image_data
+            stop = conv.stop_str or []
+            if request.stop:
+                if isinstance(request.stop, str):
+                    stop.append(request.stop)
+                else:
+                    stop.extend(request.stop)
+    else:
+        # Use the raw prompt and stop strings if the messages is already a string.
+        prompt = request.messages
+        stop = request.stop
+        image_data = None
+
+    adapted_request = GenerateReqInput(
+        text=prompt,
+        image_data=image_data,
+        sampling_params={
+            "temperature": request.temperature,
+            "max_new_tokens": request.max_tokens,
+            "stop": stop,
+            "top_p": request.top_p,
+            "presence_penalty": request.presence_penalty,
+            "frequency_penalty": request.frequency_penalty,
+            "regex": request.regex,
+        },
+        stream=request.stream,
+    )
+
+    if adapted_request.stream:
+
+        async def generate_stream_resp():
+            is_first = True
+
+            stream_buffer = ""
+            try:
+                async for content in tokenizer_manager.generate_request(
+                    adapted_request, raw_request
+                ):
+                    if is_first:
+                        # First chunk with role
+                        is_first = False
+                        choice_data = ChatCompletionResponseStreamChoice(
+                            index=0,
+                            delta=DeltaMessage(role="assistant"),
+                            finish_reason=content["meta_info"]["finish_reason"],
+                        )
+                        chunk = ChatCompletionStreamResponse(
+                            id=content["meta_info"]["id"],
+                            choices=[choice_data],
+                            model=request.model,
+                        )
+                        yield f"data: {chunk.model_dump_json()}\n\n"
+
+                    text = content["text"]
+                    delta = text[len(stream_buffer) :]
+                    stream_buffer = stream_buffer + delta
+                    choice_data = ChatCompletionResponseStreamChoice(
+                        index=0,
+                        delta=DeltaMessage(content=delta),
+                        finish_reason=content["meta_info"]["finish_reason"],
+                    )
+                    chunk = ChatCompletionStreamResponse(
+                        id=content["meta_info"]["id"],
+                        choices=[choice_data],
+                        model=request.model,
+                    )
+                    yield f"data: {chunk.model_dump_json()}\n\n"
+            except ValueError as e:
+                error = create_streaming_error_response(str(e))
+                yield f"data: {error}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(
+            generate_stream_resp(),
+            media_type="text/event-stream",
+            background=tokenizer_manager.create_abort_task(adapted_request),
+        )
+
+    # Non-streaming response.
+    try:
+        ret = await tokenizer_manager.generate_request(
+            adapted_request, raw_request
+        ).__anext__()
+    except ValueError as e:
+        return create_error_response(str(e))
+
+    prompt_tokens = ret["meta_info"]["prompt_tokens"]
+    completion_tokens = ret["meta_info"]["completion_tokens"]
+    choice_data = ChatCompletionResponseChoice(
+        index=0,
+        message=ChatMessage(role="assistant", content=ret["text"]),
+        finish_reason=ret["meta_info"]["finish_reason"],
+    )
+    response = ChatCompletionResponse(
+        id=ret["meta_info"]["id"],
+        model=request.model,
+        choices=[choice_data],
+        usage=UsageInfo(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        ),
+    )
+    return response
+
+
+def to_openai_style_logprobs(
+    prefill_token_logprobs=None,
+    decode_token_logprobs=None,
+    prefill_top_logprobs=None,
+    decode_top_logprobs=None,
+):
+    ret_logprobs = LogProbs()
+
+    def append_token_logprobs(token_logprobs):
+        for logprob, _, token_text in token_logprobs:
+            ret_logprobs.tokens.append(token_text)
+            ret_logprobs.token_logprobs.append(logprob)
+
+            # Not supported yet
+            ret_logprobs.text_offset.append(-1)
+
+    def append_top_logprobs(top_logprobs):
+        for tokens in top_logprobs:
+            if tokens is not None:
+                ret_logprobs.top_logprobs.append(
+                    {token[2]: token[0] for token in tokens}
+                )
+            else:
+                ret_logprobs.top_logprobs.append(None)
+
+    if prefill_token_logprobs is not None:
+        append_token_logprobs(prefill_token_logprobs)
+    if decode_token_logprobs is not None:
+        append_token_logprobs(decode_token_logprobs)
+    if prefill_top_logprobs is not None:
+        append_top_logprobs(prefill_top_logprobs)
+    if decode_top_logprobs is not None:
+        append_top_logprobs(decode_top_logprobs)
+
+    return ret_logprobs