sglang 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

sglang/srt/models/stablelm.py

@@ -0,0 +1,293 @@
+# This code is based on:
+# https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/stablelm.py
+"""Inference-only StableLM-2 (https://huggingface.co/stabilityai/stablelm-2-1_6b)
+model compatible with HuggingFace weights."""
+from typing import Optional, Tuple
+
+import torch
+from torch import nn
+from transformers import PretrainedConfig
+
+from sglang.srt.layers.logits_processor import LogitsProcessor
+from sglang.srt.layers.radix_attention import RadixAttention
+from sglang.srt.managers.router.model_runner import InputMetadata
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.linear import (
+    LinearMethodBase,
+    MergedColumnParallelLinear,
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+from vllm.model_executor.layers.rotary_embedding import get_rope
+from vllm.model_executor.layers.vocab_parallel_embedding import (
+    VocabParallelEmbedding,
+    ParallelLMHead,
+)
+from vllm.model_executor.parallel_utils.parallel_state import (
+    get_tensor_model_parallel_world_size,
+)
+from vllm.model_executor.weight_utils import (
+    default_weight_loader,
+    hf_model_weights_iterator,
+)
+
+
+class StablelmMLP(nn.Module):
+    def __init__(
+        self, config: PretrainedConfig, linear_method: Optional[LinearMethodBase] = None
+    ) -> None:
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_up_proj = MergedColumnParallelLinear(
+            config.hidden_size,
+            [config.intermediate_size] * 2,
+            bias=False,
+            linear_method=linear_method,
+        )
+        self.down_proj = RowParallelLinear(
+            config.intermediate_size, config.hidden_size, bias=False
+        )
+        self.act_fn = SiluAndMul()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        gate_up, _ = self.gate_up_proj(x)
+        x = self.act_fn(gate_up)
+        x, _ = self.down_proj(x)
+        return x
+
+
+class StablelmAttention(nn.Module):
+    def __init__(
+        self,
+        config: PretrainedConfig,
+        layer_id: int = 0,
+        linear_method: Optional[LinearMethodBase] = None,
+    ) -> None:
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        tp_size = get_tensor_model_parallel_world_size()
+        self.total_num_heads = config.num_attention_heads
+        self.num_heads = self.total_num_heads // tp_size
+
+        self.total_num_key_value_heads = config.num_key_value_heads
+        if self.total_num_key_value_heads >= tp_size:
+            # Number of KV heads is greater than TP size, so we partition
+            # the KV heads across multiple tensor parallel GPUs.
+            assert self.total_num_key_value_heads % tp_size == 0
+        else:
+            # Number of KV heads is less than TP size, so we replicate
+            # the KV heads across multiple tensor parallel GPUs.
+            assert tp_size % self.total_num_key_value_heads == 0
+        self.num_key_value_heads = max(1, self.total_num_key_value_heads // tp_size)
+        self.head_dim = self.hidden_size // self.total_num_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        rope_pct = getattr(
+            config, "rope_pct", getattr(config, "partial_rotary_factor", 1)
+        )
+        self.rotary_ndims = int(self.head_dim * rope_pct)
+        self.scaling = self.head_dim**-0.5
+        self.q_size = self.num_heads * self.head_dim
+        self.kv_size = self.num_key_value_heads * self.head_dim
+        self.qkv_bias = getattr(config, "use_qkv_bias", False)
+        if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads "
+                f"(got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+
+        self.qkv_proj = QKVParallelLinear(
+            self.hidden_size,
+            self.head_dim,
+            self.total_num_heads,
+            self.total_num_key_value_heads,
+            self.qkv_bias,
+            linear_method=linear_method,
+        )
+        self.o_proj = RowParallelLinear(
+            self.total_num_heads * self.head_dim,
+            self.hidden_size,
+            bias=False,
+            linear_method=linear_method,
+        )
+        self.rotary_emb = get_rope(
+            self.head_dim,
+            rotary_dim=self.rotary_ndims,
+            max_position=self.config.max_position_embeddings,
+            base=self.config.rope_theta,
+        )
+        self.attn = RadixAttention(
+            self.num_heads,
+            self.head_dim,
+            self.scaling,
+            num_kv_heads=self.num_key_value_heads,
+            layer_id=layer_id,
+        )
+
+    def forward(
+        self,
+        positions: torch.Tensor,
+        hidden_states: torch.Tensor,
+        input_metadata: InputMetadata,
+    ) -> torch.Tensor:
+        qkv, _ = self.qkv_proj(hidden_states)
+        q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+        q, k = self.rotary_emb(positions, q, k)
+        attn_output = self.attn(q, k, v, input_metadata)
+        output, _ = self.o_proj(attn_output)
+        return output
+
+
+class StablelmDecoderLayer(nn.Module):
+    def __init__(
+        self,
+        config: PretrainedConfig,
+        layer_id: int = 0,
+        linear_method: Optional[LinearMethodBase] = None,
+    ) -> None:
+        super().__init__()
+        self.self_attn = StablelmAttention(config, layer_id=layer_id)
+        self.mlp = StablelmMLP(config, linear_method)
+        norm_eps = getattr(config, "norm_eps", getattr(config, "layer_norm_eps", 1e-05))
+        self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
+        self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
+
+    def forward(
+        self,
+        positions: torch.Tensor,
+        hidden_states: torch.Tensor,
+        input_metadata: InputMetadata,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        # Self Attention
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        hidden_states = self.self_attn(
+            positions=positions,
+            hidden_states=hidden_states,
+            input_metadata=input_metadata,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        return hidden_states, residual
+
+
+class StableLMEpochModel(nn.Module):
+    def __init__(
+        self, config: PretrainedConfig, linear_method: Optional[LinearMethodBase] = None
+    ) -> None:
+        super().__init__()
+        self.embed_tokens = VocabParallelEmbedding(
+            config.vocab_size,
+            config.hidden_size,
+        )
+        self.layers = nn.ModuleList(
+            [
+                StablelmDecoderLayer(config, i, linear_method)
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        norm_eps = getattr(config, "norm_eps", getattr(config, "layer_norm_eps", 1e-05))
+        self.norm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        input_metadata: InputMetadata,
+        input_embeds: torch.Tensor = None,
+    ) -> torch.Tensor:
+        if input_embeds is None:
+            hidden_states = self.embed_tokens(input_ids)
+        else:
+            hidden_states = input_embeds
+        for i in range(len(self.layers)):
+            layer = self.layers[i]
+            hidden_states, residual = layer(
+                positions,
+                hidden_states,
+                input_metadata,
+            )
+        hidden_states = self.norm(hidden_states)
+        return hidden_states
+
+
+class StableLmForCausalLM(nn.Module):
+    def __init__(
+        self,
+        config: PretrainedConfig,
+        linear_method: Optional[LinearMethodBase] = None,
+    ) -> None:
+        super().__init__()
+        self.config = config
+        self.linear_method = linear_method
+        self.model = StableLMEpochModel(config, linear_method)
+        self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
+        self.logits_processor = LogitsProcessor(config)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        input_metadata: InputMetadata,
+        input_embeds: torch.Tensor = None,
+    ) -> torch.Tensor:
+        hidden_states = self.model(input_ids, positions, input_metadata, input_embeds)
+        return self.logits_processor(
+            input_ids, hidden_states, self.lm_head.weight, input_metadata
+        )
+
+    def load_weights(
+        self,
+        model_name_or_path: str,
+        cache_dir: Optional[str] = None,
+        load_format: str = "auto",
+        revision: Optional[str] = None,
+    ):
+        stacked_params_mapping = [
+            # (param_name, shard_name, shard_id)
+            ("qkv_proj", "q_proj", "q"),
+            ("qkv_proj", "k_proj", "k"),
+            ("qkv_proj", "v_proj", "v"),
+            ("gate_up_proj", "gate_proj", 0),
+            ("gate_up_proj", "up_proj", 1),
+        ]
+        params_dict = dict(self.named_parameters())
+        for name, loaded_weight in hf_model_weights_iterator(
+            model_name_or_path, cache_dir, load_format, revision
+        ):
+            if "rotary_emb.inv_freq" in name:
+                continue
+            if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
+                # Models trained using ColossalAI may include these tensors in
+                # the checkpoint. Skip them.
+                continue
+            for param_name, weight_name, shard_id in stacked_params_mapping:
+                if weight_name not in name:
+                    continue
+                name = name.replace(weight_name, param_name)
+                # Skip loading extra bias for GPTQ models.
+                if name.endswith(".bias") and name not in params_dict:
+                    continue
+                param = params_dict[name]
+                weight_loader = param.weight_loader
+                weight_loader(param, loaded_weight, shard_id)
+                break
+            else:
+                # Skip loading extra bias for GPTQ models.
+                if name.endswith(".bias") and name not in params_dict:
+                    continue
+                param = params_dict[name]
+                weight_loader = getattr(param, "weight_loader", default_weight_loader)
+                weight_loader(param, loaded_weight)
+
+
+EntryClass = StableLmForCausalLM

sglang/srt/server.py

@@ -1,6 +1,7 @@
 """SRT: SGLang Runtime"""
 
 import asyncio
+import dataclasses
 import json
 import multiprocessing as mp
 import os
@@ -52,10 +53,31 @@ from sglang.srt.managers.openai_protocol import (
 from sglang.srt.managers.router.manager import start_router_process
 from sglang.srt.managers.tokenizer_manager import TokenizerManager
 from sglang.srt.server_args import PortArgs, ServerArgs
-from sglang.srt.utils import alloc_usable_network_port, handle_port_init
+from sglang.srt.utils import handle_port_init
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import JSONResponse
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
+API_KEY_HEADER_NAME = "X-API-Key"
+
+
+class APIKeyValidatorMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app, api_key: str):
+        super().__init__(app)
+        self.api_key = api_key
+
+    async def dispatch(self, request: Request, call_next):
+        # extract API key from the request headers
+        api_key_header = request.headers.get(API_KEY_HEADER_NAME)
+        if not api_key_header or api_key_header != self.api_key:
+            return JSONResponse(
+                status_code=403,
+                content={"detail": "Invalid API Key"},
+            )
+        response = await call_next(request)
+        return response
+
 
 app = FastAPI()
 tokenizer_manager = None
@@ -86,6 +108,11 @@ async def get_model_info():
     return result
 
 
+@app.get("/get_server_args")
+async def get_server_args():
+    return dataclasses.asdict(tokenizer_manager.server_args)
+
+
 @app.get("/flush_cache")
 async def flush_cache():
     await tokenizer_manager.flush_cache()
@@ -96,19 +123,25 @@ async def flush_cache():
     )
 
 
-async def stream_generator(obj):
+async def detokenize_logprob_tokens(token_logprobs):
+    token_ids = [tid for tid, _ in token_logprobs]
+    token_texts = await tokenizer_manager.detokenize(DetokenizeReqInput(token_ids))
+    return [(text, logprob) for text, (_, logprob) in zip(token_texts, token_logprobs)]
+
+
+async def stream_generator(obj: GenerateReqInput):
     async for out in tokenizer_manager.generate_request(obj):
+        if obj.return_logprob and obj.return_text_in_logprobs:
+            out["meta_info"]["token_logprob"] = await detokenize_logprob_tokens(
+                out["meta_info"]["token_logprob"]
+            )
         yield out
 
 
 async def make_openai_style_logprobs(token_logprobs):
     ret_logprobs = LogProbs()
 
-    # Detokenize
-    token_ids = [tid for tid, _ in token_logprobs]
-    token_texts = await tokenizer_manager.detokenize(DetokenizeReqInput(token_ids))
-
-    for token_text, (_, token_logprob) in zip(token_texts, token_logprobs):
+    for token_text, token_logprob in token_logprobs:
         ret_logprobs.tokens.append(token_text)
         ret_logprobs.token_logprobs.append(token_logprob)
 
@@ -132,6 +165,11 @@ async def generate_request(obj: GenerateReqInput):
         return StreamingResponse(stream_results(), media_type="text/event-stream")
 
     ret = await tokenizer_manager.generate_request(obj).__anext__()
+    if obj.return_logprob and obj.return_text_in_logprobs:
+        ret["meta_info"]["token_logprob"] = await detokenize_logprob_tokens(
+            ret["meta_info"]["token_logprob"]
+        )
+
     return ret
 
 
@@ -155,6 +193,7 @@ async def v1_completions(raw_request: Request):
             "regex": request.regex,
         },
         return_logprob=request.logprobs is not None,
+        return_text_in_logprobs=True,
         stream=request.stream,
     )
     adapted_request.post_init()
@@ -211,6 +250,7 @@ async def v1_completions(raw_request: Request):
 
     # Non-streaming response.
     ret = await generate_request(adapted_request)
+    ret = ret[0] if isinstance(ret, list) else ret
 
     prompt_tokens = ret["meta_info"]["prompt_tokens"]
     completion_tokens = ret["meta_info"]["completion_tokens"]
@@ -463,8 +503,10 @@ def launch_server(server_args, pipe_finish_writer):
 
     assert proc_router.is_alive() and proc_detoken.is_alive()
 
+    if server_args.api_key and server_args.api_key != "":
+        app.add_middleware(APIKeyValidatorMiddleware, api_key=server_args.api_key)
+
     def _launch_server():
-        # Launch api server
         uvicorn.run(
             app,
             host=server_args.host,
@@ -474,49 +516,59 @@ def launch_server(server_args, pipe_finish_writer):
             loop="uvloop",
         )
 
-    t = threading.Thread(target=_launch_server)
-    t.start()
+    def _wait_and_warmup():
+        headers = {}
+        url = server_args.url()
+        if server_args.api_key and server_args.api_key != "":
+            headers[API_KEY_HEADER_NAME] = server_args.api_key
 
-    url = server_args.url()
-    for _ in range(60):
-        time.sleep(1)
-        try:
-            requests.get(url + "/get_model_info", timeout=5)
-            break
-        except requests.exceptions.RequestException as e:
-            pass
-    else:
-        if pipe_finish_writer is not None:
-            pipe_finish_writer.send(str(e))
+        for _ in range(120):
+            time.sleep(0.5)
+            try:
+                requests.get(url + "/get_model_info", timeout=5, headers=headers)
+                break
+            except requests.exceptions.RequestException as e:
+                pass
         else:
-            print(e, flush=True)
-        return
+            if pipe_finish_writer is not None:
+                pipe_finish_writer.send(str(e))
+            else:
+                print(e, flush=True)
+            return
 
-    # Warmup
-    try:
-        # print("Warmup...", flush=True)
-        res = requests.post(
-            url + "/generate",
-            json={
-                "text": "Say this is a warmup request.",
-                "sampling_params": {
-                    "temperature": 0,
-                    "max_new_tokens": 16,
+        # Warmup
+        try:
+            # print("Warmup...", flush=True)
+            res = requests.post(
+                url + "/generate",
+                json={
+                    "text": "Say this is a warmup request.",
+                    "sampling_params": {
+                        "temperature": 0,
+                        "max_new_tokens": 16,
+                    },
                 },
-            },
-            timeout=60,
-        )
-        # print(f"Warmup done. model response: {res.json()['text']}")
-        # print("=" * 20, "Server is ready", "=" * 20, flush=True)
-    except requests.exceptions.RequestException as e:
+                headers=headers,
+                timeout=60,
+            )
+            # print(f"Warmup done. model response: {res.json()['text']}")
+            # print("=" * 20, "Server is ready", "=" * 20, flush=True)
+        except requests.exceptions.RequestException as e:
+            if pipe_finish_writer is not None:
+                pipe_finish_writer.send(str(e))
+            else:
+                print(e, flush=True)
+            return
+
         if pipe_finish_writer is not None:
-            pipe_finish_writer.send(str(e))
-        else:
-            print(e, flush=True)
-        return
+            pipe_finish_writer.send("init ok")
 
-    if pipe_finish_writer is not None:
-        pipe_finish_writer.send("init ok")
+    t = threading.Thread(target=_wait_and_warmup)
+    t.start()
+    try:
+        _launch_server()
+    finally:
+        t.join()
 
 
 class Runtime:
@@ -529,11 +581,17 @@ class Runtime:
         trust_remote_code: bool = True,
         mem_fraction_static: float = ServerArgs.mem_fraction_static,
         max_prefill_num_token: int = ServerArgs.max_prefill_num_token,
+        context_length: int = ServerArgs.context_length,
         tp_size: int = 1,
-        model_mode: List[str] = (),
         schedule_heuristic: str = "lpm",
+        attention_reduce_in_fp32: bool = False,
         random_seed: int = 42,
         log_level: str = "error",
+        disable_radix_cache: bool = False,
+        enable_flashinfer: bool = False,
+        disable_regex_jump_forward: bool = False,
+        disable_disk_cache: bool = False,
+        api_key: str = "",
         port: Optional[int] = None,
         additional_ports: Optional[Union[List[int], int]] = None,
     ):
@@ -550,11 +608,17 @@ class Runtime:
             trust_remote_code=trust_remote_code,
             mem_fraction_static=mem_fraction_static,
             max_prefill_num_token=max_prefill_num_token,
+            context_length=context_length,
             tp_size=tp_size,
-            model_mode=model_mode,
             schedule_heuristic=schedule_heuristic,
+            attention_reduce_in_fp32=attention_reduce_in_fp32,
             random_seed=random_seed,
             log_level=log_level,
+            disable_radix_cache=disable_radix_cache,
+            enable_flashinfer=enable_flashinfer,
+            disable_regex_jump_forward=disable_regex_jump_forward,
+            disable_disk_cache=disable_disk_cache,
+            api_key=api_key,
         )
 
         self.url = self.server_args.url()

sglang/srt/server_args.py

@@ -16,17 +16,23 @@ class ServerArgs:
     trust_remote_code: bool = True
     mem_fraction_static: Optional[float] = None
     max_prefill_num_token: Optional[int] = None
+    context_length: Optional[int] = None
     tp_size: int = 1
-    model_mode: List[str] = ()
     schedule_heuristic: str = "lpm"
     schedule_conservativeness: float = 1.0
+    attention_reduce_in_fp32: bool = False
     random_seed: int = 42
     stream_interval: int = 8
     disable_log_stats: bool = False
     log_stats_interval: int = 10
     log_level: str = "info"
+
+    # optional modes
+    disable_radix_cache: bool = False
+    enable_flashinfer: bool = False
     disable_regex_jump_forward: bool = False
     disable_disk_cache: bool = False
+    api_key: str = ""
 
     def __post_init__(self):
         if self.tokenizer_path is None:
@@ -117,20 +123,18 @@ class ServerArgs:
             default=ServerArgs.max_prefill_num_token,
             help="The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length.",
         )
+        parser.add_argument(
+            "--context-length",
+            type=int,
+            default=ServerArgs.context_length,
+            help="The model's maximum context length. Use this to reduce the context length to save memory. Defaults to None (will use the value from the model's config.json instead).",
+        )
         parser.add_argument(
             "--tp-size",
             type=int,
             default=ServerArgs.tp_size,
             help="Tensor parallelism degree.",
         )
-        parser.add_argument(
-            "--model-mode",
-            type=str,
-            default=[],
-            nargs="+",
-            choices=["flashinfer", "no-cache"],
-            help="Model mode: [flashinfer, no-cache]",
-        )
         parser.add_argument(
             "--schedule-heuristic",
             type=str,
@@ -149,6 +153,11 @@ class ServerArgs:
             default=ServerArgs.random_seed,
             help="Random seed.",
         )
+        parser.add_argument(
+            "--attention-reduce-in-fp32",
+            action="store_true",
+            help="Cast the intermidiate attention results to fp32 to avoid possible crashes related to fp16.",
+        )
         parser.add_argument(
             "--stream-interval",
             type=int,
@@ -172,6 +181,17 @@ class ServerArgs:
             default=ServerArgs.log_stats_interval,
             help="Log stats interval in second.",
         )
+        # optional modes
+        parser.add_argument(
+            "--disable-radix-cache",
+            action="store_true",
+            help="Disable RadixAttention",
+        )
+        parser.add_argument(
+            "--enable-flashinfer",
+            action="store_true",
+            help="Enable flashinfer inference kernels",
+        )
         parser.add_argument(
             "--disable-regex-jump-forward",
             action="store_true",
@@ -182,6 +202,12 @@ class ServerArgs:
             action="store_true",
             help="Disable disk cache to avoid possible crashes related to file system or high concurrency.",
         )
+        parser.add_argument(
+            "--api-key",
+            type=str,
+            default=ServerArgs.api_key,
+            help="Set API Key",
+        )
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):
@@ -191,6 +217,15 @@ class ServerArgs:
     def url(self):
         return f"http://{self.host}:{self.port}"
 
+    def get_optional_modes_logging(self):
+        return (
+            f"disable_radix_cache={self.disable_radix_cache}, "
+            f"enable_flashinfer={self.enable_flashinfer}, "
+            f"disable_regex_jump_forward={self.disable_regex_jump_forward}, "
+            f"disable_disk_cache={self.disable_disk_cache}, "
+            f"attention_reduce_in_fp32={self.attention_reduce_in_fp32}"
+        )
+
 
 @dataclasses.dataclass
 class PortArgs:

sglang/srt/utils.py

@@ -103,6 +103,7 @@ def alloc_usable_network_port(num, used_list=()):
 def check_port(port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         try:
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             s.bind(("", port))
             return True
         except socket.error:

sglang/test/test_utils.py

@@ -155,7 +155,7 @@ def select_sglang_backend(args):
             global_config.enable_parallel_decoding = False
             global_config.enable_parallel_encoding = False
         backend = RuntimeEndpoint(f"{args.host}:{args.port}")
-    elif args.backend.startswith("gpt"):
+    elif args.backend.startswith("gpt-"):
         backend = OpenAI(args.backend)
     else:
         raise ValueError(f"Invalid backend: {args.backend}")