sglang 0.1.21__py3-none-any.whl → 0.1.22__py3-none-any.whl

sglang/srt/openai_api/protocol.py

@@ -0,0 +1,208 @@
+"""Pydantic models for OpenAI API protocol"""
+
+import time
+from typing import Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field
+from typing_extensions import Literal
+
+
+class ModelCard(BaseModel):
+    """Model cards."""
+
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "sglang"
+    root: Optional[str] = None
+
+
+class ModelList(BaseModel):
+    """Model list consists of model cards."""
+
+    object: str = "list"
+    data: List[ModelCard] = []
+
+
+class ErrorResponse(BaseModel):
+    object: str = "error"
+    message: str
+    type: str
+    param: Optional[str] = None
+    code: int
+
+
+class LogProbs(BaseModel):
+    text_offset: List[int] = Field(default_factory=list)
+    token_logprobs: List[Optional[float]] = Field(default_factory=list)
+    tokens: List[str] = Field(default_factory=list)
+    top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
+
+
+class UsageInfo(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens: Optional[int] = 0
+
+
+class CompletionRequest(BaseModel):
+    # Ordered by official OpenAI API documentation
+    # https://platform.openai.com/docs/api-reference/completions/create
+    model: str
+    prompt: Union[List[int], List[List[int]], str, List[str]]
+    best_of: Optional[int] = None
+    echo: Optional[bool] = False
+    frequency_penalty: Optional[float] = 0.0
+    logit_bias: Optional[Dict[str, float]] = None
+    logprobs: Optional[int] = None
+    max_tokens: Optional[int] = 16
+    n: int = 1
+    presence_penalty: Optional[float] = 0.0
+    seed: Optional[int] = None
+    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
+    stream: Optional[bool] = False
+    suffix: Optional[str] = None
+    temperature: Optional[float] = 1.0
+    top_p: Optional[float] = 1.0
+    user: Optional[str] = None
+
+    # Extra parameters for SRT backend only and will be ignored by OpenAI models.
+    regex: Optional[str] = None
+    ignore_eos: Optional[bool] = False
+
+
+class CompletionResponseChoice(BaseModel):
+    index: int
+    text: str
+    logprobs: Optional[LogProbs] = None
+    finish_reason: Optional[str] = None
+
+
+class CompletionResponse(BaseModel):
+    id: str
+    object: str = "text_completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[CompletionResponseChoice]
+    usage: UsageInfo
+
+
+class CompletionResponseStreamChoice(BaseModel):
+    index: int
+    text: str
+    logprobs: Optional[LogProbs] = None
+    finish_reason: Optional[str] = None
+
+
+class CompletionStreamResponse(BaseModel):
+    id: str
+    object: str = "text_completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[CompletionResponseStreamChoice]
+    usage: UsageInfo
+
+
+class ChatCompletionMessageGenericParam(BaseModel):
+    role: Literal["system", "assistant"]
+    content: str
+
+
+class ChatCompletionMessageContentTextPart(BaseModel):
+    type: Literal["text"]
+    text: str
+
+
+class ChatCompletionMessageContentImageURL(BaseModel):
+    url: str
+    detail: Optional[Literal["auto", "low", "high"]] = "auto"
+
+
+class ChatCompletionMessageContentImagePart(BaseModel):
+    type: Literal["image_url"]
+    image_url: ChatCompletionMessageContentImageURL
+
+
+ChatCompletionMessageContentPart = Union[
+    ChatCompletionMessageContentTextPart, ChatCompletionMessageContentImagePart
+]
+
+
+class ChatCompletionMessageUserParam(BaseModel):
+    role: Literal["user"]
+    content: Union[str, List[ChatCompletionMessageContentPart]]
+
+
+ChatCompletionMessageParam = Union[
+    ChatCompletionMessageGenericParam, ChatCompletionMessageUserParam
+]
+
+
+class ResponseFormat(BaseModel):
+    # type must be "json_object" or "text"
+    type: Literal["text", "json_object"]
+
+
+class ChatCompletionRequest(BaseModel):
+    # Ordered by official OpenAI API documentation
+    # https://platform.openai.com/docs/api-reference/chat/create
+    messages: List[ChatCompletionMessageParam]
+    model: str
+    frequency_penalty: Optional[float] = 0.0
+    logit_bias: Optional[Dict[str, float]] = None
+    logprobs: Optional[bool] = False
+    top_logprobs: Optional[int] = None
+    max_tokens: Optional[int] = 16
+    n: Optional[int] = 1
+    presence_penalty: Optional[float] = 0.0
+    response_format: Optional[ResponseFormat] = None
+    seed: Optional[int] = None
+    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
+    stream: Optional[bool] = False
+    temperature: Optional[float] = 0.7
+    top_p: Optional[float] = 1.0
+    user: Optional[str] = None
+
+    # Extra parameters for SRT backend only and will be ignored by OpenAI models.
+    regex: Optional[str] = None
+
+
+class ChatMessage(BaseModel):
+    role: Optional[str] = None
+    content: Optional[str] = None
+
+
+class ChatCompletionResponseChoice(BaseModel):
+    index: int
+    message: ChatMessage
+    logprobs: Optional[LogProbs] = None
+    finish_reason: Optional[str] = None
+
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: str = "chat.completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[ChatCompletionResponseChoice]
+    usage: UsageInfo
+
+
+class DeltaMessage(BaseModel):
+    role: Optional[str] = None
+    content: Optional[str] = None
+
+
+class ChatCompletionResponseStreamChoice(BaseModel):
+    index: int
+    delta: DeltaMessage
+    logprobs: Optional[LogProbs] = None
+    finish_reason: Optional[str] = None
+
+
+class ChatCompletionStreamResponse(BaseModel):
+    id: str
+    object: str = "chat.completion.chunk"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: List[ChatCompletionResponseStreamChoice]

sglang/srt/openai_protocol.py

@@ -7,6 +7,23 @@ from pydantic import BaseModel, Field
 from typing_extensions import Literal
 
 
+class ModelCard(BaseModel):
+    """Model cards."""
+
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "sglang"
+    root: Optional[str] = None
+
+
+class ModelList(BaseModel):
+    """Model list consists of model cards."""
+
+    object: str = "list"
+    data: List[ModelCard] = []
+
+
 class ErrorResponse(BaseModel):
     object: str = "error"
     message: str

sglang/srt/sampling_params.py

@@ -20,6 +20,7 @@ class SamplingParams:
         spaces_between_special_tokens: bool = True,
         dtype: Optional[str] = None,
         regex: Optional[str] = None,
+        n: int = 1,
     ) -> None:
         self.temperature = temperature
         self.top_p = top_p
@@ -33,6 +34,7 @@ class SamplingParams:
         self.spaces_between_special_tokens = spaces_between_special_tokens
         self.dtype = dtype
         self.regex = regex
+        self.n = n
 
         # Process some special cases
         if self.temperature < _SAMPLING_EPS:

sglang/srt/server.py

@@ -26,33 +26,33 @@ import uvloop
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, Response, StreamingResponse
 
-from sglang.backend.runtime_endpoint import RuntimeEndpoint
+from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
 from sglang.srt.constrained import disable_cache
 from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.managers.controller.manager_multi import (
     start_controller_process as start_controller_process_multi,
 )
+from sglang.srt.managers.controller.manager_single import launch_tp_servers
 from sglang.srt.managers.controller.manager_single import (
-    launch_tp_servers,
     start_controller_process as start_controller_process_single,
 )
 from sglang.srt.managers.detokenizer_manager import start_detokenizer_process
 from sglang.srt.managers.io_struct import GenerateReqInput
 from sglang.srt.managers.tokenizer_manager import TokenizerManager
-from sglang.srt.openai_api_adapter import (
+from sglang.srt.openai_api.adapter import (
     load_chat_template_for_openai_api,
     v1_chat_completions,
     v1_completions,
 )
-from sglang.srt.server_args import ModelPortArgs, PortArgs, ServerArgs
+from sglang.srt.openai_api.protocol import ModelCard, ModelList
+from sglang.srt.server_args import PortArgs, ServerArgs
 from sglang.srt.utils import (
     API_KEY_HEADER_NAME,
     APIKeyValidatorMiddleware,
     allocate_init_ports,
     assert_pkg_version,
     enable_show_time_cost,
-    receive_addrs,
-    send_addrs_to_rank_0,
+    set_ulimit,
 )
 from sglang.utils import get_exception_traceback
 
@@ -64,6 +64,9 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 app = FastAPI()
 tokenizer_manager = None
 
+# Put some args for easily access
+global_server_args_dict = {}
+
 
 @app.get("/health")
 async def health() -> Response:
@@ -95,6 +98,7 @@ async def flush_cache():
 
 
 async def generate_request(obj: GenerateReqInput, request: Request):
+    """Handle a generate request."""
     if obj.stream:
 
         async def stream_results():
@@ -135,7 +139,30 @@ async def openai_v1_chat_completions(raw_request: Request):
     return await v1_chat_completions(tokenizer_manager, raw_request)
 
 
-def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_args=None):
+@app.get("/v1/models")
+def available_models():
+    """Show available models."""
+    model_names = [tokenizer_manager.model_path]
+    model_cards = []
+    for model_name in model_names:
+        model_cards.append(ModelCard(id=model_name, root=model_name))
+    return ModelList(data=model_cards)
+
+
+def _set_global_server_args(server_args: ServerArgs):
+    global global_server_args_dict
+    global_server_args_dict = {
+        "disable_flashinfer": server_args.disable_flashinfer,
+        "attention_reduce_in_fp32": server_args.attention_reduce_in_fp32,
+    }
+
+
+def launch_server(
+    server_args: ServerArgs,
+    model_overide_args: Optional[dict] = None,
+    pipe_finish_writer: Optional[mp.connection.Connection] = None,
+):
+    """Launch an HTTP server."""
     global tokenizer_manager
 
     logging.basicConfig(
@@ -146,6 +173,8 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
     # Set global environments
     os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
     os.environ["NCCL_CUMEM_ENABLE"] = "0"
+    os.environ["NCCL_NVLS_ENABLE"] = "0"
+    set_ulimit()
     if server_args.show_time_cost:
         enable_show_time_cost()
     if server_args.disable_disk_cache:
@@ -153,7 +182,7 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
     if not server_args.disable_flashinfer:
         assert_pkg_version(
             "flashinfer",
-            "0.0.8",
+            "0.1.0",
             "Please uninstall the old version and "
             "reinstall the latest version by following the instructions "
             "at https://docs.flashinfer.ai/installation.html.",
@@ -161,64 +190,61 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
     if server_args.chat_template:
         # TODO: replace this with huggingface transformers template
         load_chat_template_for_openai_api(server_args.chat_template)
+    _set_global_server_args(server_args)
 
     # Allocate ports
-    assert server_args.tp_size % server_args.nnodes == 0
-    tp_size_local = server_args.tp_size // server_args.nnodes
     server_args.port, server_args.additional_ports = allocate_init_ports(
         server_args.port,
         server_args.additional_ports,
-        tp_size_local,
         server_args.dp_size,
     )
-
     ports = server_args.additional_ports
-    model_port_args = []
-    for i in range(server_args.dp_size):
-        model_port_args.append(
-            ModelPortArgs(
-                nccl_port=ports[3 + i * (tp_size_local + 1)],
-                model_tp_ips=[None] * tp_size_local,
-                model_tp_ports=ports[
-                    3 + i * (tp_size_local + 1) + 1 : 3 + (i + 1) * (tp_size_local + 1)
-                ],
-            )
-        )
     port_args = PortArgs(
         tokenizer_port=ports[0],
-        router_port=ports[1],
+        controller_port=ports[1],
         detokenizer_port=ports[2],
-        model_port_args=model_port_args,
+        nccl_ports=ports[3:],
     )
 
-    # Handle multi-node tp
+    # Handle multi-node tensor parallelism
     if server_args.nnodes > 1:
         assert server_args.dp_size == 1, "Multi-node dp is not supported."
 
         if server_args.node_rank != 0:
             tp_size_local = server_args.tp_size // server_args.nnodes
-            gpu_ids = [i for _ in range(server_args.nnodes) for i in range(tp_size_local)]
-            tp_rank_range = list(range(server_args.node_rank * tp_size_local,
-                                  (server_args.node_rank + 1) * tp_size_local))
-            procs = launch_tp_servers(gpu_ids, tp_rank_range, server_args,
-                                      port_args.model_port_args[0], model_overide_args)
+            gpu_ids = [
+                i for _ in range(server_args.nnodes) for i in range(tp_size_local)
+            ]
+            tp_rank_range = list(
+                range(
+                    server_args.node_rank * tp_size_local,
+                    (server_args.node_rank + 1) * tp_size_local,
+                )
+            )
+            procs = launch_tp_servers(
+                gpu_ids,
+                tp_rank_range,
+                server_args,
+                ports[3],
+                model_overide_args,
+            )
             while True:
                 pass
 
     # Launch processes
     tokenizer_manager = TokenizerManager(server_args, port_args, model_overide_args)
-    pipe_router_reader, pipe_router_writer = mp.Pipe(duplex=False)
+    pipe_controller_reader, pipe_controller_writer = mp.Pipe(duplex=False)
     pipe_detoken_reader, pipe_detoken_writer = mp.Pipe(duplex=False)
 
     if server_args.dp_size == 1:
         start_process = start_controller_process_single
     else:
         start_process = start_controller_process_multi
-    proc_router = mp.Process(
+    proc_controller = mp.Process(
         target=start_process,
-        args=(server_args, port_args, pipe_router_writer, model_overide_args),
+        args=(server_args, port_args, pipe_controller_writer, model_overide_args),
     )
-    proc_router.start()
+    proc_controller.start()
     proc_detoken = mp.Process(
         target=start_detokenizer_process,
         args=(
@@ -230,68 +256,30 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
     proc_detoken.start()
 
     # Wait for the model to finish loading
-    router_init_state = pipe_router_reader.recv()
+    controller_init_state = pipe_controller_reader.recv()
     detoken_init_state = pipe_detoken_reader.recv()
 
-    if router_init_state != "init ok" or detoken_init_state != "init ok":
-        proc_router.kill()
+    if controller_init_state != "init ok" or detoken_init_state != "init ok":
+        proc_controller.kill()
         proc_detoken.kill()
         print(
-            f"Initialization failed. router_init_state: {router_init_state}", flush=True
+            f"Initialization failed. controller_init_state: {controller_init_state}",
+            flush=True,
         )
         print(
             f"Initialization failed. detoken_init_state: {detoken_init_state}",
             flush=True,
         )
         sys.exit(1)
-    assert proc_router.is_alive() and proc_detoken.is_alive()
+    assert proc_controller.is_alive() and proc_detoken.is_alive()
 
     if server_args.api_key and server_args.api_key != "":
         app.add_middleware(APIKeyValidatorMiddleware, api_key=server_args.api_key)
 
     # Send a warmup request
-    def _wait_and_warmup():
-        headers = {}
-        url = server_args.url()
-        if server_args.api_key:
-            headers[API_KEY_HEADER_NAME] = server_args.api_key
-
-        # Wait until the server is launched
-        for _ in range(120):
-            time.sleep(0.5)
-            try:
-                requests.get(url + "/get_model_info", timeout=5, headers=headers)
-                break
-            except requests.exceptions.RequestException:
-                pass
-
-        # Send a warmup request
-        try:
-            for _ in range(server_args.dp_size):
-                res = requests.post(
-                    url + "/generate",
-                    json={
-                        "text": "The capital city of France is",
-                        "sampling_params": {
-                            "temperature": 0,
-                            "max_new_tokens": 8,
-                        },
-                    },
-                    headers=headers,
-                    timeout=600,
-                )
-                assert res.status_code == 200
-        except Exception as e:
-            if pipe_finish_writer is not None:
-                pipe_finish_writer.send(get_exception_traceback())
-            print(f"Initialization failed. warmup error: {e}", flush=True)
-            raise e
-
-        logger.info("The server is fired up and ready to roll!")
-        if pipe_finish_writer is not None:
-            pipe_finish_writer.send("init ok")
-
-    t = threading.Thread(target=_wait_and_warmup)
+    t = threading.Thread(
+        target=_wait_and_warmup, args=(server_args, pipe_finish_writer)
+    )
     t.start()
 
     # Listen for requests
@@ -308,6 +296,48 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
         t.join()
 
 
+def _wait_and_warmup(server_args, pipe_finish_writer):
+    headers = {}
+    url = server_args.url()
+    if server_args.api_key:
+        headers[API_KEY_HEADER_NAME] = server_args.api_key
+
+    # Wait until the server is launched
+    for _ in range(120):
+        time.sleep(0.5)
+        try:
+            requests.get(url + "/get_model_info", timeout=5, headers=headers)
+            break
+        except requests.exceptions.RequestException:
+            pass
+
+    # Send a warmup request
+    try:
+        for _ in range(server_args.dp_size):
+            res = requests.post(
+                url + "/generate",
+                json={
+                    "text": "The capital city of France is",
+                    "sampling_params": {
+                        "temperature": 0,
+                        "max_new_tokens": 8,
+                    },
+                },
+                headers=headers,
+                timeout=600,
+            )
+            assert res.status_code == 200
+    except Exception as e:
+        if pipe_finish_writer is not None:
+            pipe_finish_writer.send(get_exception_traceback())
+        print(f"Initialization failed. warmup error: {e}", flush=True)
+        raise e
+
+    logger.info("The server is fired up and ready to roll!")
+    if pipe_finish_writer is not None:
+        pipe_finish_writer.send("init ok")
+
+
 class Runtime:
     """
     A wrapper for the server.
@@ -329,7 +359,6 @@ class Runtime:
         self.server_args.port, self.server_args.additional_ports = allocate_init_ports(
             self.server_args.port,
             self.server_args.additional_ports,
-            self.server_args.tp_size,
             self.server_args.dp_size,
         )
 
@@ -342,7 +371,7 @@ class Runtime:
         pipe_reader, pipe_writer = mp.Pipe(duplex=False)
         proc = mp.Process(
             target=launch_server,
-            args=(self.server_args, pipe_writer, model_overide_args),
+            args=(self.server_args, model_overide_args, pipe_writer),
         )
         proc.start()
         pipe_writer.close()

sglang/srt/server_args.py

@@ -33,7 +33,7 @@ class ServerArgs:
 
     # Other runtime options
     tp_size: int = 1
-    stream_interval: int = 8
+    stream_interval: int = 1
     random_seed: Optional[int] = None
 
     # Logging
@@ -57,6 +57,7 @@ class ServerArgs:
     disable_disk_cache: bool = False
     attention_reduce_in_fp32: bool = False
     enable_p2p_check: bool = False
+    efficient_weight_load: bool = False
 
     # Distributed args
     nccl_init_addr: Optional[str] = None
@@ -166,6 +167,15 @@ class ServerArgs:
             "--quantization",
             type=str,
             default=ServerArgs.quantization,
+            choices=[
+                "awq",
+                "fp8",
+                "gptq",
+                "marlin",
+                "gptq_marlin",
+                "squeezellm",
+                "bitsandbytes",
+            ],
             help="The quantization method.",
         )
         parser.add_argument(
@@ -243,13 +253,13 @@ class ServerArgs:
         parser.add_argument(
             "--show-time-cost",
             action="store_true",
-            help="Show time cost of custom marks",
+            help="Show time cost of custom marks.",
         )
         parser.add_argument(
             "--api-key",
             type=str,
             default=ServerArgs.api_key,
-            help="Set API key of the server",
+            help="Set API key of the server.",
         )
 
         # Data parallelism
@@ -285,17 +295,17 @@ class ServerArgs:
         parser.add_argument(
             "--disable-flashinfer",
             action="store_true",
-            help="Disable flashinfer inference kernels",
+            help="Disable flashinfer inference kernels.",
         )
         parser.add_argument(
             "--disable-radix-cache",
             action="store_true",
-            help="Disable RadixAttention",
+            help="Disable RadixAttention for prefix caching.",
         )
         parser.add_argument(
             "--disable-regex-jump-forward",
             action="store_true",
-            help="Disable regex jump-forward",
+            help="Disable regex jump-forward.",
         )
         parser.add_argument(
             "--disable-cuda-graph",
@@ -318,6 +328,11 @@ class ServerArgs:
             action="store_true",
             help="Enable P2P check for GPU access, otherwise the p2p access is allowed by default.",
         )
+        parser.add_argument(
+            "--efficient-weight-load",
+            action="store_true",
+            help="Turn on memory efficient weight loading with quantization (quantize per layer during loading).",
+        )
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):
@@ -337,16 +352,9 @@ class ServerArgs:
         )
 
 
-@dataclasses.dataclass
-class ModelPortArgs:
-    nccl_port: int
-    model_tp_ips: List[str]
-    model_tp_ports: List[int]
-
-
 @dataclasses.dataclass
 class PortArgs:
     tokenizer_port: int
-    router_port: int
+    controller_port: int
     detokenizer_port: int
-    model_port_args: List[ModelPortArgs]
+    nccl_ports: List[int]