sglang 0.1.17__py3-none-any.whl → 0.1.18__py3-none-any.whl

sglang/srt/server.py

@@ -1,4 +1,7 @@
-"""SRT: SGLang Runtime"""
+"""
+The entry point of inference server.
+SRT = SGLang Runtime.
+"""
 
 import asyncio
 import dataclasses
@@ -10,7 +13,7 @@ import sys
 import threading
 import time
 from http import HTTPStatus
-from typing import Optional
+from typing import Dict, Optional
 
 # Fix a bug of Python threading
 setattr(threading, "_register_atexit", lambda *args, **kwargs: None)
@@ -26,10 +29,15 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
 from sglang.backend.runtime_endpoint import RuntimeEndpoint
 from sglang.srt.constrained import disable_cache
 from sglang.srt.hf_transformers_utils import get_tokenizer
+from sglang.srt.managers.controller.manager_multi import (
+    start_controller_process as start_controller_process_multi,
+)
+from sglang.srt.managers.controller.manager_single import (
+    start_controller_process as start_controller_process_single,
+)
+from sglang.srt.managers.controller.tp_worker import ModelTpService
 from sglang.srt.managers.detokenizer_manager import start_detokenizer_process
 from sglang.srt.managers.io_struct import GenerateReqInput
-from sglang.srt.managers.controller.manager_single import start_controller_process as start_controller_process_single
-from sglang.srt.managers.controller.manager_multi import start_controller_process as start_controller_process_multi
 from sglang.srt.managers.tokenizer_manager import TokenizerManager
 from sglang.srt.openai_api_adapter import (
     load_chat_template_for_openai_api,
@@ -43,9 +51,15 @@ from sglang.srt.utils import (
     allocate_init_ports,
     assert_pkg_version,
     enable_show_time_cost,
+    send_addrs_to_rank_0,
+    receive_addrs,
+    start_rpyc_service_process,
 )
 from sglang.utils import get_exception_traceback
 
+
+logger = logging.getLogger(__name__)
+
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 
@@ -94,8 +108,11 @@ async def generate_request(obj: GenerateReqInput, request: Request):
                 yield f"data: {json.dumps(out, ensure_ascii=False)}\n\n"
             yield "data: [DONE]\n\n"
 
-        return StreamingResponse(stream_results(), media_type="text/event-stream",
-                                 background=tokenizer_manager.create_abort_task(obj))
+        return StreamingResponse(
+            stream_results(),
+            media_type="text/event-stream",
+            background=tokenizer_manager.create_abort_task(obj),
+        )
     else:
         try:
             ret = await tokenizer_manager.generate_request(obj, request).__anext__()
@@ -134,29 +151,32 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
         enable_show_time_cost()
     if server_args.disable_disk_cache:
         disable_cache()
-    if server_args.enable_flashinfer:
-        assert_pkg_version("flashinfer", "0.0.4")
+    if not server_args.disable_flashinfer:
+        assert_pkg_version("flashinfer", "0.0.8", "Please uninstall the old version and "
+                           "reinstall the latest version by following the instructions "
+                           "at https://docs.flashinfer.ai/installation.html.")
     if server_args.chat_template:
         # TODO: replace this with huggingface transformers template
         load_chat_template_for_openai_api(server_args.chat_template)
 
     # Allocate ports
+    assert server_args.tp_size % server_args.nnodes == 0
+    tp_size_local = server_args.tp_size // server_args.nnodes
     server_args.port, server_args.additional_ports = allocate_init_ports(
         server_args.port,
         server_args.additional_ports,
-        server_args.tp_size,
+        tp_size_local,
         server_args.dp_size,
     )
 
-    # Init local models port args
     ports = server_args.additional_ports
-    tp = server_args.tp_size
     model_port_args = []
     for i in range(server_args.dp_size):
         model_port_args.append(
             ModelPortArgs(
-                nccl_port=ports[3 + i * (tp + 1)],
-                model_tp_ports=ports[3 + i * (tp + 1) + 1 : 3 + (i + 1) * (tp + 1)],
+                nccl_port=ports[3 + i * (tp_size_local + 1)],
+                model_tp_ips=[None] * tp_size_local,
+                model_tp_ports=ports[3 + i * (tp_size_local + 1) + 1 : 3 + (i + 1) * (tp_size_local + 1)],
             )
         )
     port_args = PortArgs(
@@ -166,6 +186,20 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
         model_port_args=model_port_args,
     )
 
+    # TODO multi-node dp is not supported
+    assert not (server_args.dp_size > 1 and server_args.node_rank is not None)
+    if server_args.nnodes > 1:
+        if server_args.node_rank != 0:
+            send_addrs_to_rank_0(model_port_args[0], server_args)
+        else:
+            receive_addrs(model_port_args[0], server_args)
+        for i in range(tp_size_local):
+            start_rpyc_service_process(ModelTpService, model_port_args[0].model_tp_ports[i])
+        if server_args.node_rank != 0:
+            logger.info(f"[node_rank={server_args.node_rank}]: Listen for connections...")
+            while True:
+                pass
+
     # Launch processes
     tokenizer_manager = TokenizerManager(server_args, port_args, model_overide_args)
     pipe_router_reader, pipe_router_writer = mp.Pipe(duplex=False)
@@ -223,7 +257,7 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
             try:
                 requests.get(url + "/get_model_info", timeout=5, headers=headers)
                 break
-            except requests.exceptions.RequestException as e:
+            except requests.exceptions.RequestException:
                 pass
 
         # Send a warmup request
@@ -235,19 +269,20 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
                         "text": "The capital city of France is",
                         "sampling_params": {
                             "temperature": 0,
-                            "max_new_tokens": 16,
+                            "max_new_tokens": 8,
                         },
                     },
                     headers=headers,
                     timeout=600,
                 )
                 assert res.status_code == 200
-        except Exception:
+        except Exception as e:
             if pipe_finish_writer is not None:
                 pipe_finish_writer.send(get_exception_traceback())
-            print(f"Initialization failed. warmup error: {e}")
+            print(f"Initialization failed. warmup error: {e}", flush=True)
             raise e
 
+        logger.info("The server is fired up and ready to roll!")
         if pipe_finish_writer is not None:
             pipe_finish_writer.send("init ok")
 
@@ -260,7 +295,7 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
             app,
             host=server_args.host,
             port=server_args.port,
-            log_level=server_args.log_level,
+            log_level=server_args.log_level_http or server_args.log_level,
             timeout_keep_alive=5,
             loop="uvloop",
         )
@@ -269,6 +304,12 @@ def launch_server(server_args: ServerArgs, pipe_finish_writer, model_overide_arg
 
 
 class Runtime:
+    """
+    A wrapper for the server.
+    This is used for launching the server in a python program without
+    using the commond line interface.
+    """
+
     def __init__(
         self,
         log_level: str = "error",
@@ -339,7 +380,7 @@ class Runtime:
     async def add_request(
         self,
         prompt: str,
-        sampling_params,
+        sampling_params: Dict,
     ):
         json_data = {
             "text": prompt,

sglang/srt/server_args.py

@@ -11,12 +11,13 @@ class ServerArgs:
     # Model and tokenizer
     model_path: str
     tokenizer_path: Optional[str] = None
-    load_format: str = "auto"
     tokenizer_mode: str = "auto"
-    chat_template: Optional[str] = None
+    load_format: str = "auto"
+    dtype: str = "auto"
     trust_remote_code: bool = True
     context_length: Optional[int] = None
     quantization: Optional[str] = None
+    chat_template: Optional[str] = None
 
     # Port
     host: str = "127.0.0.1"
@@ -37,9 +38,8 @@ class ServerArgs:
 
     # Logging
     log_level: str = "info"
+    log_level_http: Optional[str] = None
     log_requests: bool = False
-    disable_log_stats: bool = False
-    log_stats_interval: int = 10
     show_time_cost: bool = False
 
     # Other
@@ -50,11 +50,16 @@ class ServerArgs:
     load_balance_method: str = "round_robin"
 
     # Optimization/debug options
-    enable_flashinfer: bool = False
-    attention_reduce_in_fp32: bool = False
+    disable_flashinfer: bool = False
     disable_radix_cache: bool = False
     disable_regex_jump_forward: bool = False
     disable_disk_cache: bool = False
+    attention_reduce_in_fp32: bool = False
+
+    # Distributed args
+    nccl_init_addr: Optional[str] = None
+    nnodes: int = 1
+    node_rank: Optional[int] = None
 
     def __post_init__(self):
         if self.tokenizer_path is None:
@@ -101,7 +106,16 @@ class ServerArgs:
             type=int,
             nargs="*",
             default=[],
-            help="Additional ports specified for the server.",
+            help="The additional ports specified for the server.",
+        )
+        parser.add_argument(
+            "--tokenizer-mode",
+            type=str,
+            default=ServerArgs.tokenizer_mode,
+            choices=["auto", "slow"],
+            help="Tokenizer mode. 'auto' will use the fast "
+            "tokenizer if available, and 'slow' will "
+            "always use the slow tokenizer.",
         )
         parser.add_argument(
             "--load-format",
@@ -120,20 +134,20 @@ class ServerArgs:
             "which is mainly for profiling.",
         )
         parser.add_argument(
-            "--tokenizer-mode",
-            type=str,
-            default=ServerArgs.tokenizer_mode,
-            choices=["auto", "slow"],
-            help="Tokenizer mode. 'auto' will use the fast "
-            "tokenizer if available, and 'slow' will "
-            "always use the slow tokenizer.",
-        )
-        parser.add_argument(
-            "--chat-template",
+            "--dtype",
             type=str,
-            default=ServerArgs.chat_template,
-            help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server",
-        )
+            default=ServerArgs.dtype,
+            choices=[
+                "auto", "half", "float16", "bfloat16", "float", "float32"
+            ],
+            help='Data type for model weights and activations.\n\n'
+            '* "auto" will use FP16 precision for FP32 and FP16 models, and '
+            'BF16 precision for BF16 models.\n'
+            '* "half" for FP16. Recommended for AWQ quantization.\n'
+            '* "float16" is the same as "half".\n'
+            '* "bfloat16" for a balance between precision and range.\n'
+            '* "float" is shorthand for FP32 precision.\n'
+            '* "float32" for FP32 precision.')
         parser.add_argument(
             "--trust-remote-code",
             action="store_true",
@@ -151,6 +165,12 @@ class ServerArgs:
             default=ServerArgs.quantization,
             help="The quantization method.",
         )
+        parser.add_argument(
+            "--chat-template",
+            type=str,
+            default=ServerArgs.chat_template,
+            help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
+        )
         parser.add_argument(
             "--mem-fraction-static",
             type=float,
@@ -174,7 +194,7 @@ class ServerArgs:
             type=str,
             default=ServerArgs.schedule_heuristic,
             choices=["lpm", "random", "fcfs", "dfs-weight"],
-            help="Scheduling Heuristic.",
+            help="The scheduling heuristic.",
         )
         parser.add_argument(
             "--schedule-conservativeness",
@@ -186,7 +206,7 @@ class ServerArgs:
             "--tp-size",
             type=int,
             default=ServerArgs.tp_size,
-            help="Tensor parallelism size.",
+            help="The tensor parallelism size.",
         )
         parser.add_argument(
             "--stream-interval",
@@ -198,29 +218,24 @@ class ServerArgs:
             "--random-seed",
             type=int,
             default=ServerArgs.random_seed,
-            help="Random seed.",
+            help="The random seed.",
         )
         parser.add_argument(
             "--log-level",
             type=str,
             default=ServerArgs.log_level,
-            help="Logging level",
+            help="The logging level of all loggers.",
         )
         parser.add_argument(
-            "--log-requests",
-            action="store_true",
-            help="Log all requests",
+            "--log-level-http",
+            type=str,
+            default=ServerArgs.log_level_http,
+            help="The logging level of HTTP server. If not set, reuse --log-level by default.",
         )
         parser.add_argument(
-            "--disable-log-stats",
+            "--log-requests",
             action="store_true",
-            help="Disable logging throughput stats.",
-        )
-        parser.add_argument(
-            "--log-stats-interval",
-            type=int,
-            default=ServerArgs.log_stats_interval,
-            help="Log stats interval in second.",
+            help="Log the inputs and outputs of all requests.",
         )
         parser.add_argument(
             "--show-time-cost",
@@ -239,29 +254,42 @@ class ServerArgs:
             "--dp-size",
             type=int,
             default=ServerArgs.dp_size,
-            help="Data parallelism size.",
+            help="The data parallelism size.",
         )
         parser.add_argument(
             "--load-balance-method",
             type=str,
             default=ServerArgs.load_balance_method,
-            help="Load balancing strategy for data parallelism.",
+            help="The load balancing strategy for data parallelism.",
             choices=[
                 "round_robin",
                 "shortest_queue",
             ],
         )
 
-        # Optimization/debug options
+        # Multi-node distributed serving args
         parser.add_argument(
-            "--enable-flashinfer",
-            action="store_true",
-            help="Enable flashinfer inference kernels",
+            "--nccl-init-addr",
+            type=str,
+            help="The nccl init address of multi-node server."
         )
         parser.add_argument(
-            "--attention-reduce-in-fp32",
+            "--nnodes",
+            type=int,
+            default=1,
+            help="The number of nodes."
+        )
+        parser.add_argument(
+            "--node-rank",
+            type=int,
+            help="The node rank."
+        )
+
+        # Optimization/debug options
+        parser.add_argument(
+            "--disable-flashinfer",
             action="store_true",
-            help="Cast the intermidiate attention results to fp32 to avoid possible crashes related to fp16.",
+            help="Disable flashinfer inference kernels",
         )
         parser.add_argument(
             "--disable-radix-cache",
@@ -278,6 +306,12 @@ class ServerArgs:
             action="store_true",
             help="Disable disk cache to avoid possible crashes related to file system or high concurrency.",
         )
+        parser.add_argument(
+            "--attention-reduce-in-fp32",
+            action="store_true",
+            help="Cast the intermidiate attention results to fp32 to avoid possible crashes related to fp16."
+            "This only affects Triton attention kernels",
+        )
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):
@@ -289,7 +323,7 @@ class ServerArgs:
 
     def print_mode_args(self):
         return (
-            f"enable_flashinfer={self.enable_flashinfer}, "
+            f"disable_flashinfer={self.disable_flashinfer}, "
             f"attention_reduce_in_fp32={self.attention_reduce_in_fp32}, "
             f"disable_radix_cache={self.disable_radix_cache}, "
             f"disable_regex_jump_forward={self.disable_regex_jump_forward}, "
@@ -300,6 +334,7 @@ class ServerArgs:
 @dataclasses.dataclass
 class ModelPortArgs:
     nccl_port: int
+    model_tp_ips: List[str]
     model_tp_ports: List[int]