sglang 0.5.4__py3-none-any.whl → 0.5.4.post1__py3-none-any.whl

sglang/srt/entrypoints/http_server.py

@@ -20,7 +20,7 @@ This file implements HTTP APIs for the inference engine via fastapi.
 import asyncio
 import dataclasses
 import logging
-import multiprocessing as multiprocessing
+import multiprocessing
 import os
 import tempfile
 import threading
@@ -165,6 +165,7 @@ async def init_multi_tokenizer() -> ServerArgs:
         server_args.api_key is None
     ), "API key is not supported in multi-tokenizer mode"
 
+    # Create a new ipc name for the current process
     port_args.tokenizer_ipc_name = (
         f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}"
     )
@@ -184,6 +185,7 @@ async def init_multi_tokenizer() -> ServerArgs:
     )
 
     tokenizer_manager.max_req_input_len = scheduler_info["max_req_input_len"]
+
     set_global_state(
         _GlobalState(
             tokenizer_manager=tokenizer_manager,
@@ -192,36 +194,35 @@ async def init_multi_tokenizer() -> ServerArgs:
         )
     )
 
-    if server_args.enable_trace:
-        process_tracing_init(server_args.oltp_traces_endpoint, "sglang")
-        if server_args.disaggregation_mode == "null":
-            thread_label = f"MultiTokenizer-{tokenizer_manager.worker_id}"
-            trace_set_thread_info(thread_label)
-
     return server_args
 
 
 @asynccontextmanager
 async def lifespan(fast_api_app: FastAPI):
-    if not getattr(fast_api_app, "is_single_tokenizer_mode", False):
+    if getattr(fast_api_app, "is_single_tokenizer_mode", False):
+        server_args = fast_api_app.server_args
+        warmup_thread_args = fast_api_app.warmup_thread_args
+        thread_label = "Tokenizer"
+    else:
         # Initialize multi-tokenizer support for worker processes
-        fast_api_app.server_args: ServerArgs = await init_multi_tokenizer()
-
-        # only metrics middleware is supported in multi-tokenizer mode
-        worker_pid = os.getpid()
-        if fast_api_app.server_args.enable_metrics:
-            add_prometheus_middleware(app)
-            enable_func_timer()
-
-        logger.info(f"Worker {worker_pid} added prometheus middleware")
-        fast_api_app.warmup_thread = threading.Thread(
-            target=_wait_and_warmup,
-            args=(
-                fast_api_app.server_args,
-                None,  # pipe_finish_writer not needed in worker
-                None,  # launch_callback not needed in worker
-            ),
+        server_args = await init_multi_tokenizer()
+        warmup_thread_args = (
+            server_args,
+            None,
+            None,
         )
+        thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
+
+    # Add prometheus middleware
+    if server_args.enable_metrics:
+        add_prometheus_middleware(app)
+        enable_func_timer()
+
+    # Init tracing
+    if server_args.enable_trace:
+        process_tracing_init(server_args.oltp_traces_endpoint, "sglang")
+        if server_args.disaggregation_mode == "null":
+            trace_set_thread_info(thread_label)
 
     # Initialize OpenAI serving handlers
     fast_api_app.state.openai_serving_completion = OpenAIServingCompletion(
@@ -249,8 +250,7 @@ async def lifespan(fast_api_app: FastAPI):
         _global_state.tokenizer_manager
     )
 
-    server_args: ServerArgs = fast_api_app.server_args
-
+    # Launch tool server
     tool_server = None
     if server_args.tool_server == "demo":
         from sglang.srt.entrypoints.openai.tool_server import DemoToolServer
@@ -274,12 +274,11 @@ async def lifespan(fast_api_app: FastAPI):
             enable_force_include_usage=True,
             tool_server=tool_server,
         )
-    except Exception as e:
-        import traceback
-
-        traceback.print_exc()
-        logger.warning(f"Can not initialize OpenAIServingResponses, error: {e}")
+    except Exception:
+        traceback = get_exception_traceback()
+        logger.warning(f"Can not initialize OpenAIServingResponses, error: {traceback}")
 
+    # Execute custom warmups
     if server_args.warmups is not None:
         await execute_warmups(
             server_args.disaggregation_mode,
@@ -288,18 +287,18 @@ async def lifespan(fast_api_app: FastAPI):
         )
         logger.info("Warmup ended")
 
-    warmup_thread = getattr(fast_api_app, "warmup_thread", None)
-    if warmup_thread is not None:
-        warmup_thread.start()
+    # Execute the general warmup
+    warmup_thread = threading.Thread(
+        target=_wait_and_warmup,
+        args=warmup_thread_args,
+    )
+    warmup_thread.start()
 
+    # Start the HTTP server
     try:
         yield
     finally:
-        if server_args.tokenizer_worker_num > 1:
-            pid = os.getpid()
-            logger.info(f"uvicorn worker {pid} ending...")
-            warmup_thread.join()
-            logger.info(f"uvicorn worker {pid} ended.")
+        warmup_thread.join()
 
 
 # Fast API
@@ -499,6 +498,11 @@ async def get_server_info():
     internal_states: List[Dict[Any, Any]] = (
         await _global_state.tokenizer_manager.get_internal_state()
     )
+
+    # This field is not serializable.
+    if hasattr(_global_state.tokenizer_manager.server_args, "model_config"):
+        del _global_state.tokenizer_manager.server_args.model_config
+
     return {
         **dataclasses.asdict(_global_state.tokenizer_manager.server_args),
         **_global_state.scheduler_info,
@@ -1328,27 +1332,12 @@ def launch_server(
         3. DetokenizerManager (subprocess): Detokenizes the output tokens and sends the result back to the Tokenizer Manager.
 
     Note:
-    1. The HTTP server, Engine, and TokenizerManager both run in the main process.
+    1. The HTTP server, Engine, and TokenizerManager all run in the main process.
     2. Inter-process communication is done through IPC (each process uses a different port) via the ZMQ library.
     """
-    if server_args.tokenizer_worker_num > 1:
-        port_args = PortArgs.init_new(server_args)
-        port_args.tokenizer_worker_ipc_name = (
-            f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}"
-        )
-        tokenizer_manager, template_manager, scheduler_info = _launch_subprocesses(
-            server_args=server_args, port_args=port_args
-        )
-    else:
-        tokenizer_manager, template_manager, scheduler_info = _launch_subprocesses(
-            server_args=server_args,
-        )
-
-        if server_args.enable_trace:
-            process_tracing_init(server_args.oltp_traces_endpoint, "sglang")
-            if server_args.disaggregation_mode == "null":
-                thread_label = "Tokenizer"
-                trace_set_thread_info(thread_label)
+    tokenizer_manager, template_manager, scheduler_info, port_args = (
+        _launch_subprocesses(server_args=server_args)
+    )
 
     set_global_state(
         _GlobalState(
@@ -1358,40 +1347,45 @@ def launch_server(
         )
     )
 
-    if server_args.tokenizer_worker_num > 1:
-        multi_tokenizer_args_shm = write_data_for_multi_tokenizer(
-            port_args,
+    # Pass additional arguments to the lifespan function.
+    # They will be used for additional initialization setups.
+    if server_args.tokenizer_worker_num == 1:
+        # If it is single tokenizer mode, we can pass the arguments by attributes of the app object.
+        app.is_single_tokenizer_mode = True
+        app.server_args = server_args
+        app.warmup_thread_args = (
             server_args,
-            scheduler_info,
+            pipe_finish_writer,
+            launch_callback,
         )
-    else:
+
         # Add api key authorization
+        # This is only supported in single tokenizer mode.
         if server_args.api_key:
             add_api_key_middleware(app, server_args.api_key)
-
-        # Add prometheus middleware
-        if server_args.enable_metrics:
-            add_prometheus_middleware(app)
-            enable_func_timer()
-
-        # Send a warmup request - we will create the thread launch it
-        # in the lifespan after all other warmups have fired.
-        warmup_thread = threading.Thread(
-            target=_wait_and_warmup,
-            args=(
-                server_args,
-                pipe_finish_writer,
-                launch_callback,
-            ),
+    else:
+        # If it is multi-tokenizer mode, we need to write the arguments to shared memory
+        # for other worker processes to read.
+        app.is_single_tokenizer_mode = False
+        multi_tokenizer_args_shm = write_data_for_multi_tokenizer(
+            port_args, server_args, scheduler_info
         )
-        app.warmup_thread = warmup_thread
 
     try:
         # Update logging configs
         set_uvicorn_logging_configs()
-        app.server_args = server_args
+
         # Listen for HTTP requests
-        if server_args.tokenizer_worker_num > 1:
+        if server_args.tokenizer_worker_num == 1:
+            uvicorn.run(
+                app,
+                host=server_args.host,
+                port=server_args.port,
+                log_level=server_args.log_level_http or server_args.log_level,
+                timeout_keep_alive=5,
+                loop="uvloop",
+            )
+        else:
             from uvicorn.config import LOGGING_CONFIG
 
             LOGGING_CONFIG["loggers"]["sglang.srt.entrypoints.http_server"] = {
@@ -1399,7 +1393,6 @@ def launch_server(
                 "level": "INFO",
                 "propagate": False,
             }
-
             monkey_patch_uvicorn_multiprocessing()
 
             uvicorn.run(
@@ -1411,22 +1404,10 @@ def launch_server(
                 loop="uvloop",
                 workers=server_args.tokenizer_worker_num,
             )
-        else:
-            app.is_single_tokenizer_mode = True
-            uvicorn.run(
-                app,
-                host=server_args.host,
-                port=server_args.port,
-                log_level=server_args.log_level_http or server_args.log_level,
-                timeout_keep_alive=5,
-                loop="uvloop",
-            )
     finally:
         if server_args.tokenizer_worker_num > 1:
             multi_tokenizer_args_shm.unlink()
             _global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
-        else:
-            warmup_thread.join()
 
 
 def _execute_server_warmup(

sglang/srt/environ.py

@@ -111,18 +111,21 @@ class Envs:
     # Model & File Download
     SGLANG_USE_MODELSCOPE = EnvBool(False)
 
+    # Logging Options
+    SGLANG_LOG_GC = EnvBool(False)
+    SGLANG_LOG_FORWARD_ITERS = EnvBool(False)
+    SGLANG_DISABLE_REQUEST_LOGGING = EnvBool(False)
+
     # Test & Debug
     SGLANG_IS_IN_CI = EnvBool(False)
     SGLANG_IS_IN_CI_AMD = EnvBool(False)
     SGLANG_SET_CPU_AFFINITY = EnvBool(False)
     SGLANG_PROFILE_WITH_STACK = EnvBool(True)
     SGLANG_RECORD_STEP_TIME = EnvBool(False)
-    SGLANG_GC_LOG = EnvBool(False)
     SGLANG_FORCE_SHUTDOWN = EnvBool(False)
     SGLANG_DEBUG_MEMORY_POOL = EnvBool(False)
     SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False)
     SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False)
-    SGLANG_DISABLE_REQUEST_LOGGING = EnvBool(False)
     SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1)
     SGLANG_SIMULATE_ACC_METHOD = EnvStr("multinomial")
     SGLANG_TORCH_PROFILER_DIR = EnvStr("/tmp")
@@ -228,6 +231,7 @@ class Envs:
     SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256)
 
     # Overlap Spec V2
+    SGLANG_ENABLE_SPEC_V2 = EnvBool(False)
     SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False)
 
     # VLM
@@ -251,7 +255,17 @@ class Envs:
 envs = Envs()
 
 
+def _print_deprecated_env(new_name: str, old_name: str):
+    if old_name in os.environ:
+        warnings.warn(
+            f"Environment variable {old_name} will be deprecated, please use {new_name} instead"
+        )
+        os.environ[new_name] = os.environ[old_name]
+
+
 def _convert_SGL_to_SGLANG():
+    _print_deprecated_env("SGLANG_LOG_GC", "SGLANG_GC_LOG")
+
     for key, value in os.environ.items():
         if key.startswith("SGL_"):
             new_key = key.replace("SGL_", "SGLANG_", 1)

sglang/srt/eplb/expert_distribution.py

@@ -415,10 +415,19 @@ class _DetailSinglePassGatherer(_SinglePassGatherer):
 
     def collect(self) -> Dict:
         num_tokens = len(self._metadata["input_ids"])
+
+        global_physical_count = _convert_per_token_to_global_physical_count(
+            num_tokens,
+            num_layers=self._expert_location_metadata.num_layers,
+            num_physical_experts=self._expert_location_metadata.num_physical_experts,
+            _topk_ids_of_layer=self._topk_ids_of_layer,
+        )
+
         return dict(
             **self._metadata,
             topk_ids_of_layer=self._topk_ids_of_layer[:, :num_tokens, :].clone().cpu(),
             misc_objects=self._misc_objects,
+            global_physical_count=global_physical_count,
         )
 
 
@@ -547,6 +556,27 @@ class _DeepepLowLatencySinglePassGatherer(_LayerBasedGpuSinglePassGatherer):
         self._data[layer_idx, :] += local_physical_count_of_layer
 
 
+def _convert_per_token_to_global_physical_count(
+    num_tokens: int,
+    num_layers: int,
+    num_physical_experts: int,
+    _topk_ids_of_layer: torch.Tensor,
+) -> torch.Tensor:
+    topk_ids_layer_major = _topk_ids_of_layer[:, :num_tokens, :].reshape(num_layers, -1)
+    mask = topk_ids_layer_major != -1
+
+    index = topk_ids_layer_major.masked_fill(~mask, 0).long()
+    src = mask.int()
+
+    ans = torch.zeros(
+        (num_layers, num_physical_experts),
+        dtype=_topk_ids_of_layer.dtype,
+        device=_topk_ids_of_layer.device,
+    )
+    ans.scatter_add_(dim=1, index=index, src=src)
+    return ans
+
+
 def _convert_local_to_global_physical_count(
     local_physical_count: torch.Tensor,
     rank: int,

sglang/srt/function_call/function_call_parser.py

@@ -16,6 +16,7 @@ from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector
 from sglang.srt.function_call.gpt_oss_detector import GptOssDetector
 from sglang.srt.function_call.kimik2_detector import KimiK2Detector
 from sglang.srt.function_call.llama32_detector import Llama32Detector
+from sglang.srt.function_call.minimax_m2 import MinimaxM2Detector
 from sglang.srt.function_call.mistral_detector import MistralDetector
 from sglang.srt.function_call.pythonic_detector import PythonicDetector
 from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
@@ -49,6 +50,7 @@ class FunctionCallParser:
         "qwen25": Qwen25Detector,
         "qwen3_coder": Qwen3CoderDetector,
         "step3": Step3Detector,
+        "minimax-m2": MinimaxM2Detector,
     }
 
     def __init__(self, tools: List[Tool], tool_call_parser: str):