sglang 0.2.8__py3-none-any.whl → 0.2.9.post1__py3-none-any.whl

sglang/bench_serving.py

@@ -21,7 +21,7 @@ import sys
 import time
 import traceback
 import warnings
-from argparse import ArgumentParser as FlexibleArgumentParser
+from argparse import ArgumentParser
 from dataclasses import dataclass, field
 from datetime import datetime
 from typing import AsyncGenerator, List, Optional, Tuple, Union
@@ -868,14 +868,12 @@ def set_ulimit(target_soft_limit=65535):
 
 
 if __name__ == "__main__":
-    parser = FlexibleArgumentParser(
-        description="Benchmark the online serving throughput."
-    )
+    parser = ArgumentParser(description="Benchmark the online serving throughput.")
     parser.add_argument(
         "--backend",
         type=str,
-        required=True,
         choices=list(ASYNC_REQUEST_FUNCS.keys()),
+        default="sglang",
         help="Must specify a backend, depending on the LLM Inference Engine.",
     )
     parser.add_argument(

sglang/check_env.py

@@ -30,6 +30,7 @@ PACKAGE_LIST = [
     "zmq",
     "vllm",
     "outlines",
+    "multipart",
     "openai",
     "tiktoken",
     "anthropic",

sglang/srt/layers/logits_processor.py

@@ -209,7 +209,7 @@ class LogitsProcessor(nn.Module):
                 all_logits = all_logits[:, : self.config.vocab_size].float()
 
                 all_logprobs = all_logits
-                del all_logits
+                del all_logits, hidden_states
                 all_logprobs[:] = torch.nn.functional.log_softmax(all_logprobs, dim=-1)
 
                 # Get the logprob of top-k tokens

sglang/srt/managers/tokenizer_manager.py

@@ -79,6 +79,7 @@ class TokenizerManager:
         self.send_to_router.connect(f"tcp://127.0.0.1:{port_args.controller_port}")
 
         self.model_path = server_args.model_path
+        self.served_model_name = server_args.served_model_name
         self.hf_config = get_config(
             self.model_path,
             trust_remote_code=server_args.trust_remote_code,

sglang/srt/model_executor/model_runner.py

@@ -312,10 +312,12 @@ class ModelRunner:
             self.cuda_graph_runner.capture(batch_size_list)
         except RuntimeError as e:
             raise Exception(
-                f"Capture cuda graph failed: {e}. Possible solutions:\n"
-                f"1. disable cuda graph by --disable-cuda-graph\n"
-                f"2. set --mem-fraction-static to a smaller value\n"
-                f"Open an issue on GitHub with reproducible scripts if you need help.\n"
+                f"Capture cuda graph failed: {e}\n"
+                "Possible solutions:\n"
+                "1. disable torch compile by not using --enable-torch-compile\n"
+                "2. disable cuda graph by --disable-cuda-graph\n"
+                "3. set --mem-fraction-static to a smaller value\n"
+                "Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose \n"
             )
 
     @torch.inference_mode()

sglang/srt/openai_api/adapter.py

@@ -594,7 +594,7 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
 
 def v1_chat_generate_request(all_requests, tokenizer_manager):
 
-    texts = []
+    input_ids = []
     sampling_params_list = []
     image_data_list = []
     return_logprobs = []
@@ -608,8 +608,8 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
         if not isinstance(request.messages, str):
             # Apply chat template and its stop strings.
             if chat_template_name is None:
-                prompt = tokenizer_manager.tokenizer.apply_chat_template(
-                    request.messages, tokenize=False, add_generation_prompt=True
+                prompt_ids = tokenizer_manager.tokenizer.apply_chat_template(
+                    request.messages, tokenize=True, add_generation_prompt=True
                 )
                 stop = request.stop
                 image_data = None
@@ -623,12 +623,13 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
                         stop.append(request.stop)
                     else:
                         stop.extend(request.stop)
+                prompt_ids = tokenizer_manager.tokenizer.encode(prompt)
         else:
             # Use the raw prompt and stop strings if the messages is already a string.
             prompt = request.messages
             stop = request.stop
             image_data = None
-        texts.append(prompt)
+        input_ids.append(prompt_ids)
         return_logprobs.append(request.logprobs)
         top_logprobs_nums.append(request.top_logprobs)
         sampling_params_list.append(
@@ -645,13 +646,13 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
         )
         image_data_list.append(image_data)
     if len(all_requests) == 1:
-        texts = texts[0]
+        input_ids = input_ids[0]
         sampling_params_list = sampling_params_list[0]
         image_data = image_data_list[0]
         return_logprobs = return_logprobs[0]
         top_logprobs_nums = top_logprobs_nums[0]
     adapted_request = GenerateReqInput(
-        text=texts,
+        input_ids=input_ids,
         image_data=image_data,
         sampling_params=sampling_params_list,
         return_logprob=return_logprobs,

sglang/srt/server.py

@@ -72,6 +72,7 @@ from sglang.srt.utils import (
     allocate_init_ports,
     assert_pkg_version,
     enable_show_time_cost,
+    kill_child_process,
     maybe_set_triton_cache_manager,
     set_ulimit,
 )
@@ -189,10 +190,10 @@ async def retrieve_file_content(file_id: str):
 @app.get("/v1/models")
 def available_models():
     """Show available models."""
-    model_names = [tokenizer_manager.model_path]
+    served_model_names = [tokenizer_manager.served_model_name]
     model_cards = []
-    for model_name in model_names:
-        model_cards.append(ModelCard(id=model_name, root=model_name))
+    for served_model_name in served_model_names:
+        model_cards.append(ModelCard(id=served_model_name, root=served_model_name))
     return ModelList(data=model_cards)
 
 
@@ -467,16 +468,7 @@ class Runtime:
 
     def shutdown(self):
         if self.pid is not None:
-            try:
-                parent = psutil.Process(self.pid)
-            except psutil.NoSuchProcess:
-                return
-            children = parent.children(recursive=True)
-            for child in children:
-                child.kill()
-            psutil.wait_procs(children, timeout=5)
-            parent.kill()
-            parent.wait(timeout=5)
+            kill_child_process(self.pid)
             self.pid = None
 
     def cache_prefix(self, prefix: str):

sglang/srt/server_args.py

@@ -32,6 +32,7 @@ class ServerArgs:
     trust_remote_code: bool = True
     context_length: Optional[int] = None
     quantization: Optional[str] = None
+    served_model_name: Optional[str] = None
     chat_template: Optional[str] = None
 
     # Port
@@ -90,6 +91,10 @@ class ServerArgs:
     def __post_init__(self):
         if self.tokenizer_path is None:
             self.tokenizer_path = self.model_path
+
+        if self.served_model_name is None:
+            self.served_model_name = self.model_path
+
         if self.mem_fraction_static is None:
             if self.tp_size >= 16:
                 self.mem_fraction_static = 0.79
@@ -202,6 +207,12 @@ class ServerArgs:
             ],
             help="The quantization method.",
         )
+        parser.add_argument(
+            "--served-model-name",
+            type=str,
+            default=ServerArgs.served_model_name,
+            help="Override the model name returned by the v1/models endpoint in OpenAI API server.",
+        )
         parser.add_argument(
             "--chat-template",
             type=str,

sglang/srt/utils.py

@@ -366,6 +366,26 @@ def kill_parent_process():
     os.kill(parent_process.pid, 9)
 
 
+def kill_child_process(pid, including_parent=True):
+    try:
+        parent = psutil.Process(pid)
+    except psutil.NoSuchProcess:
+        return
+
+    children = parent.children(recursive=True)
+    for child in children:
+        try:
+            child.kill()
+        except psutil.NoSuchProcess:
+            pass
+
+    if including_parent:
+        try:
+            parent.kill()
+        except psutil.NoSuchProcess:
+            pass
+
+
 def monkey_patch_vllm_p2p_access_check(gpu_id: int):
     """
     Monkey patch the slow p2p access check in vllm.

sglang/test/run_eval.py

@@ -0,0 +1,104 @@
+"""
+Usage:
+python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10
+"""
+
+import argparse
+import json
+import os
+import time
+
+from sglang.test.simple_eval_common import (
+    ChatCompletionSampler,
+    download_dataset,
+    make_report,
+    set_ulimit,
+)
+
+
+def run_eval(args):
+    if "OPENAI_API_KEY" not in os.environ:
+        os.environ["OPENAI_API_KEY"] = "EMPTY"
+
+    base_url = (
+        f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
+    )
+
+    if args.eval_name == "mmlu":
+        from sglang.test.simple_eval_mmlu import MMLUEval
+
+        dataset_path = "mmlu.csv"
+
+        if not os.path.exists(dataset_path):
+            download_dataset(
+                dataset_path,
+                "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv",
+            )
+        eval_obj = MMLUEval(dataset_path, args.num_examples, args.num_threads)
+    elif args.eval_name == "humaneval":
+        from sglang.test.simple_eval_humaneval import HumanEval
+
+        eval_obj = HumanEval(args.num_examples, args.num_threads)
+    else:
+        raise ValueError(f"Invalid eval name: {args.eval_name}")
+
+    sampler = ChatCompletionSampler(
+        model=args.model,
+        max_tokens=2048,
+        base_url=base_url,
+    )
+
+    # Run eval
+    tic = time.time()
+    result = eval_obj(sampler)
+    latency = time.time() - tic
+
+    # Dump reports
+    metrics = result.metrics | {"score": result.score}
+    file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
+    report_filename = f"/tmp/{file_stem}.html"
+    print(f"Writing report to {report_filename}")
+    with open(report_filename, "w") as fh:
+        fh.write(make_report(result))
+    metrics = result.metrics | {"score": result.score}
+    print(metrics)
+    result_filename = f"/tmp/{file_stem}.json"
+    with open(result_filename, "w") as f:
+        f.write(json.dumps(metrics, indent=2))
+    print(f"Writing results to {result_filename}")
+
+    # Print results
+    print(f"Total latency: {latency:.3f} s")
+    print(f"Score: {metrics['score']:.3f}")
+
+    return metrics
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--base-url",
+        type=str,
+        default=None,
+        help="Server or API base url if not using http host and port.",
+    )
+    parser.add_argument(
+        "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0."
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        help="If not set, the default port is configured according to its default value for different LLM Inference Engines.",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
+    )
+    parser.add_argument("--eval-name", type=str, default="mmlu")
+    parser.add_argument("--num-examples", type=int)
+    parser.add_argument("--num-threads", type=int, default=64)
+    set_ulimit()
+    args = parser.parse_args()
+
+    run_eval(args)