sglang 0.5.3rc2__py3-none-any.whl → 0.5.4__py3-none-any.whl

sglang/test/runners.py

@@ -519,6 +519,7 @@ class SRTRunner:
         lora_target_modules: Optional[List[str]] = None,
         enable_lora: Optional[bool] = None,
         max_loaded_loras: Optional[int] = None,
+        lora_eviction_policy: str = "lru",
     ):
         self.model_type = model_type
         self.is_generation = model_type == "generation"
@@ -565,6 +566,7 @@ class SRTRunner:
             lora_target_modules=lora_target_modules,
             enable_lora=enable_lora,
             max_loaded_loras=max_loaded_loras,
+            lora_eviction_policy=lora_eviction_policy,
             **spec_kwargs,
         )
 

sglang/test/send_one.py

@@ -3,6 +3,8 @@ Run one test prompt.
 
 Usage:
 python3 -m sglang.test.send_one
+python3 -m sglang.test.send_one --profile --profile-steps 5
+python3 -m sglang.test.send_one --profile --profile-by-stage
 """
 
 import argparse
@@ -10,6 +12,9 @@ import dataclasses
 import json
 
 import requests
+import tabulate
+
+from sglang.profiler import run_profile
 
 
 @dataclasses.dataclass
@@ -29,6 +34,9 @@ class BenchArgs:
     image: bool = False
     many_images: bool = False
     stream: bool = False
+    profile: bool = False
+    profile_steps: int = 3
+    profile_by_stage: bool = False
 
     @staticmethod
     def add_cli_args(parser: argparse.ArgumentParser):
@@ -51,6 +59,11 @@ class BenchArgs:
         parser.add_argument("--image", action="store_true")
         parser.add_argument("--many-images", action="store_true")
         parser.add_argument("--stream", action="store_true")
+        parser.add_argument("--profile", action="store_true")
+        parser.add_argument(
+            "--profile-steps", type=int, default=BenchArgs.profile_steps
+        )
+        parser.add_argument("--profile-by-stage", action="store_true")
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):
@@ -59,6 +72,8 @@ class BenchArgs:
 
 
 def send_one_prompt(args):
+    base_url = f"http://{args.host}:{args.port}"
+
     if args.image:
         args.prompt = (
             "Human: Describe this image in a very short sentence.\n\nAssistant:"
@@ -108,19 +123,35 @@ def send_one_prompt(args):
         "stream": args.stream,
     }
 
+    # Run profiler if requested
+    if args.profile:
+        print(f"Running profiler with {args.profile_steps} steps...")
+        run_profile(
+            base_url,
+            args.profile_steps,
+            ["CPU", "GPU"],
+            None,
+            None,
+            args.profile_by_stage,
+        )
+
     response = requests.post(
-        f"http://{args.host}:{args.port}/generate",
+        f"{base_url}/generate",
         json=json_data,
         stream=args.stream,
     )
 
     if args.stream:
+        last_len = 0
         for chunk in response.iter_lines(decode_unicode=False):
             chunk = chunk.decode("utf-8")
             if chunk and chunk.startswith("data:"):
                 if chunk == "data: [DONE]":
                     break
                 ret = json.loads(chunk[5:].strip("\n"))
+                chunk_str = ret["text"][last_len:]
+                last_len = len(ret["text"])
+                print(chunk_str, end="", flush=True)
     else:
         ret = response.json()
 
@@ -131,21 +162,25 @@ def send_one_prompt(args):
         print(ret)
         return 0, 0
 
-    latency = ret["meta_info"]["e2e_latency"]
-
-    if "spec_verify_ct" in ret["meta_info"]:
+    if "spec_verify_ct" in ret["meta_info"] and ret["meta_info"]["spec_verify_ct"] > 0:
         acc_length = (
             ret["meta_info"]["completion_tokens"] / ret["meta_info"]["spec_verify_ct"]
         )
     else:
         acc_length = 1.0
 
+    latency = ret["meta_info"]["e2e_latency"]
     speed = ret["meta_info"]["completion_tokens"] / latency
+    tokens = ret["meta_info"]["completion_tokens"]
+
+    if not args.stream:
+        print(ret["text"])
 
-    print(ret["text"])
     print()
-    print(f"{acc_length=:.2f}")
-    print(f"{speed=:.2f} token/s")
+    headers = ["Latency (s)", "Tokens", "Acc Length", "Speed (token/s)"]
+    rows = [[f"{latency:.3f}", f"{tokens}", f"{acc_length:.3f}", f"{speed:.2f}"]]
+    msg = tabulate.tabulate(rows, headers=headers, tablefmt="pretty")
+    print(msg)
 
     return acc_length, speed
 

sglang/test/simple_eval_common.py

@@ -290,6 +290,9 @@ def aggregate_results(
     htmls = []
     convos = []
     for single_eval_result in single_eval_results:
+        # Skip None results
+        if single_eval_result is None:
+            continue
         for name, value in single_eval_result.metrics.items():
             name2values[name].append(value)
         if single_eval_result.score is not None:

sglang/test/simple_eval_gpqa.py

@@ -18,7 +18,6 @@ from sglang.test.simple_eval_common import (
     HTML_JINJA,
     Eval,
     EvalResult,
-    MessageList,
     SamplerBase,
     SingleEvalResult,
     format_multichoice_question,

sglang/test/simple_eval_humaneval.py

@@ -11,8 +11,6 @@ import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Dict, List, Optional
 
-import tqdm
-
 try:
     from human_eval.data import read_problems
     from human_eval.evaluation import estimate_pass_at_k
@@ -41,7 +39,6 @@ def evaluate_functional_correctness(
     Evaluates the functional correctness of generated samples, and writes
     results to f"{sample_file}_results.jsonl.gz"
     """
-    import copy
 
     # Check the generated samples against test suites.
     with ThreadPoolExecutor(max_workers=n_workers) as executor:

sglang/test/simple_eval_longbench_v2.py

@@ -0,0 +1,344 @@
+# Adapted from https://github.com/openai/simple-evals/
+
+"""
+LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-Context Multitasks
+Yushi Bai, Shangqing Tu, Jiajie Zhang, Hao Peng, Xiaozhi Wang, Xin Lv, Shulin Cao, Jiazheng Xu, Lei Hou, Yuxiao Dong, Jie Tang, Juanzi Li
+https://arxiv.org/abs/2412.15204
+"""
+
+import csv
+import json
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from transformers import AutoTokenizer
+
+from sglang.test import simple_eval_common as common
+from sglang.test.simple_eval_common import (
+    ANSWER_PATTERN_MULTICHOICE,
+    HTML_JINJA,
+    Eval,
+    EvalResult,
+    SamplerBase,
+    SingleEvalResult,
+)
+
+# LongBench-v2 task categories
+TASK_CATEGORIES = {
+    "single_document_qa",
+    "multi_document_qa",
+    "long_in_context_learning",
+    "long_dialogue_history",
+    "code_repo_understanding",
+    "long_structured_data",
+}
+
+DEFAULT_DATASET = "THUDM/LongBench-v2"
+DEFAULT_DATASET_SPLIT = "train"
+
+
+def format_longbench_v2_question(row: dict) -> str:
+    """Format a LongBench-v2 question using the official template."""
+    context = row.get("context", "")
+    question = row.get("question", "")
+
+    # Handle both standard format (A, B, C, D) and alternative format (choices list)
+    if "choices" in row:
+        choices = row["choices"]
+        choice_A = choices[0] if len(choices) > 0 else ""
+        choice_B = choices[1] if len(choices) > 1 else ""
+        choice_C = choices[2] if len(choices) > 2 else ""
+        choice_D = choices[3] if len(choices) > 3 else ""
+    else:
+        choice_A = row.get("A", row.get("choice_A", ""))
+        choice_B = row.get("B", row.get("choice_B", ""))
+        choice_C = row.get("C", row.get("choice_C", ""))
+        choice_D = row.get("D", row.get("choice_D", ""))
+
+    # Official LongBench-v2 template
+    prompt = f"""
+Please read the following text and answer the question below.
+<text>
+{context.strip()}
+</text>
+
+What is the correct answer to this question: {question.strip()}
+Choices:
+(A) {choice_A.strip()}
+(B) {choice_B.strip()}
+(C) {choice_C.strip()}
+(D) {choice_D.strip()}
+
+Format your response as follows: "The correct answer is (insert answer here)"."""
+
+    return prompt
+
+
+def extract_longbench_v2_answer(response: str) -> Optional[str]:
+    """Extract answer from model response using official LongBench-v2 method."""
+    response = response.replace("*", "")
+
+    # First try: "The correct answer is (A)"
+    match = re.search(r"The correct answer is \(([A-D])\)", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    # Second try: "The correct answer is A"
+    match = re.search(r"The correct answer is ([A-D])", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    # Fallback: Standard SGLang multichoice pattern
+    match = re.search(ANSWER_PATTERN_MULTICHOICE, response)
+    if match:
+        return match.group(1).upper()
+
+    # Generic fallback when model says "answer is A"
+    match = re.search(r"answer\s+is\s*\(?([A-D])\)?", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    return None
+
+
+class LongBenchV2Eval(Eval):
+    """
+    Evaluation utility for LongBench-v2 dataset.
+
+    LongBench-v2 is designed to assess the ability of LLMs to handle long-context problems
+    requiring deep understanding and reasoning across real-world multitasks.
+    """
+
+    def __init__(
+        self,
+        model: str = None,
+        data_source: str = DEFAULT_DATASET,
+        num_examples: Optional[int] = None,
+        num_threads: int = 1,
+        n_repeats: int = 1,
+        categories: Optional[List[str]] = None,
+        max_context_length: Optional[int] = None,
+        min_context_length: Optional[int] = None,
+    ):
+        """
+        Initialize LongBench-v2 evaluation.
+
+        Args:
+            data_source: HuggingFace dataset name, local file path (CSV/JSON)
+            num_examples: Number of examples to evaluate (None for all)
+            num_threads: Number of threads for parallel processing
+            n_repeats: Number of times to repeat evaluation for error bars
+            categories: List of task categories to include (None for all)
+            max_context_length: Maximum context length in characters
+            min_context_length: Minimum context length in characters
+        """
+        self.tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
+        self.min_context_length = min_context_length
+        self.max_context_length = max_context_length
+        # Load dataset based on data source type
+        examples = self._load_dataset(data_source)
+
+        # Apply filtering
+        if categories:
+            examples = [ex for ex in examples if ex.get("category") in categories]
+
+        # Sample examples if specified
+        if num_examples:
+            assert n_repeats == 1, "n_repeats only supported when not sampling examples"
+            examples = examples[: min(num_examples, len(examples))]
+
+        # Repeat examples for multiple runs
+        examples = examples * n_repeats
+
+        if not examples:
+            raise ValueError(
+                "No examples available for LongBench-v2 evaluation after filtering"
+            )
+
+        self.examples = examples
+        self.n_repeats = n_repeats
+        self.num_threads = num_threads
+
+        print(f"Loaded {len(self.examples)} examples from LongBench-v2")
+        if categories:
+            print(f"Filtered to categories: {categories}")
+        if min_context_length or max_context_length:
+            print(
+                f"Context length filter: {min_context_length}-{max_context_length} characters"
+            )
+
+    def _load_dataset(self, data_source: str) -> List[Dict[str, Any]]:
+        """Load dataset from HuggingFace hub or local files."""
+
+        if not data_source:
+            data_source = DEFAULT_DATASET
+
+        if os.path.exists(data_source):
+            raw_examples = self._load_local_file(data_source)
+        else:
+            raw_examples = self._load_hf_dataset(data_source)
+
+        return [self._normalize_example(example) for example in raw_examples]
+
+    def _load_local_file(self, path: str) -> List[Dict[str, Any]]:
+        """Load examples from a local CSV/JSON/JSONL file."""
+
+        suffix = os.path.splitext(path)[1].lower()
+        if suffix in {".json", ".jsonl"}:
+            with open(path, "r", encoding="utf-8") as fh:
+                if suffix == ".jsonl":
+                    data = [json.loads(line) for line in fh if line.strip()]
+                else:
+                    data = json.load(fh)
+        elif suffix == ".csv":
+            with open(path, "r", encoding="utf-8") as fh:
+                reader = csv.DictReader(fh)
+                data = list(reader)
+        else:
+            # Try JSON, then CSV as fallback
+            try:
+                with open(path, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+            except json.JSONDecodeError:
+                with open(path, "r", encoding="utf-8") as fh:
+                    reader = csv.DictReader(fh)
+                    data = list(reader)
+
+        if isinstance(data, dict):
+            data = data.get("data", [])
+
+        if not isinstance(data, list):
+            raise ValueError("Expected list of examples from local file")
+
+        return data
+
+    def _load_hf_dataset(self, identifier: str) -> List[Dict[str, Any]]:
+        """Load the dataset from HuggingFace Hub."""
+
+        parts = identifier.split(":", maxsplit=1)
+        dataset_name = parts[0]
+        split = parts[1] if len(parts) == 2 else DEFAULT_DATASET_SPLIT
+
+        try:
+            from datasets import load_dataset  # type: ignore
+        except ImportError as exc:
+            raise ImportError(
+                "Please install the 'datasets' package to load LongBench-v2 from HuggingFace: pip install datasets"
+            ) from exc
+
+        dataset = load_dataset(dataset_name, split=split)
+        return [dict(row) for row in dataset]
+
+    def _normalize_example(self, example: Dict[str, Any]) -> Dict[str, Any]:
+        """Ensure each example exposes the expected keys."""
+
+        normalized = dict(example)
+
+        for letter in ["A", "B", "C", "D"]:
+            choice_key = f"choice_{letter}"
+            if letter not in normalized and choice_key in normalized:
+                normalized[letter] = normalized[choice_key]
+
+        if "category" not in normalized and "domain" in normalized:
+            normalized["category"] = normalized["domain"]
+
+        answer = normalized.get("answer")
+        if isinstance(answer, str):
+            normalized["answer"] = answer.strip().upper()
+        elif isinstance(answer, int) and 0 <= answer < 4:
+            normalized["answer"] = ["A", "B", "C", "D"][answer]
+
+        return normalized
+
+    def _check_context_length(
+        self,
+        formatted_question: str,
+        tokenizer: AutoTokenizer,
+        min_length: Optional[int],
+        max_length: Optional[int],
+    ) -> bool:
+        """Filter examples by context length measured in characters."""
+        input_ids = tokenizer.encode(formatted_question)
+        context_length = len(input_ids)
+
+        if min_length is not None and context_length < min_length:
+            return False
+        if max_length is not None and context_length > max_length:
+            return False
+
+        return True
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        """Run the evaluation."""
+
+        def fn(row: dict):
+            # Format the question using official template
+            formatted_question = format_longbench_v2_question(row)
+
+            if self.min_context_length or self.max_context_length:
+                if not self._check_context_length(
+                    formatted_question,
+                    self.tokenizer,
+                    self.min_context_length,
+                    self.max_context_length,
+                ):
+                    # Skip this example
+                    return None
+
+            prompt_messages = [
+                sampler._pack_message(content=formatted_question, role="user")
+            ]
+
+            # Get model response
+            response_text = sampler(prompt_messages)
+            if response_text is None:
+                response_text = ""
+
+            # Extract answer using official method
+            extracted_answer = extract_longbench_v2_answer(response_text)
+
+            # Get correct answer
+            correct_answer = row.get("answer", "")
+            if isinstance(correct_answer, str):
+                correct_answer = correct_answer.strip().upper()
+            elif isinstance(correct_answer, int) and 0 <= correct_answer < 4:
+                correct_answer = ["A", "B", "C", "D"][correct_answer]
+
+            # Calculate score
+            score = 1.0 if extracted_answer == correct_answer else 0.0
+
+            # Generate HTML report
+            html = common.jinja_env.from_string(HTML_JINJA).render(
+                prompt_messages=prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=correct_answer,
+                extracted_answer=extracted_answer,
+            )
+
+            # Build conversation
+            convo = prompt_messages + [dict(content=response_text, role="assistant")]
+
+            # Prepare metrics
+            metrics = {"chars": len(response_text)}
+
+            # Add category-specific metrics
+            category = row.get("category", row.get("domain", "unknown"))
+            if category in TASK_CATEGORIES:
+                metrics[category] = score
+
+            difficulty = row.get("difficulty")
+            if isinstance(difficulty, str) and difficulty:
+                metrics[f"difficulty_{difficulty.lower()}"] = score
+
+            return SingleEvalResult(
+                html=html,
+                score=score,
+                convo=convo,
+                metrics=metrics,
+            )
+
+        # Run evaluation with progress tracking
+        results = common.map_with_progress(fn, self.examples, self.num_threads)
+        return common.aggregate_results(results)

sglang/test/test_block_fp8.py

@@ -1,5 +1,4 @@
 import itertools
-import os
 import unittest
 
 import torch
@@ -577,7 +576,7 @@ class TestW8A8BlockFP8BatchedDeepGemm(CustomTestCase):
         if not torch.cuda.is_available():
             raise unittest.SkipTest("CUDA is not available")
         try:
-            import deep_gemm
+            import deep_gemm  # noqa: F401
         except ImportError:
             raise unittest.SkipTest("DeepGEMM is not available")
         torch.set_default_device("cuda")

sglang/test/test_block_fp8_deep_gemm_blackwell.py

@@ -1,5 +1,4 @@
 import itertools
-import os
 import unittest
 from typing import List, Tuple
 

sglang/test/test_cutlass_moe.py

@@ -1,5 +1,4 @@
 import argparse
-import time
 
 import torch
 import triton  # Added import
@@ -34,7 +33,7 @@ def get_model_config(tp_size: int):
         "topk": topk,
         "hidden_size": config.hidden_size,
         "shard_intermediate_size": shard_intermediate_size,
-        "dtype": config.torch_dtype,
+        "dtype": config.dtype,
         "block_shape": config.quantization_config["weight_block_size"],
     }
 

sglang/test/test_cutlass_w4a8_moe.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import Literal, Optional
+from typing import Optional
 
 import pytest
 import torch
@@ -120,7 +120,7 @@ def test_cutlass_w4a8_moe(M, N, K, E, tp_size, use_ep_moe, topk, group_size, dty
     )
     topk_weights, topk_ids, _ = topk_output
     expert_map = torch.arange(E, dtype=torch.int32, device=device)
-    expert_map[local_e:] = E
+    expert_map[local_e:] = -1
 
     output = cutlass_moe(
         a,
@@ -138,9 +138,7 @@ def test_cutlass_w4a8_moe(M, N, K, E, tp_size, use_ep_moe, topk, group_size, dty
         c_strides2,
         s_strides13,
         s_strides2,
-        0,
-        local_e - 1,
-        E,
+        local_e,
         a1_scale,
         a2_scale,
         expert_map,
@@ -178,7 +176,7 @@ def cutlass_moe(
     w1_scale: torch.Tensor,
     w2_scale: torch.Tensor,
     topk_weights: torch.Tensor,
-    topk_ids_: torch.Tensor,
+    topk_ids: torch.Tensor,
     a_strides1: torch.Tensor,
     b_strides1: torch.Tensor,
     c_strides1: torch.Tensor,
@@ -187,40 +185,32 @@ def cutlass_moe(
     c_strides2: torch.Tensor,
     s_strides13: torch.Tensor,
     s_strides2: torch.Tensor,
-    start_expert_id: int,
-    end_expert_id: int,
-    E: int,
+    num_local_experts: int,
     a1_scale: Optional[torch.Tensor] = None,
     a2_scale: Optional[torch.Tensor] = None,
     expert_map: Optional[torch.Tensor] = None,
     apply_router_weight_on_input: bool = False,
 ):
-    local_topk_ids = topk_ids_
-    local_topk_ids = torch.where(expert_map[topk_ids_] != E, expert_map[topk_ids_], E)
+    topk_ids = expert_map[topk_ids]
     device = a.device
 
-    local_num_experts = end_expert_id - start_expert_id + 1
     expert_offsets = torch.empty(
-        (local_num_experts + 1), dtype=torch.int32, device=device
+        (num_local_experts + 1), dtype=torch.int32, device=device
     )
     problem_sizes1 = torch.empty(
-        (local_num_experts, 3), dtype=torch.int32, device=device
+        (num_local_experts, 3), dtype=torch.int32, device=device
     )
     problem_sizes2 = torch.empty(
-        (local_num_experts, 3), dtype=torch.int32, device=device
+        (num_local_experts, 3), dtype=torch.int32, device=device
     )
     return cutlass_w4a8_moe(
-        start_expert_id,
-        end_expert_id,
-        E,
         a,
         w1_q,
         w2_q,
         w1_scale,
         w2_scale,
         topk_weights,
-        topk_ids_,
-        local_topk_ids,
+        topk_ids,
         a_strides1,
         b_strides1,
         c_strides1,