sglang 0.5.2rc2__py3-none-any.whl → 0.5.3.post1__py3-none-any.whl

sglang/test/run_eval.py

@@ -10,11 +10,46 @@ import time
 
 from sglang.test.simple_eval_common import (
     ChatCompletionSampler,
+    Eval,
     make_report,
     set_ulimit,
 )
 
 
+def get_thinking_kwargs(args):
+    thinking_mode = getattr(args, "thinking_mode", None)
+    if thinking_mode in THINKING_MODE_CHOICES:
+        if thinking_mode == "deepseek-v3":
+            thinking_param = "thinking"
+        else:
+            thinking_param = "enable_thinking"
+        return {
+            "chat_template_kwargs": {thinking_param: True},
+        }
+    return {}
+
+
+def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
+    # Get thinking kwargs based on user's choice
+    thinking_kwargs = get_thinking_kwargs(args)
+
+    sampler = ChatCompletionSampler(
+        model=args.model,
+        max_tokens=getattr(args, "max_tokens", 2048),
+        base_url=base_url,
+        temperature=getattr(args, "temperature", 0.0),
+        reasoning_effort=getattr(args, "reasoning_effort", None),
+        extra_body=thinking_kwargs,
+    )
+
+    # Run eval
+    tic = time.perf_counter()
+    result = eval_obj(sampler)
+    latency = time.perf_counter() - tic
+
+    return result, latency, sampler
+
+
 def run_eval(args):
     set_ulimit()
 
@@ -60,21 +95,55 @@ def run_eval(args):
         from sglang.test.simple_eval_humaneval import HumanEval
 
         eval_obj = HumanEval(args.num_examples, args.num_threads)
+    elif args.eval_name == "longbench_v2":
+        from sglang.test.simple_eval_longbench_v2 import LongBenchV2Eval
+
+        # Default to HuggingFace dataset, can be overridden with --dataset-path
+        data_source = args.dataset_path
+        categories = args.categories.split(",") if args.categories else None
+
+        eval_obj = LongBenchV2Eval(
+            data_source=data_source,
+            num_examples=args.num_examples,
+            num_threads=args.num_threads,
+            categories=categories,
+            max_context_length=getattr(args, "max_context_length", None),
+            min_context_length=getattr(args, "min_context_length", None),
+        )
+    elif args.eval_name == "mmmu":
+        # VLM MMMU evaluation with fixed 100 examples by default
+        from sglang.test.simple_eval_mmmu_vlm import MMMUVLMEval
+
+        eval_obj = MMMUVLMEval(args.num_examples, args.num_threads)
     else:
         raise ValueError(f"Invalid eval name: {args.eval_name}")
 
-    sampler = ChatCompletionSampler(
-        model=args.model,
-        max_tokens=getattr(args, "max_tokens", 2048),
-        base_url=base_url,
-        temperature=getattr(args, "temperature", 0.0),
-        reasoning_effort=getattr(args, "reasoning_effort", None),
-    )
+    if getattr(args, "repeat", 1) == 1:
+        result, latency, sampler = run_eval_once(args, base_url, eval_obj)
+    else:
+        from concurrent.futures import ThreadPoolExecutor
 
-    # Run eval
-    tic = time.perf_counter()
-    result = eval_obj(sampler)
-    latency = time.perf_counter() - tic
+        executor = ThreadPoolExecutor(max_workers=args.repeat)
+
+        futures = [
+            executor.submit(run_eval_once, args, base_url, eval_obj)
+            for _ in range(args.repeat)
+        ]
+
+        scores_repeat = []
+
+        for f in futures:
+            result, latency, sampler = f.result()
+            scores_repeat.append(result.score)
+
+        mean_score = sum(scores_repeat) / len(scores_repeat)
+        scores_repeat = [f"{s:.3f}" for s in scores_repeat]
+        print("=" * 20)
+        print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
+        print(f"Scores: {scores_repeat}")
+        print("=" * 20)
+
+        executor.shutdown()
 
     # Dump reports
     metrics = result.metrics | {"score": result.score}
@@ -94,9 +163,13 @@ def run_eval(args):
     print(f"Total latency: {latency:.3f} s")
     print(f"Score: {metrics['score']:.3f}")
 
+    if getattr(args, "return_latency", False):
+        return metrics, latency
     return metrics
 
 
+THINKING_MODE_CHOICES = ["deepseek-r1", "deepseek-v3", "qwen3"]
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -118,12 +191,47 @@ if __name__ == "__main__":
         type=str,
         help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
     )
+    parser.add_argument(
+        "--repeat", type=int, default=1, help="repeat the evaluation n times"
+    )
     parser.add_argument("--eval-name", type=str, default="mmlu")
     parser.add_argument("--num-examples", type=int)
     parser.add_argument("--num-threads", type=int, default=512)
     parser.add_argument("--max-tokens", type=int, default=2048)
     parser.add_argument("--temperature", type=float, default=0.0)
     parser.add_argument("--reasoning-effort", type=str)
+    parser.add_argument(
+        "--thinking-mode",
+        default=None,
+        type=str,
+        choices=THINKING_MODE_CHOICES,
+        help="Enable thinking mode in Deepseek R1, V3.1/3.2, or Qwen3",
+    )
+
+    # LongBench-v2 specific arguments
+    parser.add_argument(
+        "--dataset-path",
+        type=str,
+        default="THUDM/LongBench-v2",
+        help="Path to dataset file or HuggingFace dataset name for LongBench-v2",
+    )
+    parser.add_argument(
+        "--categories",
+        type=str,
+        default=None,
+        help="Comma-separated list of categories to evaluate for LongBench-v2",
+    )
+    parser.add_argument(
+        "--max-context-length",
+        type=int,
+        help="Maximum context length in characters for LongBench-v2",
+    )
+    parser.add_argument(
+        "--min-context-length",
+        type=int,
+        help="Minimum context length in characters for LongBench-v2",
+    )
+
     args = parser.parse_args()
 
     run_eval(args)

sglang/test/runners.py

@@ -30,8 +30,8 @@ from transformers import (
 )
 
 from sglang.srt.entrypoints.engine import Engine
-from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.utils import load_image
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 from sglang.test.test_utils import DEFAULT_PORT_FOR_SRT_TEST_RUNNER, calculate_rouge_l
 
 DEFAULT_PROMPTS = [
@@ -505,6 +505,7 @@ class SRTRunner:
         mem_fraction_static: float = 0.65,
         trust_remote_code: bool = False,
         speculative_draft_model_path: Optional[str] = None,
+        speculative_draft_model_revision: Optional[str] = None,
         speculative_algorithm: Optional[str] = None,
         speculative_num_steps: Optional[int] = None,
         speculative_eagle_topk: Optional[int] = None,
@@ -526,6 +527,9 @@ class SRTRunner:
         spec_kwargs = {}
         if speculative_draft_model_path:
             spec_kwargs["speculative_draft_model_path"] = speculative_draft_model_path
+            spec_kwargs["speculative_draft_model_revision"] = (
+                speculative_draft_model_revision
+            )
             spec_kwargs["speculative_algorithm"] = speculative_algorithm
             spec_kwargs["speculative_num_steps"] = speculative_num_steps
             spec_kwargs["speculative_eagle_topk"] = speculative_eagle_topk

sglang/test/simple_eval_common.py

@@ -93,6 +93,7 @@ class ChatCompletionSampler(SamplerBase):
         temperature: float = 0.0,
         reasoning_effort: Optional[str] = None,
         max_tokens: int = 2048,
+        extra_body: Optional[Dict[str, Any]] = None,
     ):
         self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
 
@@ -104,9 +105,10 @@ class ChatCompletionSampler(SamplerBase):
         self.temperature = temperature
         self.max_tokens = max_tokens
         self.reasoning_effort = reasoning_effort
+        self.extra_body = extra_body
         self.image_format = "url"
         print(
-            f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=}"
+            f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=}"
         )
 
     def _handle_image(
@@ -136,7 +138,7 @@ class ChatCompletionSampler(SamplerBase):
                 self._pack_message("system", self.system_message)
             ] + message_list
         trial = 0
-        while True:
+        while trial < 6:  # 126 seconds in total
             try:
                 response = self.client.chat.completions.create(
                     model=self.model,
@@ -144,6 +146,7 @@ class ChatCompletionSampler(SamplerBase):
                     temperature=self.temperature,
                     max_tokens=self.max_tokens,
                     reasoning_effort=self.reasoning_effort,
+                    extra_body=self.extra_body,
                 )
                 return response.choices[0].message.content
             # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are rerunning MMMU

sglang/test/simple_eval_longbench_v2.py

@@ -0,0 +1,332 @@
+# Adapted from https://github.com/openai/simple-evals/
+
+"""
+LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-Context Multitasks
+Yushi Bai, Shangqing Tu, Jiajie Zhang, Hao Peng, Xiaozhi Wang, Xin Lv, Shulin Cao, Jiazheng Xu, Lei Hou, Yuxiao Dong, Jie Tang, Juanzi Li
+https://arxiv.org/abs/2412.15204
+"""
+
+import csv
+import json
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from sglang.test import simple_eval_common as common
+from sglang.test.simple_eval_common import (
+    ANSWER_PATTERN_MULTICHOICE,
+    HTML_JINJA,
+    Eval,
+    EvalResult,
+    SamplerBase,
+    SingleEvalResult,
+)
+
+# LongBench-v2 task categories
+TASK_CATEGORIES = {
+    "single_document_qa",
+    "multi_document_qa",
+    "long_in_context_learning",
+    "long_dialogue_history",
+    "code_repo_understanding",
+    "long_structured_data",
+}
+
+DEFAULT_DATASET = "THUDM/LongBench-v2"
+DEFAULT_DATASET_SPLIT = "train"
+
+
+def format_longbench_v2_question(row: dict) -> str:
+    """Format a LongBench-v2 question using the official template."""
+    context = row.get("context", "")
+    question = row.get("question", "")
+
+    # Handle both standard format (A, B, C, D) and alternative format (choices list)
+    if "choices" in row:
+        choices = row["choices"]
+        choice_A = choices[0] if len(choices) > 0 else ""
+        choice_B = choices[1] if len(choices) > 1 else ""
+        choice_C = choices[2] if len(choices) > 2 else ""
+        choice_D = choices[3] if len(choices) > 3 else ""
+    else:
+        choice_A = row.get("A", row.get("choice_A", ""))
+        choice_B = row.get("B", row.get("choice_B", ""))
+        choice_C = row.get("C", row.get("choice_C", ""))
+        choice_D = row.get("D", row.get("choice_D", ""))
+
+    # Official LongBench-v2 template
+    prompt = f"""{context.strip()}
+
+What is the correct answer to this question: {question.strip()}
+Choices:
+(A) {choice_A.strip()}
+(B) {choice_B.strip()}
+(C) {choice_C.strip()}
+(D) {choice_D.strip()}
+
+The correct answer is"""
+
+    return prompt
+
+
+def extract_longbench_v2_answer(response: str) -> Optional[str]:
+    """Extract answer from model response using official LongBench-v2 method."""
+    response = response.replace("*", "")
+
+    # First try: "The correct answer is (A)"
+    match = re.search(r"The correct answer is \(([A-D])\)", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    # Second try: "The correct answer is A"
+    match = re.search(r"The correct answer is ([A-D])", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    # Fallback: Standard SGLang multichoice pattern
+    match = re.search(ANSWER_PATTERN_MULTICHOICE, response)
+    if match:
+        return match.group(1).upper()
+
+    # Generic fallback when model says "answer is A"
+    match = re.search(r"answer\s+is\s*\(?([A-D])\)?", response, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+
+    return None
+
+
+class LongBenchV2Eval(Eval):
+    """
+    Evaluation utility for LongBench-v2 dataset.
+
+    LongBench-v2 is designed to assess the ability of LLMs to handle long-context problems
+    requiring deep understanding and reasoning across real-world multitasks.
+    """
+
+    def __init__(
+        self,
+        data_source: str = DEFAULT_DATASET,
+        num_examples: Optional[int] = None,
+        num_threads: int = 1,
+        n_repeats: int = 1,
+        categories: Optional[List[str]] = None,
+        max_context_length: Optional[int] = None,
+        min_context_length: Optional[int] = None,
+    ):
+        """
+        Initialize LongBench-v2 evaluation.
+
+        Args:
+            data_source: HuggingFace dataset name, local file path (CSV/JSON)
+            num_examples: Number of examples to evaluate (None for all)
+            num_threads: Number of threads for parallel processing
+            n_repeats: Number of times to repeat evaluation for error bars
+            categories: List of task categories to include (None for all)
+            max_context_length: Maximum context length in characters
+            min_context_length: Minimum context length in characters
+        """
+        # Load dataset based on data source type
+        examples = self._load_dataset(data_source)
+
+        # Apply filtering
+        if categories:
+            examples = [ex for ex in examples if ex.get("category") in categories]
+
+        if min_context_length or max_context_length:
+            examples = self._filter_by_context_length(
+                examples, min_context_length, max_context_length
+            )
+
+        # Sample examples if specified
+        if num_examples:
+            assert n_repeats == 1, "n_repeats only supported when not sampling examples"
+            examples = examples[: min(num_examples, len(examples))]
+
+        # Repeat examples for multiple runs
+        examples = examples * n_repeats
+
+        if not examples:
+            raise ValueError(
+                "No examples available for LongBench-v2 evaluation after filtering"
+            )
+
+        self.examples = examples
+        self.n_repeats = n_repeats
+        self.num_threads = num_threads
+
+        print(f"Loaded {len(self.examples)} examples from LongBench-v2")
+        if categories:
+            print(f"Filtered to categories: {categories}")
+        if min_context_length or max_context_length:
+            print(
+                f"Context length filter: {min_context_length}-{max_context_length} characters"
+            )
+
+    def _load_dataset(self, data_source: str) -> List[Dict[str, Any]]:
+        """Load dataset from HuggingFace hub or local files."""
+
+        if not data_source:
+            data_source = DEFAULT_DATASET
+
+        if os.path.exists(data_source):
+            raw_examples = self._load_local_file(data_source)
+        else:
+            raw_examples = self._load_hf_dataset(data_source)
+
+        return [self._normalize_example(example) for example in raw_examples]
+
+    def _load_local_file(self, path: str) -> List[Dict[str, Any]]:
+        """Load examples from a local CSV/JSON/JSONL file."""
+
+        suffix = os.path.splitext(path)[1].lower()
+        if suffix in {".json", ".jsonl"}:
+            with open(path, "r", encoding="utf-8") as fh:
+                if suffix == ".jsonl":
+                    data = [json.loads(line) for line in fh if line.strip()]
+                else:
+                    data = json.load(fh)
+        elif suffix == ".csv":
+            with open(path, "r", encoding="utf-8") as fh:
+                reader = csv.DictReader(fh)
+                data = list(reader)
+        else:
+            # Try JSON, then CSV as fallback
+            try:
+                with open(path, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+            except json.JSONDecodeError:
+                with open(path, "r", encoding="utf-8") as fh:
+                    reader = csv.DictReader(fh)
+                    data = list(reader)
+
+        if isinstance(data, dict):
+            data = data.get("data", [])
+
+        if not isinstance(data, list):
+            raise ValueError("Expected list of examples from local file")
+
+        return data
+
+    def _load_hf_dataset(self, identifier: str) -> List[Dict[str, Any]]:
+        """Load the dataset from HuggingFace Hub."""
+
+        parts = identifier.split(":", maxsplit=1)
+        dataset_name = parts[0]
+        split = parts[1] if len(parts) == 2 else DEFAULT_DATASET_SPLIT
+
+        try:
+            from datasets import load_dataset  # type: ignore
+        except ImportError as exc:
+            raise ImportError(
+                "Please install the 'datasets' package to load LongBench-v2 from HuggingFace: pip install datasets"
+            ) from exc
+
+        dataset = load_dataset(dataset_name, split=split)
+        return [dict(row) for row in dataset]
+
+    def _normalize_example(self, example: Dict[str, Any]) -> Dict[str, Any]:
+        """Ensure each example exposes the expected keys."""
+
+        normalized = dict(example)
+
+        for letter in ["A", "B", "C", "D"]:
+            choice_key = f"choice_{letter}"
+            if letter not in normalized and choice_key in normalized:
+                normalized[letter] = normalized[choice_key]
+
+        if "category" not in normalized and "domain" in normalized:
+            normalized["category"] = normalized["domain"]
+
+        answer = normalized.get("answer")
+        if isinstance(answer, str):
+            normalized["answer"] = answer.strip().upper()
+        elif isinstance(answer, int) and 0 <= answer < 4:
+            normalized["answer"] = ["A", "B", "C", "D"][answer]
+
+        return normalized
+
+    def _filter_by_context_length(
+        self,
+        examples: List[Dict[str, Any]],
+        min_length: Optional[int],
+        max_length: Optional[int],
+    ) -> List[Dict[str, Any]]:
+        """Filter examples by context length measured in characters."""
+        filtered = []
+        for example in examples:
+            context = example.get("context", "")
+            context_length = len(context)
+
+            if min_length is not None and context_length < min_length:
+                continue
+            if max_length is not None and context_length > max_length:
+                continue
+
+            filtered.append(example)
+
+        return filtered
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        """Run the evaluation."""
+
+        def fn(row: dict):
+            # Format the question using official template
+            formatted_question = format_longbench_v2_question(row)
+
+            prompt_messages = [
+                sampler._pack_message(content=formatted_question, role="user")
+            ]
+
+            # Get model response
+            response_text = sampler(prompt_messages)
+            if response_text is None:
+                response_text = ""
+
+            # Extract answer using official method
+            extracted_answer = extract_longbench_v2_answer(response_text)
+
+            # Get correct answer
+            correct_answer = row.get("answer", "")
+            if isinstance(correct_answer, str):
+                correct_answer = correct_answer.strip().upper()
+            elif isinstance(correct_answer, int) and 0 <= correct_answer < 4:
+                correct_answer = ["A", "B", "C", "D"][correct_answer]
+
+            # Calculate score
+            score = 1.0 if extracted_answer == correct_answer else 0.0
+
+            # Generate HTML report
+            html = common.jinja_env.from_string(HTML_JINJA).render(
+                prompt_messages=prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=correct_answer,
+                extracted_answer=extracted_answer,
+            )
+
+            # Build conversation
+            convo = prompt_messages + [dict(content=response_text, role="assistant")]
+
+            # Prepare metrics
+            metrics = {"chars": len(response_text)}
+
+            # Add category-specific metrics
+            category = row.get("category", row.get("domain", "unknown"))
+            if category in TASK_CATEGORIES:
+                metrics[category] = score
+
+            difficulty = row.get("difficulty")
+            if isinstance(difficulty, str) and difficulty:
+                metrics[f"difficulty_{difficulty.lower()}"] = score
+
+            return SingleEvalResult(
+                html=html,
+                score=score,
+                convo=convo,
+                metrics=metrics,
+            )
+
+        # Run evaluation with progress tracking
+        results = common.map_with_progress(fn, self.examples, self.num_threads)
+        return common.aggregate_results(results)