sglang 0.5.3rc0__py3-none-any.whl → 0.5.3rc2__py3-none-any.whl

sglang/test/test_utils.py

@@ -9,15 +9,18 @@ import os
 import random
 import re
 import subprocess
+import sys
 import threading
 import time
 import unittest
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
+from datetime import datetime
 from functools import partial
 from pathlib import Path
 from types import SimpleNamespace
-from typing import Awaitable, Callable, List, Optional, Tuple
+from typing import Any, Awaitable, Callable, List, Optional, Tuple
+from urllib.parse import quote
 
 import aiohttp
 import numpy as np
@@ -41,6 +44,7 @@ from sglang.utils import get_exception_traceback
 DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct"
 DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
 DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE = "meta-llama/Llama-3.2-1B"
+DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE = "Qwen/Qwen3-Reranker-0.6B"
 DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1"
 DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B"
 DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat"
@@ -75,11 +79,13 @@ DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE = "nytopop/Qwen3-30B-A3B.w8a8"
 # EAGLE
 DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf"
 DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B"
-DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B"
+DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 = "meta-llama/Llama-3.1-8B-Instruct"
+DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"
 DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST = (
     "meta-llama/Llama-3.1-8B-Instruct"
 )
 DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
+DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST = "Qwen/Qwen2.5-Coder-7B-Instruct"
 
 # Other use cases
 DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION = (
@@ -561,11 +567,30 @@ def popen_launch_server(
     if return_stdout_stderr:
         process = subprocess.Popen(
             command,
-            stdout=return_stdout_stderr[0],
-            stderr=return_stdout_stderr[1],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
             env=env,
             text=True,
+            bufsize=1,
         )
+
+        def _dump(src, sinks):
+            for line in iter(src.readline, ""):
+                for sink in sinks:
+                    sink.write(line)
+                    sink.flush()
+            src.close()
+
+        threading.Thread(
+            target=_dump,
+            args=(process.stdout, [return_stdout_stderr[0], sys.stdout]),
+            daemon=True,
+        ).start()
+        threading.Thread(
+            target=_dump,
+            args=(process.stderr, [return_stdout_stderr[1], sys.stderr]),
+            daemon=True,
+        ).start()
     else:
         process = subprocess.Popen(command, stdout=None, stderr=None, env=env)
 
@@ -869,6 +894,154 @@ def run_bench_serving(
     return res
 
 
+def run_score_benchmark(
+    model,
+    num_requests=100,
+    batch_size=5,
+    other_server_args=None,
+    need_warmup=False,
+    device="auto",
+):
+    """Score API benchmark function compatible with run_bench_serving pattern"""
+    if other_server_args is None:
+        other_server_args = []
+
+    if device == "auto":
+        device = auto_config_device()
+
+    # Launch the server (consistent with run_bench_serving)
+    base_url = DEFAULT_URL_FOR_TEST
+    process = popen_launch_server(
+        model,
+        base_url,
+        timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        other_args=other_server_args,
+    )
+
+    async def _run_benchmark():
+
+        # Load tokenizer for generating test data
+        from sglang.srt.utils.hf_transformers_utils import get_tokenizer
+
+        tokenizer = get_tokenizer(model)
+
+        # Score API configuration
+        score_query_tokens = 120
+        score_item_tokens = 180
+        score_label_token_ids = [9454, 2753]  # Yes/No token IDs
+        special_token = "<|im_start|>"
+
+        def generate_text_with_token_count(num_tokens):
+            """Generate text with precise token count using replicated token."""
+            text = special_token * num_tokens
+            actual_tokens = len(tokenizer.encode(text, add_special_tokens=False))
+            if actual_tokens != num_tokens:
+                text = special_token * (
+                    num_tokens
+                    // len(tokenizer.encode(special_token, add_special_tokens=False))
+                )
+            return text
+
+        if need_warmup:
+            warmup_data = {
+                "query": generate_text_with_token_count(score_query_tokens),
+                "items": [
+                    generate_text_with_token_count(score_item_tokens) for _ in range(3)
+                ],
+                "label_token_ids": score_label_token_ids,
+                "model": model,
+                "apply_softmax": True,
+            }
+
+            async with aiohttp.ClientSession() as session:
+                try:
+                    await session.post(
+                        f"{base_url}/v1/score",
+                        json=warmup_data,
+                        timeout=aiohttp.ClientTimeout(total=30),
+                    )
+                except:
+                    pass  # Ignore warmup errors
+
+        test_requests = []
+        for i in range(num_requests):
+            query = generate_text_with_token_count(score_query_tokens)
+            items = [
+                generate_text_with_token_count(score_item_tokens)
+                for _ in range(batch_size)
+            ]
+
+            score_data = {
+                "query": query,
+                "items": items,
+                "label_token_ids": score_label_token_ids,
+                "model": model,
+                "apply_softmax": True,
+            }
+            test_requests.append(score_data)
+
+        start_time = time.monotonic()
+        successful_requests = 0
+        total_latency = 0
+        latencies = []
+
+        async with aiohttp.ClientSession() as session:
+            for request_data in test_requests:
+                try:
+                    request_start = time.monotonic()
+                    async with session.post(
+                        f"{base_url}/v1/score",
+                        json=request_data,
+                        timeout=aiohttp.ClientTimeout(total=30),
+                    ) as response:
+                        if response.status == 200:
+                            response_data = await response.json()
+                            request_end = time.monotonic()
+
+                            if "scores" in response_data or "logprobs" in response_data:
+                                latency_ms = (request_end - request_start) * 1000
+                                latencies.append(latency_ms)
+                                total_latency += latency_ms
+                                successful_requests += 1
+                except Exception:
+                    continue
+
+        end_time = time.monotonic()
+        total_time = end_time - start_time
+
+        if successful_requests > 0:
+            throughput = successful_requests / total_time
+            avg_latency = total_latency / successful_requests
+            latencies.sort()
+            p95_latency = latencies[int(len(latencies) * 0.95)] if latencies else 0
+
+            return {
+                "completed": successful_requests,
+                "total_requests": num_requests,
+                "throughput": throughput,
+                "avg_latency_ms": avg_latency,
+                "p95_latency_ms": p95_latency,
+                "successful_requests": successful_requests,
+            }
+        else:
+            return {
+                "completed": 0,
+                "total_requests": num_requests,
+                "throughput": 0,
+                "avg_latency_ms": 0,
+                "p95_latency_ms": 0,
+                "successful_requests": 0,
+            }
+
+    try:
+        res = asyncio.run(_run_benchmark())
+    finally:
+        kill_process_tree(process.pid)
+
+    assert res["completed"] == res["successful_requests"]
+    return res
+
+
 def run_bench_serving_multi(
     model,
     base_url,
@@ -1390,6 +1563,41 @@ async def send_concurrent_generate_requests(
     return await asyncio.gather(*tasks)
 
 
+async def send_concurrent_generate_requests_with_custom_params(
+    base_url: str,
+    custom_params: List[dict[str, Any]],
+) -> Tuple[int, Any]:
+    """Sends generate request concurrently with custom parameters and returns status code and response json tuple. Max concurrency is num_requests."""
+
+    base_payload = {
+        "text": """
+                System: You are a helpful assistant.
+                User: What is the capital of France?
+                Assistant: The capital of France is
+                """,
+        "sampling_params": {
+            "temperature": 0,
+            "max_new_tokens": 50,
+        },
+    }
+
+    async def async_generate_with_priority(req):
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                f"{base_url}/generate",
+                json=req,
+            ) as response:
+                resp_json = await response.json()
+                return (response.status, resp_json)
+
+    tasks = []
+    for c in custom_params:
+        req = base_payload.copy()
+        req.update(c)
+        tasks.append(asyncio.create_task(async_generate_with_priority(req)))
+    return await asyncio.gather(*tasks)
+
+
 class CustomTestCase(unittest.TestCase):
     def _callTestMethod(self, method):
         max_retry = int(
@@ -1431,3 +1639,146 @@ def dump_bench_raw_result(
 def _ensure_remove_suffix(text: str, suffix: str):
     assert text.endswith(suffix)
     return text.removesuffix(suffix)
+
+
+class ModelDeploySetup:
+    def __init__(self, model_path: str, extra_args: List[str] = []):
+        self.model_path = model_path
+        if "--enable-multimodal" not in extra_args:
+            extra_args.append("--enable-multimodal")
+        if "--trust-remote-code" not in extra_args:
+            extra_args.append("--trust-remote-code")
+
+        self.extra_args = extra_args
+
+
+class ModelEvalMetrics:
+    def __init__(self, accuracy: float, eval_time: float):
+        self.accuracy = accuracy
+        self.eval_time = eval_time
+
+
+def extract_trace_link_from_bench_one_batch_server_output(output: str) -> str:
+    match = re.search(r"\[Profile\]\((.*?)\)", output)
+    if match:
+        trace_link = match.group(1)
+        return trace_link
+    return None
+
+
+def parse_models(model_string: str):
+    return [model.strip() for model in model_string.split(",") if model.strip()]
+
+
+def check_evaluation_test_results(
+    results,
+    test_name,
+    model_accuracy_thresholds,
+    model_latency_thresholds=None,
+    model_count=None,
+):
+    """
+    results: list of tuple of (model_path, accuracy, latency)
+    """
+    failed_models = []
+    if model_latency_thresholds is not None:
+        summary = " | model | status | score | score_threshold | latency | latency_threshold | \n"
+        summary += "| ----- | ------ | ----- | --------------- | ------- | ----------------- | \n"
+    else:
+        summary = " | model | status | score | score_threshold | \n"
+        summary += "| ----- | ------ | ----- | --------------- | \n"
+
+    results_dict = {res[0]: (res[1], res[2]) for res in results}
+
+    for model, accuracy_threshold in sorted(model_accuracy_thresholds.items()):
+        latency_threshold = (
+            model_latency_thresholds.get(model)
+            if model_latency_thresholds is not None
+            else 1e9
+        )
+
+        if model in results_dict:
+            accuracy, latency = results_dict[model]
+            is_success = accuracy >= accuracy_threshold and latency <= latency_threshold
+            status_emoji = "✅" if is_success else "❌"
+
+            if not is_success:
+                if accuracy < accuracy_threshold:
+                    failed_models.append(
+                        f"\nScore Check Failed: {model}\n"
+                        f"Model {model} score ({accuracy:.4f}) is below threshold ({accuracy_threshold:.4f})"
+                    )
+                if latency > latency_threshold:
+                    failed_models.append(
+                        f"\nLatency Check Failed: {model}\n"
+                        f"Model {model} latency ({latency:.4f}) is above threshold ({latency_threshold:.4f})"
+                    )
+
+            if model_latency_thresholds is not None:
+                line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | {latency} | {latency_threshold}\n"
+            else:
+                line = (
+                    f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold}\n"
+                )
+        else:
+            status_emoji = "❌"
+            failed_models.append(f"Model failed to launch or be evaluated: {model}")
+            if model_latency_thresholds is not None:
+                line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | N/A | {latency_threshold}\n"
+            else:
+                line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold}\n"
+
+        summary += line
+
+    print(summary)
+
+    if is_in_ci():
+        write_github_step_summary(f"## {test_name}\n{summary}")
+
+    if failed_models:
+        print("Some models failed the evaluation.")
+        raise AssertionError("\n".join(failed_models))
+
+
+# Bench knobs for bench_one_batch_server (override by env)
+def _parse_int_list_env(name: str, default_val: str):
+    val = os.environ.get(name, default_val)
+    return [int(x) for x in val.split(",") if x]
+
+
+# Return filenames
+def find_traces_under_path(path: str) -> List[str]:
+    results = []
+    for _, dirs, files in os.walk(path):
+        for file in files:
+            if file.endswith(".trace.json.gz"):
+                results.append(f"{file}")
+    return results
+
+
+def write_results_to_json(model, metrics, mode="a"):
+    result = {
+        "timestamp": datetime.now().isoformat(),
+        "model": model,
+        "metrics": metrics,
+        "score": metrics["score"],
+    }
+
+    if "latency" in metrics:
+        result["latency"] = (metrics.get("latency"),)
+
+    existing_results = []
+    if mode == "a" and os.path.exists("results.json"):
+        try:
+            with open("results.json", "r") as f:
+                existing_results = json.load(f)
+        except json.JSONDecodeError:
+            existing_results = []
+
+    if isinstance(existing_results, list):
+        existing_results.append(result)
+    else:
+        existing_results = [result]
+
+    with open("results.json", "w") as f:
+        json.dump(existing_results, f, indent=2)

sglang/utils.py

@@ -6,6 +6,7 @@ import logging
 import os
 import random
 import socket
+import ssl
 import subprocess
 import sys
 import time
@@ -155,7 +156,15 @@ def http_request(
             data = bytes(dumps(json), encoding="utf-8")
 
         try:
-            resp = urllib.request.urlopen(req, data=data, cafile=verify)
+            if sys.version_info >= (3, 13):
+                # Python 3.13+: Use SSL context (cafile removed)
+                if verify and isinstance(verify, str):
+                    context = ssl.create_default_context(cafile=verify)
+                else:
+                    context = ssl.create_default_context()
+                resp = urllib.request.urlopen(req, data=data, context=context)
+            else:
+                resp = urllib.request.urlopen(req, data=data, cafile=verify)
             return HttpResponse(resp)
         except urllib.error.HTTPError as e:
             return HttpResponse(e)

sglang/version.py

@@ -1 +1 @@
-__version__ = "0.5.3rc0"
+__version__ = "0.5.3rc2"

{sglang-0.5.3rc0.dist-info → sglang-0.5.3rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sglang
-Version: 0.5.3rc0
+Version: 0.5.3rc2
 Summary: SGLang is a fast serving framework for large language models and vision language models.
 License:                                  Apache License
                                    Version 2.0, January 2004
@@ -211,18 +211,17 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiohttp
-Requires-Dist: requests
-Requires-Dist: tqdm
-Requires-Dist: numpy
 Requires-Dist: IPython
-Requires-Dist: setproctitle
+Requires-Dist: aiohttp
+Requires-Dist: anthropic>=0.20.0
 Requires-Dist: blobfile==3.0.0
 Requires-Dist: build
 Requires-Dist: compressed-tensors
+Requires-Dist: cuda-python
 Requires-Dist: datasets
 Requires-Dist: einops
 Requires-Dist: fastapi
+Requires-Dist: flashinfer_python==0.4.0rc3
 Requires-Dist: hf_transfer
 Requires-Dist: huggingface_hub
 Requires-Dist: interegular
@@ -230,8 +229,10 @@ Requires-Dist: llguidance<0.8.0,>=0.7.11
 Requires-Dist: modelscope
 Requires-Dist: msgspec
 Requires-Dist: ninja
-Requires-Dist: openai==1.99.1
+Requires-Dist: numpy
+Requires-Dist: nvidia-cutlass-dsl==4.2.1
 Requires-Dist: openai-harmony==0.0.4
+Requires-Dist: openai==1.99.1
 Requires-Dist: orjson
 Requires-Dist: outlines==0.1.11
 Requires-Dist: packaging
@@ -239,32 +240,34 @@ Requires-Dist: partial_json_parser
 Requires-Dist: pillow
 Requires-Dist: prometheus-client>=0.20.0
 Requires-Dist: psutil
+Requires-Dist: py-spy
 Requires-Dist: pybase64
 Requires-Dist: pydantic
 Requires-Dist: pynvml
 Requires-Dist: python-multipart
 Requires-Dist: pyzmq>=25.1.2
+Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: sentencepiece
+Requires-Dist: setproctitle
+Requires-Dist: sgl-kernel==0.3.14.post1
 Requires-Dist: soundfile==0.13.1
-Requires-Dist: timm==1.0.16
 Requires-Dist: tiktoken
+Requires-Dist: timm==1.0.16
+Requires-Dist: torch==2.8.0
+Requires-Dist: torch_memory_saver==0.0.9rc2
 Requires-Dist: torchao==0.9.0
-Requires-Dist: transformers==4.56.1
+Requires-Dist: torchaudio==2.8.0
+Requires-Dist: torchvision
+Requires-Dist: tqdm
+Requires-Dist: transformers==4.57.0
 Requires-Dist: uvicorn
 Requires-Dist: uvloop
 Requires-Dist: xgrammar==0.1.24
-Requires-Dist: sgl-kernel==0.3.9.post2
-Requires-Dist: torch==2.8.0
-Requires-Dist: torchaudio==2.8.0
-Requires-Dist: torchvision
-Requires-Dist: cuda-python
-Requires-Dist: flashinfer_python==0.3.1
-Requires-Dist: openai==1.99.1
-Requires-Dist: tiktoken
-Requires-Dist: anthropic>=0.20.0
-Requires-Dist: torch_memory_saver==0.0.8
-Requires-Dist: decord
+Requires-Dist: grpcio==1.75.1
+Requires-Dist: grpcio-tools==1.75.1
+Provides-Extra: decord
+Requires-Dist: decord; extra == "decord"
 Provides-Extra: test
 Requires-Dist: accelerate; extra == "test"
 Requires-Dist: expecttest; extra == "test"
@@ -272,21 +275,27 @@ Requires-Dist: jsonlines; extra == "test"
 Requires-Dist: matplotlib; extra == "test"
 Requires-Dist: pandas; extra == "test"
 Requires-Dist: peft; extra == "test"
-Requires-Dist: sentence_transformers; extra == "test"
 Requires-Dist: pytest; extra == "test"
+Requires-Dist: sentence_transformers; extra == "test"
 Requires-Dist: tabulate; extra == "test"
 Provides-Extra: tracing
-Requires-Dist: opentelemetry-sdk; extra == "tracing"
 Requires-Dist: opentelemetry-api; extra == "tracing"
 Requires-Dist: opentelemetry-exporter-otlp; extra == "tracing"
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc; extra == "tracing"
+Requires-Dist: opentelemetry-sdk; extra == "tracing"
 Provides-Extra: all
 Requires-Dist: sglang[test]; extra == "all"
-Provides-Extra: blackwell
-Requires-Dist: nvidia-cutlass-dsl==4.1.0; extra == "blackwell"
-Requires-Dist: sglang[test]; extra == "blackwell"
+Requires-Dist: sglang[decord]; extra == "all"
+Provides-Extra: all-aarch64
+Requires-Dist: sglang[test]; extra == "all-aarch64"
 Provides-Extra: dev
 Requires-Dist: sglang[test]; extra == "dev"
+Requires-Dist: sglang[decord]; extra == "dev"
+Provides-Extra: blackwell
+Requires-Dist: sglang[test]; extra == "blackwell"
+Requires-Dist: sglang[decord]; extra == "blackwell"
+Provides-Extra: blackwell-aarch64
+Requires-Dist: sglang[test]; extra == "blackwell-aarch64"
 Dynamic: license-file
 
 <div align="center" id="sglangtop">