sglang 0.2.15__py3-none-any.whl → 0.3.1__py3-none-any.whl

sglang/bench_latency.py

@@ -57,10 +57,9 @@ import pandas as pd
 import torch
 import torch.distributed as dist
 
+from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
-from sglang.srt.model_config import ModelConfig
-from sglang.srt.model_executor.forward_batch_info import ForwardMode
 from sglang.srt.model_executor.model_runner import ModelRunner
 from sglang.srt.sampling.sampling_params import SamplingParams
 from sglang.srt.server_args import ServerArgs
@@ -165,6 +164,7 @@ def prepare_inputs_for_correctness_test(bench_args, tokenizer):
         req.prefix_indices = []
         req.sampling_params = sampling_params
         req.fill_ids = req.origin_input_ids
+        req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices)
         reqs.append(req)
 
     return input_ids, reqs
@@ -179,6 +179,7 @@ def prepare_extend_inputs_for_correctness_test(
         req.prefix_indices = model_runner.req_to_token_pool.req_to_token[
             i, : bench_args.cut_len
         ]
+        req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices)
     return reqs
 
 
@@ -195,6 +196,7 @@ def prepare_synthetic_inputs_for_latency_test(batch_size, input_len):
         req.prefix_indices = []
         req.sampling_params = sampling_params
         req.fill_ids = req.origin_input_ids
+        req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices)
         reqs.append(req)
 
     return reqs
@@ -208,15 +210,15 @@ def extend(reqs, model_runner):
         tree_cache=None,
     )
     batch.prepare_for_extend(model_runner.model_config.vocab_size)
-    sample_output, logits_output = model_runner.forward(batch, ForwardMode.EXTEND)
-    next_token_ids = sample_output.batch_next_token_ids.tolist()
+    logits_output = model_runner.forward(batch)
+    next_token_ids = model_runner.sample(logits_output, batch).tolist()
     return next_token_ids, logits_output.next_token_logits, batch
 
 
 def decode(input_token_ids, batch, model_runner):
     batch.prepare_for_decode(input_token_ids)
-    sample_output, logits_output = model_runner.forward(batch, ForwardMode.DECODE)
-    next_token_ids = sample_output.batch_next_token_ids.tolist()
+    logits_output = model_runner.forward(batch)
+    next_token_ids = model_runner.sample(logits_output, batch).tolist()
     return next_token_ids, logits_output.next_token_logits
 
 
@@ -480,6 +482,8 @@ def main(server_args, bench_args):
 
 
 if __name__ == "__main__":
+    multiprocessing.set_start_method("spawn", force=True)
+
     parser = argparse.ArgumentParser()
     ServerArgs.add_cli_args(parser)
     BenchArgs.add_cli_args(parser)

sglang/bench_serving.py

@@ -298,34 +298,41 @@ class BenchmarkMetrics:
     median_e2e_latency_ms: float
 
 
-default_sharegpt_path = "ShareGPT_V3_unfiltered_cleaned_split.json"
+SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
 
 
-def download_sharegpt_dataset(path):
-    url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
+def download_and_cache_file(url: str, filename: Optional[str] = None):
+    """Read and cache a file from a url."""
+    if filename is None:
+        filename = os.path.join("/tmp", url.split("/")[-1])
 
-    print(f"Downloading dataset from {url}")
-    try:
-        response = requests.get(url, stream=True)
-        response.raise_for_status()
+    # Check if the cache file already exists
+    if os.path.exists(filename):
+        return filename
+
+    print(f"Downloading from {url} to {filename}")
 
-        total_size = int(response.headers.get("content-length", 0))
-        block_size = 8192
+    # Stream the response to show the progress bar
+    response = requests.get(url, stream=True)
+    response.raise_for_status()  # Check for request errors
 
-        with open(path, "wb") as f, tqdm(
-            desc="Downloading",
-            total=total_size,
-            unit="iB",
-            unit_scale=True,
-            unit_divisor=1024,
-        ) as progress_bar:
-            for data in response.iter_content(block_size):
-                size = f.write(data)
-                progress_bar.update(size)
+    # Total size of the file in bytes
+    total_size = int(response.headers.get("content-length", 0))
+    chunk_size = 1024  # Download in chunks of 1KB
 
-        print(f"Dataset downloaded and saved to {path}")
-    except requests.RequestException as e:
-        raise Exception(f"Failed to download dataset: {e}")
+    # Use tqdm to display the progress bar
+    with open(filename, "wb") as f, tqdm(
+        desc=filename,
+        total=total_size,
+        unit="B",
+        unit_scale=True,
+        unit_divisor=1024,
+    ) as bar:
+        for chunk in response.iter_content(chunk_size=chunk_size):
+            f.write(chunk)
+            bar.update(len(chunk))
+
+    return filename
 
 
 def sample_sharegpt_requests(
@@ -338,13 +345,8 @@ def sample_sharegpt_requests(
         raise ValueError("output_len too small")
 
     # Download sharegpt if necessary
-    if not os.path.isfile(dataset_path) and not os.path.isfile(default_sharegpt_path):
-        download_sharegpt_dataset(default_sharegpt_path)
-        dataset_path = default_sharegpt_path
-    else:
-        dataset_path = (
-            dataset_path if os.path.isfile(dataset_path) else default_sharegpt_path
-        )
+    if not os.path.isfile(dataset_path):
+        dataset_path = download_and_cache_file(SHAREGPT_URL)
 
     # Load the dataset.
     with open(dataset_path) as f:
@@ -412,15 +414,8 @@ def sample_random_requests(
         # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens
 
         # Download sharegpt if necessary
-        if not os.path.isfile(dataset_path) and not os.path.isfile(
-            default_sharegpt_path
-        ):
-            download_sharegpt_dataset(default_sharegpt_path)
-            dataset_path = default_sharegpt_path
-        else:
-            dataset_path = (
-                dataset_path if os.path.isfile(dataset_path) else default_sharegpt_path
-            )
+        if not os.path.isfile(dataset_path):
+            dataset_path = download_and_cache_file(SHAREGPT_URL)
 
         # Load the dataset.
         with open(dataset_path) as f:

sglang/global_config.py

@@ -11,10 +11,6 @@ class GlobalConfig:
         # Default backend of the language
         self.default_backend = None
 
-        # Runtime constants: Request dependency time due to network delay
-        self.request_dependency_delay = 0.02
-        self.wait_for_new_request_delay = 0.0006
-
         # Runtime constants: New generation token ratio estimation
         self.init_new_token_ratio = 0.7
         self.base_min_new_token_ratio = 0.1

sglang/lang/backend/runtime_endpoint.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 
 from sglang.global_config import global_config
 from sglang.lang.backend.base_backend import BaseBackend
-from sglang.lang.chat_template import get_chat_template_by_model_path
+from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path
 from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod
 from sglang.lang.interpreter import StreamExecutor
 from sglang.lang.ir import (
@@ -23,6 +23,7 @@ class RuntimeEndpoint(BaseBackend):
         base_url: str,
         api_key: Optional[str] = None,
         verify: Optional[str] = None,
+        chat_template_name: Optional[str] = None,
     ):
         super().__init__()
         self.support_concate_and_append = True
@@ -39,9 +40,12 @@ class RuntimeEndpoint(BaseBackend):
         self._assert_success(res)
         self.model_info = res.json()
 
-        self.chat_template = get_chat_template_by_model_path(
-            self.model_info["model_path"]
-        )
+        if chat_template_name:
+            self.chat_template = get_chat_template(chat_template_name)
+        else:
+            self.chat_template = get_chat_template_by_model_path(
+                self.model_info["model_path"]
+            )
 
     def get_model_name(self):
         return self.model_info["model_path"]
@@ -235,9 +239,12 @@ class RuntimeEndpoint(BaseBackend):
         # Compute logprob
         data = {
             "text": [s.text_ + c for c in choices],
-            "sampling_params": {"max_new_tokens": 0},
+            "sampling_params": {
+                "max_new_tokens": 0,
+                "temperature": 0,
+            },
             "return_logprob": True,
-            "logprob_start_len": max(prompt_len - 2, 0),
+            "logprob_start_len": max(prompt_len - 2, 0),  # for token healing
         }
         obj = self._generate_http_request(s, data)
 

sglang/lang/interpreter.py

@@ -9,7 +9,7 @@ import uuid
 import warnings
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional
 
 import tqdm
 

sglang/launch_server.py

@@ -1,17 +1,14 @@
 """Launch the inference server."""
 
-import argparse
 import os
+import sys
 
 from sglang.srt.server import launch_server
-from sglang.srt.server_args import ServerArgs
+from sglang.srt.server_args import prepare_server_args
 from sglang.srt.utils import kill_child_process
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    ServerArgs.add_cli_args(parser)
-    args = parser.parse_args()
-    server_args = ServerArgs.from_cli_args(args)
+    server_args = prepare_server_args(sys.argv[1:])
 
     try:
         launch_server(server_args)

sglang/launch_server_llavavid.py

@@ -1,14 +1,12 @@
 """Launch the inference server for Llava-video model."""
 
-import argparse
+import json
+import sys
 
-from sglang.srt.server import ServerArgs, launch_server
+from sglang.srt.server import launch_server, prepare_server_args
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    ServerArgs.add_cli_args(parser)
-    args = parser.parse_args()
-    server_args = ServerArgs.from_cli_args(args)
+    server_args = prepare_server_args(sys.argv[1:])
 
     model_override_args = {}
     model_override_args["mm_spatial_pool_stride"] = 2
@@ -20,7 +18,8 @@ if __name__ == "__main__":
         model_override_args["max_sequence_length"] = 4096 * 2
         model_override_args["tokenizer_model_max_length"] = 4096 * 2
         model_override_args["model_max_length"] = 4096 * 2
-    if "34b" in args.model_path.lower():
+    if "34b" in server_args.model_path.lower():
         model_override_args["image_token_index"] = 64002
+    server_args.json_model_override_args = json.dumps(model_override_args)
 
-    launch_server(server_args, model_override_args, None)
+    launch_server(server_args)

sglang/srt/{model_config.py → configs/model_config.py}

@@ -64,6 +64,11 @@ class ModelConfig:
             self.attention_arch = AttentionArch.MLA
             self.kv_lora_rank = self.hf_config.kv_lora_rank
             self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
+        elif "MiniCPM3ForCausalLM" in self.hf_config.architectures:
+            self.head_dim = 128
+            self.attention_arch = AttentionArch.MLA
+            self.kv_lora_rank = self.hf_config.kv_lora_rank
+            self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim
         else:
             self.attention_arch = AttentionArch.MHA
 

sglang/srt/constrained/__init__.py

@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+"""For constrained decoding."""
+
 import json
 from typing import Dict, Optional, Union
 

sglang/srt/constrained/fsm_cache.py

@@ -16,6 +16,7 @@ limitations under the License.
 """Cache for the compressed finite state machine."""
 
 from outlines.fsm.json_schema import build_regex_from_schema
+from transformers import AutoTokenizer
 
 from sglang.srt.constrained import RegexGuide, TransformerTokenizer
 from sglang.srt.constrained.base_tool_cache import BaseToolCache
@@ -28,12 +29,9 @@ class FSMCache(BaseToolCache):
         tokenizer_args_dict,
         enable=True,
         skip_tokenizer_init=False,
-        json_schema_mode=False,
     ):
         super().__init__(enable=enable)
 
-        self.json_schema_mode = json_schema_mode
-
         if (
             skip_tokenizer_init
             or tokenizer_path.endswith(".json")
@@ -42,44 +40,37 @@ class FSMCache(BaseToolCache):
             # Do not support TiktokenTokenizer or SentencePieceTokenizer
             return
 
-        from importlib.metadata import version
+        tokenizer_args_dict.setdefault("padding_side", "left")
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, **tokenizer_args_dict)
+        try:
+            self.outlines_tokenizer = TransformerTokenizer(tokenizer)
+        except AttributeError:
+            # FIXME: tmp fix for chatglm2 & chatglm3 (pad_token_id=0)
+            origin_pad_token_id = tokenizer.pad_token_id
 
-        if version("outlines") >= "0.0.35":
-            from transformers import AutoTokenizer
+            def fset(self, value):
+                self._value = value
 
-            tokenizer_args_dict.setdefault("padding_side", "left")
-            tokenizer = AutoTokenizer.from_pretrained(
-                tokenizer_path, **tokenizer_args_dict
+            type(tokenizer).pad_token_id = property(
+                fget=type(tokenizer).pad_token_id.fget, fset=fset
             )
-            try:
-                self.outlines_tokenizer = TransformerTokenizer(tokenizer)
-            except AttributeError:
-                # FIXME: tmp fix for chatglm2 & chatglm3 (pad_token_id=0)
-                origin_pad_token_id = tokenizer.pad_token_id
-
-                def fset(self, value):
-                    self._value = value
-
-                type(tokenizer).pad_token_id = property(
-                    fget=type(tokenizer).pad_token_id.fget, fset=fset
-                )
-                self.outlines_tokenizer = TransformerTokenizer(tokenizer)
-                self.outlines_tokenizer.tokenizer.pad_token_id = origin_pad_token_id
-                self.outlines_tokenizer.pad_token_id = origin_pad_token_id
-                self.outlines_tokenizer.pad_token = (
-                    self.outlines_tokenizer.tokenizer.pad_token
-                )
-                self.outlines_tokenizer.vocabulary = (
-                    self.outlines_tokenizer.tokenizer.get_vocab()
-                )
-        else:
-            self.outlines_tokenizer = TransformerTokenizer(
-                tokenizer_path, **tokenizer_args_dict
+            self.outlines_tokenizer = TransformerTokenizer(tokenizer)
+            self.outlines_tokenizer.tokenizer.pad_token_id = origin_pad_token_id
+            self.outlines_tokenizer.pad_token_id = origin_pad_token_id
+            self.outlines_tokenizer.pad_token = (
+                self.outlines_tokenizer.tokenizer.pad_token
+            )
+            self.outlines_tokenizer.vocabulary = (
+                self.outlines_tokenizer.tokenizer.get_vocab()
             )
 
-    def init_value(self, value):
-        if self.json_schema_mode:
-            regex = build_regex_from_schema(value, whitespace_pattern=r"[\n\t ]*")
-            return RegexGuide(regex, self.outlines_tokenizer), regex
+    def init_value(self, key):
+        key_type, key_string = key
+        if key_type == "json":
+            regex = build_regex_from_schema(key_string, whitespace_pattern=r"[\n\t ]*")
+        elif key_type == "regex":
+            regex = key_string
         else:
-            return RegexGuide(value, self.outlines_tokenizer)
+            raise ValueError(f"Invalid key_type: {key_type}")
+
+        return RegexGuide(regex, self.outlines_tokenizer), regex

sglang/srt/constrained/jump_forward.py

@@ -23,7 +23,6 @@ from collections import defaultdict
 
 import interegular
 import outlines.caching
-from outlines.fsm.json_schema import build_regex_from_schema
 
 from sglang.srt.constrained import (
     FSMInfo,

sglang/srt/conversation.py

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-"""Conversation templates."""
+"""Conversation chat templates."""
 
 # Adapted from
 # https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
@@ -71,6 +71,7 @@ class Conversation:
     # Stop criteria (the default one is EOS token)
     stop_str: Union[str, List[str]] = None
     image_data: Optional[List[str]] = None
+    modalities: Optional[List[str]] = None
 
     def get_prompt(self) -> str:
         """Get the prompt for generation."""
@@ -379,6 +380,7 @@ def generate_chat_conv(
         sep2=conv.sep2,
         stop_str=conv.stop_str,
         image_data=[],
+        modalities=[],
     )
 
     if isinstance(request.messages, str):
@@ -408,6 +410,7 @@ def generate_chat_conv(
                 for content in message.content:
                     if content.type == "image_url":
                         num_image_url += 1
+                        conv.modalities.append(content.modalities)
                 if num_image_url > 1:
                     image_token = "<image>"
                 else:

sglang/srt/hf_transformers_utils.py

@@ -16,11 +16,9 @@ limitations under the License.
 """Utilities for Huggingface Transformers."""
 
 import contextlib
-import functools
-import json
 import os
 import warnings
-from typing import AbstractSet, Collection, Dict, List, Literal, Optional, Type, Union
+from typing import Dict, Optional, Type, Union
 
 from huggingface_hub import snapshot_download
 from transformers import (
@@ -92,7 +90,7 @@ def get_context_length(config):
     """Get the context length of a model from a huggingface model configs."""
     rope_scaling = getattr(config, "rope_scaling", None)
     if rope_scaling:
-        rope_scaling_factor = config.rope_scaling["factor"]
+        rope_scaling_factor = config.rope_scaling.get("factor", 1)
         if "original_max_position_embeddings" in rope_scaling:
             rope_scaling_factor = 1
         if config.rope_scaling.get("rope_type", None) == "llama3":