sglang 0.3.1.post3__py3-none-any.whl → 0.3.3__py3-none-any.whl

sglang/__init__.py

@@ -1,6 +1,7 @@
 # SGL API Components
 
 from sglang.api import (
+    Engine,
     Runtime,
     assistant,
     assistant_begin,
@@ -31,6 +32,7 @@ from sglang.lang.choices import (
 # SGLang DSL APIs
 __all__ = [
     "Runtime",
+    "Engine",
     "assistant",
     "assistant_begin",
     "assistant_end",

sglang/api.py

@@ -33,13 +33,23 @@ def function(
 
 
 def Runtime(*args, **kwargs):
-    # Avoid importing unnecessary dependency
     os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+    # Avoid importing unnecessary dependency
     from sglang.srt.server import Runtime
 
     return Runtime(*args, **kwargs)
 
 
+def Engine(*args, **kwargs):
+    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+    # Avoid importing unnecessary dependency
+    from sglang.srt.server import Engine
+
+    return Engine(*args, **kwargs)
+
+
 def set_default_backend(backend: BaseBackend):
     global_config.default_backend = backend
 
@@ -48,6 +58,10 @@ def flush_cache(backend: Optional[BaseBackend] = None):
     backend = backend or global_config.default_backend
     if backend is None:
         return False
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
     return backend.flush_cache()
 
 
@@ -55,12 +69,17 @@ def get_server_args(backend: Optional[BaseBackend] = None):
     backend = backend or global_config.default_backend
     if backend is None:
         return None
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
     return backend.get_server_args()
 
 
 def gen(
     name: Optional[str] = None,
     max_tokens: Optional[int] = None,
+    min_tokens: Optional[int] = None,
     stop: Optional[Union[str, List[str]]] = None,
     stop_token_ids: Optional[List[int]] = None,
     temperature: Optional[float] = None,
@@ -100,6 +119,7 @@ def gen(
     return SglGen(
         name,
         max_tokens,
+        min_tokens,
         stop,
         stop_token_ids,
         temperature,
@@ -139,6 +159,7 @@ def gen_int(
     return SglGen(
         name,
         max_tokens,
+        None,
         stop,
         stop_token_ids,
         temperature,
@@ -177,6 +198,7 @@ def gen_string(
     return SglGen(
         name,
         max_tokens,
+        None,
         stop,
         stop_token_ids,
         temperature,

sglang/bench_latency.py

@@ -47,6 +47,7 @@ I'm going to the park
 import argparse
 import dataclasses
 import itertools
+import json
 import logging
 import multiprocessing
 import os
@@ -62,10 +63,11 @@ import torch.distributed as dist
 from sglang.srt.configs.model_config import ModelConfig
 from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 from sglang.srt.model_executor.model_runner import ModelRunner
 from sglang.srt.sampling.sampling_params import SamplingParams
 from sglang.srt.server import _set_envs_and_config
-from sglang.srt.server_args import ServerArgs
+from sglang.srt.server_args import PortArgs, ServerArgs
 from sglang.srt.utils import (
     configure_logger,
     kill_child_process,
@@ -121,7 +123,7 @@ class BenchArgs:
         )
 
 
-def load_model(server_args, tp_rank):
+def load_model(server_args, port_args, tp_rank):
     suppress_other_loggers()
     rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None
 
@@ -129,6 +131,7 @@ def load_model(server_args, tp_rank):
         server_args.model_path,
         server_args.trust_remote_code,
         context_length=server_args.context_length,
+        model_override_args=json.loads(server_args.json_model_override_args),
     )
     model_runner = ModelRunner(
         model_config=model_config,
@@ -136,7 +139,7 @@ def load_model(server_args, tp_rank):
         gpu_id=tp_rank,
         tp_rank=tp_rank,
         tp_size=server_args.tp_size,
-        nccl_port=28888,
+        nccl_port=port_args.nccl_ports[0],
         server_args=server_args,
     )
     rank_print(f"max_total_num_tokens={model_runner.max_total_num_tokens}")
@@ -167,9 +170,13 @@ def prepare_inputs_for_correctness_test(bench_args, tokenizer):
         assert len(input_ids[i]) > bench_args.cut_len
 
         tmp_input_ids = input_ids[i][: bench_args.cut_len]
-        req = Req(rid=i, origin_input_text=prompts[i], origin_input_ids=tmp_input_ids)
+        req = Req(
+            rid=i,
+            origin_input_text=prompts[i],
+            origin_input_ids=tmp_input_ids,
+            sampling_params=sampling_params,
+        )
         req.prefix_indices = []
-        req.sampling_params = sampling_params
         req.fill_ids = req.origin_input_ids
         req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices)
         reqs.append(req)
@@ -199,9 +206,13 @@ def prepare_synthetic_inputs_for_latency_test(batch_size, input_len):
 
     reqs = []
     for i in range(len(input_ids)):
-        req = Req(rid=i, origin_input_text="", origin_input_ids=list(input_ids[i]))
+        req = Req(
+            rid=i,
+            origin_input_text="",
+            origin_input_ids=list(input_ids[i]),
+            sampling_params=sampling_params,
+        )
         req.prefix_indices = []
-        req.sampling_params = sampling_params
         req.fill_ids = req.origin_input_ids
         req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices)
         reqs.append(req)
@@ -217,28 +228,33 @@ def extend(reqs, model_runner):
         tree_cache=None,
     )
     batch.prepare_for_extend(model_runner.model_config.vocab_size)
-    logits_output = model_runner.forward(batch)
-    next_token_ids = model_runner.sample(logits_output, batch).tolist()
+    model_worker_batch = batch.get_model_worker_batch()
+    forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner)
+    logits_output = model_runner.forward(forward_batch)
+    next_token_ids = model_runner.sample(logits_output, forward_batch).tolist()
     return next_token_ids, logits_output.next_token_logits, batch
 
 
 def decode(input_token_ids, batch, model_runner):
     batch.prepare_for_decode(input_token_ids)
-    logits_output = model_runner.forward(batch)
-    next_token_ids = model_runner.sample(logits_output, batch).tolist()
+    model_worker_batch = batch.get_model_worker_batch()
+    forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner)
+    logits_output = model_runner.forward(forward_batch)
+    next_token_ids = model_runner.sample(logits_output, forward_batch).tolist()
     return next_token_ids, logits_output.next_token_logits
 
 
 @torch.inference_mode()
 def correctness_test(
     server_args,
+    port_args,
     bench_args,
     tp_rank,
 ):
     rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None
 
     # Load the model
-    model_runner, tokenizer = load_model(server_args, tp_rank)
+    model_runner, tokenizer = load_model(server_args, port_args, tp_rank)
 
     # Prepare inputs
     input_ids, reqs = prepare_inputs_for_correctness_test(bench_args, tokenizer)
@@ -260,7 +276,7 @@ def correctness_test(
 
     # Decode
     output_ids = [input_ids[i] + [next_token_ids[i]] for i in range(len(input_ids))]
-    for _ in range(bench_args.output_len[0]):
+    for _ in range(bench_args.output_len[0] - 1):
         next_token_ids, _ = decode(next_token_ids, batch, model_runner)
         for i in range(len(reqs)):
             output_ids[i].append(next_token_ids[i])
@@ -311,7 +327,7 @@ def latency_test_run_once(
 
     # Decode
     decode_latencies = []
-    for i in range(output_len):
+    for i in range(output_len - 1):
         torch.cuda.synchronize()
         tic = time.time()
         next_token_ids, _ = decode(next_token_ids, batch, model_runner)
@@ -324,13 +340,16 @@ def latency_test_run_once(
             rank_print(
                 f"Decode.  latency: {latency:6.5f} s, throughput: {throughput:9.2f} token/s"
             )
-    med_decode_latency = np.median(decode_latencies)
-    med_decode_throughput = batch_size / med_decode_latency
-    rank_print(
-        f"Decode.  median latency: {med_decode_latency:6.5f} s, median throughput: {med_decode_throughput:9.2f} token/s"
-    )
-    measurement_results["median_decode_latency"] = med_decode_latency
-    measurement_results["median_decode_throughput"] = med_decode_throughput
+
+    # record decode timing from 2nd output
+    if output_len > 1:
+        med_decode_latency = np.median(decode_latencies)
+        med_decode_throughput = batch_size / med_decode_latency
+        rank_print(
+            f"Decode.  median latency: {med_decode_latency:6.5f} s, median throughput: {med_decode_throughput:9.2f} token/s"
+        )
+        measurement_results["median_decode_latency"] = med_decode_latency
+        measurement_results["median_decode_throughput"] = med_decode_throughput
 
     throughput = (input_len + output_len) * batch_size / tot_latency
     rank_print(
@@ -343,15 +362,15 @@ def latency_test_run_once(
 
 def latency_test(
     server_args,
+    port_args,
     bench_args,
     tp_rank,
 ):
     configure_logger(server_args, prefix=f" TP{tp_rank}")
-    _set_envs_and_config(server_args)
     rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None
 
     # Load the model
-    model_runner, tokenizer = load_model(server_args, tp_rank)
+    model_runner, tokenizer = load_model(server_args, port_args, tp_rank)
 
     # Prepare inputs for warm up
     reqs = prepare_synthetic_inputs_for_latency_test(
@@ -367,7 +386,7 @@ def latency_test(
         reqs,
         bench_args.batch_size[0],
         bench_args.input_len[0],
-        4,  # shorter decoding to speed up the warmup
+        8,  # shorter decoding to speed up the warmup
     )
     rank_print("Benchmark ...")
 
@@ -453,6 +472,7 @@ def plot_latency_test(
 
 
 def main(server_args, bench_args):
+    _set_envs_and_config(server_args)
 
     if server_args.model_path:
         if bench_args.correctness_test:
@@ -468,8 +488,10 @@ def main(server_args, bench_args):
             "provide --result-filename for plotting the results"
         )
 
+    port_args = PortArgs.init_new(server_args)
+
     if server_args.tp_size == 1:
-        work_func(server_args, bench_args, 0)
+        work_func(server_args, port_args, bench_args, 0)
     else:
         workers = []
         for tp_rank in range(server_args.tp_size):
@@ -477,6 +499,7 @@ def main(server_args, bench_args):
                 target=work_func,
                 args=(
                     server_args,
+                    port_args,
                     bench_args,
                     tp_rank,
                 ),
@@ -491,18 +514,10 @@ def main(server_args, bench_args):
 
 
 if __name__ == "__main__":
-    multiprocessing.set_start_method("spawn", force=True)
-
     parser = argparse.ArgumentParser()
     ServerArgs.add_cli_args(parser)
     BenchArgs.add_cli_args(parser)
-    # For this script, model-path is not required
-    assert (
-        parser._actions[1].option_strings[0] == "--model-path"
-    ), "options changed, this code need to be updated"
-    parser._actions[1].required = False
     args = parser.parse_args()
-
     server_args = ServerArgs.from_cli_args(args)
     bench_args = BenchArgs.from_cli_args(args)
 

sglang/bench_server_latency.py

@@ -174,13 +174,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     ServerArgs.add_cli_args(parser)
     BenchArgs.add_cli_args(parser)
-    # For this script, model-path is not required
-    assert (
-        parser._actions[1].option_strings[0] == "--model-path"
-    ), "options changed, this code need to be updated"
-    parser._actions[1].required = False
     args = parser.parse_args()
-
     server_args = ServerArgs.from_cli_args(args)
     bench_args = BenchArgs.from_cli_args(args)
 

sglang/bench_serving.py

@@ -845,6 +845,7 @@ def run_benchmark(args_: argparse.Namespace):
     tokenizer = get_tokenizer(tokenizer_id)
 
     if args.dataset_name == "sharegpt":
+        assert args.random_input_len is None and args.random_output_len is None
         input_requests = sample_sharegpt_requests(
             dataset_path=args.dataset_path,
             num_requests=args.num_prompts,
@@ -852,6 +853,7 @@ def run_benchmark(args_: argparse.Namespace):
             fixed_output_len=args.sharegpt_output_len,
         )
     elif args.dataset_name == "random":
+        assert args.random_input_len is not None and args.random_output_len is not None
         input_requests = sample_random_requests(
             input_len=args.random_input_len,
             output_len=args.random_output_len,
@@ -964,13 +966,11 @@ if __name__ == "__main__":
     parser.add_argument(
         "--random-input-len",
         type=int,
-        default=1024,
         help="Number of input tokens per request, used only for random dataset.",
     )
     parser.add_argument(
         "--random-output-len",
         type=int,
-        default=128,
         help="Number of output tokens per request, used only for random dataset.",
     )
     parser.add_argument(

sglang/lang/backend/runtime_endpoint.py

@@ -235,6 +235,7 @@ class RuntimeEndpoint(BaseBackend):
         data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
         obj = self._generate_http_request(s, data)
         prompt_len = obj["meta_info"]["prompt_tokens"]
+        logprob_start_len = max(prompt_len - 2, 0)  # For token healing
 
         # Compute logprob
         data = {
@@ -244,7 +245,8 @@ class RuntimeEndpoint(BaseBackend):
                 "temperature": 0,
             },
             "return_logprob": True,
-            "logprob_start_len": max(prompt_len - 2, 0),  # for token healing
+            "return_text_in_logprobs": True,
+            "logprob_start_len": logprob_start_len,
         }
         obj = self._generate_http_request(s, data)
 
@@ -254,6 +256,17 @@ class RuntimeEndpoint(BaseBackend):
         input_token_logprobs = [r["meta_info"]["input_token_logprobs"] for r in obj]
         output_token_logprobs = [r["meta_info"]["output_token_logprobs"] for r in obj]
 
+        # Remove extra token if no token healing occurred
+        for i in range(len(input_token_logprobs)):
+            healed_token_str = input_token_logprobs[i][0][-1]
+            if s.text_.endswith(healed_token_str):
+                healed_token_logprob = input_token_logprobs[i][0][0]
+                normalized_prompt_logprobs[i] = (
+                    normalized_prompt_logprobs[i] * len(input_token_logprobs[i])
+                    - healed_token_logprob
+                ) / (len(input_token_logprobs[i]) - 1)
+                input_token_logprobs[i] = input_token_logprobs[i][1:]
+
         # Compute unconditional logprobs if required
         if choices_method.requires_unconditional_logprobs:
             input_ids = [[el[1] for el in subl] for subl in input_token_logprobs]

sglang/lang/interpreter.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import contextvars
+import copy
 import multiprocessing
 import queue
 import threading
@@ -652,9 +653,22 @@ class StreamExecutor:
                 self._init_var_event(e)
 
     def _resolve_sampling_params(self, sampling_params):
-        clone = None
+        """
+        Construct sampling param based on default + override values
+
+        The default values of sampling are populated in `default_sampling_para` via sgl.function.run(...sampling_args)
+        , and `sampling_params` contains the override values from sgl.gen().
+
+        Here we use default_sampling_para as the base and override the values if they exist in `sampling_params`.
+        It also extends the stop tokens based on the chat template.
+        """
+
+        # deepcopy is required because the dict has lists inside
+        clone = copy.deepcopy(self.default_sampling_para)
+
         for item in [
             "max_new_tokens",
+            "min_new_tokens",
             "stop",
             "stop_token_ids",
             "temperature",
@@ -674,20 +688,16 @@ class StreamExecutor:
         ]:
             value = getattr(sampling_params, item, None)
             if value is not None:
-                if clone is None:
-                    clone = self.default_sampling_para.clone()
                 setattr(clone, item, value)
 
         if self.chat_template.stop_str:
-            if not clone:
-                clone = self.default_sampling_para.clone()
             if clone.stop == ():
                 clone.stop = []
             elif isinstance(clone.stop, str):
                 clone.stop = [clone.stop]
             clone.stop += self.chat_template.stop_str
 
-        return clone or self.default_sampling_para
+        return clone
 
     def __del__(self):
         self.end()

sglang/lang/ir.py

@@ -17,6 +17,7 @@ REGEX_STR = r"\"[\w\d\s]*\""  # bugs with regex r"\".*\"" in interegular pkg
 @dataclasses.dataclass
 class SglSamplingParams:
     max_new_tokens: int = 128
+    min_new_tokens: int = 0
     stop: Union[str, List[str]] = ()
     stop_token_ids: Optional[List[int]] = ()
     temperature: float = 1.0
@@ -39,6 +40,7 @@ class SglSamplingParams:
     def clone(self):
         return SglSamplingParams(
             self.max_new_tokens,
+            self.min_new_tokens,
             self.stop,
             self.stop_token_ids,
             self.temperature,
@@ -113,6 +115,7 @@ class SglSamplingParams:
     def to_srt_kwargs(self):
         return {
             "max_new_tokens": self.max_new_tokens,
+            "min_new_tokens": self.min_new_tokens,
             "stop": self.stop,
             "stop_token_ids": self.stop_token_ids,
             "temperature": self.temperature,
@@ -150,8 +153,8 @@ class SglFunction:
         self,
         *args,
         max_new_tokens: int = 128,
-        stop: Union[str, List[str]] = [],
-        stop_token_ids: Optional[List[int]] = [],
+        stop: Optional[Union[str, List[str]]] = None,
+        stop_token_ids: Optional[List[int]] = None,
         temperature: float = 1.0,
         top_p: float = 1.0,
         top_k: int = -1,
@@ -169,6 +172,12 @@ class SglFunction:
     ):
         from sglang.lang.interpreter import run_program
 
+        # avoid using [] as the default arg: https://nikos7am.com/posts/mutable-default-arguments/
+        if stop is None:
+            stop = []
+        if stop_token_ids is None:
+            stop_token_ids = []
+
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
             stop=stop,
@@ -193,8 +202,8 @@ class SglFunction:
         batch_kwargs,
         *,
         max_new_tokens: int = 128,
-        stop: Union[str, List[str]] = (),
-        stop_token_ids: Optional[List[int]] = [],
+        stop: Optional[Union[str, List[str]]] = None,
+        stop_token_ids: Optional[List[int]] = None,
         temperature: float = 1.0,
         top_p: float = 1.0,
         top_k: int = -1,
@@ -212,6 +221,11 @@ class SglFunction:
     ):
         from sglang.lang.interpreter import run_program_batch
 
+        if stop is None:
+            stop = []
+        if stop_token_ids is None:
+            stop_token_ids = []
+
         assert isinstance(batch_kwargs, (list, tuple))
         if len(batch_kwargs) == 0:
             return []
@@ -413,6 +427,7 @@ class SglGen(SglExpr):
         self,
         name: Optional[str] = None,
         max_new_tokens: Optional[int] = None,
+        min_new_tokens: Optional[int] = None,
         stop: Optional[Union[str, List[str]]] = None,
         stop_token_ids: Optional[List[int]] = None,
         temperature: Optional[float] = None,
@@ -435,6 +450,7 @@ class SglGen(SglExpr):
         self.name = name
         self.sampling_params = SglSamplingParams(
             max_new_tokens=max_new_tokens,
+            min_new_tokens=min_new_tokens,
             stop=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,

sglang/srt/configs/model_config.py

@@ -49,13 +49,13 @@ class ModelConfig:
         if context_length is not None:
             self.context_len = context_length
         else:
-            self.context_len = get_context_length(self.hf_config)
+            self.context_len = get_context_length(self.hf_text_config)
 
-        # Unify the config keys for hf_config
+        # Unify the config keys for hf_text_config
         self.head_dim = getattr(
-            self.hf_config,
+            self.hf_text_config,
             "head_dim",
-            self.hf_config.hidden_size // self.hf_config.num_attention_heads,
+            self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads,
         )
 
         # FIXME: temporary special judge for deepseek v2 MLA architecture
@@ -72,8 +72,10 @@ class ModelConfig:
         else:
             self.attention_arch = AttentionArch.MHA
 
-        self.num_attention_heads = self.hf_config.num_attention_heads
-        self.num_key_value_heads = getattr(self.hf_config, "num_key_value_heads", None)
+        self.num_attention_heads = self.hf_text_config.num_attention_heads
+        self.num_key_value_heads = getattr(
+            self.hf_text_config, "num_key_value_heads", None
+        )
 
         # for Dbrx and MPT models
         if self.hf_config.model_type in ["dbrx", "mpt"]:
@@ -83,9 +85,9 @@ class ModelConfig:
 
         if self.num_key_value_heads is None:
             self.num_key_value_heads = self.num_attention_heads
-        self.hidden_size = self.hf_config.hidden_size
-        self.num_hidden_layers = self.hf_config.num_hidden_layers
-        self.vocab_size = self.hf_config.vocab_size
+        self.hidden_size = self.hf_text_config.hidden_size
+        self.num_hidden_layers = self.hf_text_config.num_hidden_layers
+        self.vocab_size = self.hf_text_config.vocab_size
 
     # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py#L289
     def get_total_num_kv_heads(self) -> int:

sglang/srt/constrained/fsm_cache.py

@@ -14,13 +14,17 @@ limitations under the License.
 """
 
 """Cache for the compressed finite state machine."""
+import logging
 
+from interegular import InvalidSyntax, parse_pattern
 from outlines.fsm.json_schema import build_regex_from_schema
 from transformers import AutoTokenizer
 
 from sglang.srt.constrained import RegexGuide, TransformerTokenizer
 from sglang.srt.constrained.base_tool_cache import BaseToolCache
 
+logger = logging.getLogger(__name__)
+
 
 class FSMCache(BaseToolCache):
     def __init__(
@@ -76,5 +80,9 @@ class FSMCache(BaseToolCache):
             regex = key_string
         else:
             raise ValueError(f"Invalid key_type: {key_type}")
-
+        try:
+            parse_pattern(regex)
+        except InvalidSyntax as e:
+            logger.warning(f"skip invalid regex guide: {regex=}, {e=}")
+            return None, regex
         return RegexGuide(regex, self.outlines_tokenizer), regex

sglang/srt/constrained/jump_forward.py

@@ -19,10 +19,12 @@ Reference: https://lmsys.org/blog/2024-02-05-compressed-fsm/
 """
 
 import dataclasses
+import logging
 from collections import defaultdict
 
 import interegular
 import outlines.caching
+from interegular import InvalidSyntax
 
 from sglang.srt.constrained import (
     FSMInfo,
@@ -34,6 +36,8 @@ from sglang.srt.constrained.base_tool_cache import BaseToolCache
 
 IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)"
 
+logger = logging.getLogger(__name__)
+
 
 @dataclasses.dataclass
 class JumpEdge:
@@ -47,7 +51,12 @@ class JumpForwardMap:
     def __init__(self, regex_string):
         @disk_cache()
         def _init_state_to_jump_forward(regex_string):
-            regex_pattern = interegular.parse_pattern(regex_string)
+            try:
+                regex_pattern = interegular.parse_pattern(regex_string)
+            except InvalidSyntax as e:
+                logger.warning(f"skip invalid regex: {regex_string}, {e=}")
+                self.state_to_jump_forward = None
+                return
 
             byte_fsm = make_byte_level_fsm(
                 regex_pattern.to_fsm().reduce(), keep_utf8=True
@@ -165,7 +174,11 @@ class JumpForwardCache(BaseToolCache):
         super().__init__()
 
     def init_value(self, regex):
-        return JumpForwardMap(regex)
+        forward_map = JumpForwardMap(regex)
+        if forward_map.state_to_jump_forward:
+            return forward_map
+        else:
+            return None
 
 
 def test_main(regex_string):

sglang/srt/hf_transformers_utils.py

@@ -129,6 +129,7 @@ def get_tokenizer(
             *args,
             trust_remote_code=trust_remote_code,
             tokenizer_revision=tokenizer_revision,
+            clean_up_tokenization_spaces=False,
             **kwargs,
         )
     except TypeError as e: