sglang 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

sglang/srt/managers/controller/model_runner.py

@@ -4,11 +4,9 @@ import importlib
 import importlib.resources
 import logging
 import pkgutil
-from dataclasses import dataclass
 from functools import lru_cache
-from typing import List, Optional, Type
+from typing import Optional, Type
 
-import numpy as np
 import torch
 import torch.nn as nn
 from vllm.config import DeviceConfig, LoadConfig
@@ -17,7 +15,8 @@ from vllm.distributed import init_distributed_environment, initialize_model_para
 from vllm.model_executor.model_loader import get_model
 from vllm.model_executor.models import ModelRegistry
 
-from sglang.srt.managers.controller.infer_batch import Batch, ForwardMode
+from sglang.global_config import global_config
+from sglang.srt.managers.controller.infer_batch import Batch, ForwardMode, InputMetadata, global_server_args_dict
 from sglang.srt.memory_pool import ReqToTokenPool, TokenToKVPool
 from sglang.srt.server_args import ServerArgs
 from sglang.srt.utils import (
@@ -29,210 +28,6 @@ from sglang.srt.utils import (
 
 logger = logging.getLogger("srt.model_runner")
 
-# for server args in model endpoints
-global_server_args_dict = {}
-
-
-@dataclass
-class InputMetadata:
-    forward_mode: ForwardMode
-    batch_size: int
-    total_num_tokens: int
-    max_seq_len: int
-    req_pool_indices: torch.Tensor
-    start_loc: torch.Tensor
-    seq_lens: torch.Tensor
-    prefix_lens: torch.Tensor
-    positions: torch.Tensor
-    req_to_token_pool: ReqToTokenPool
-    token_to_kv_pool: TokenToKVPool
-
-    # for extend
-    extend_seq_lens: torch.Tensor = None
-    extend_start_loc: torch.Tensor = None
-    max_extend_len: int = 0
-
-    out_cache_loc: torch.Tensor = None
-    out_cache_cont_start: torch.Tensor = None
-    out_cache_cont_end: torch.Tensor = None
-
-    other_kv_index: torch.Tensor = None
-    return_logprob: bool = False
-    top_logprobs_nums: List[int] = None
-
-    # for flashinfer
-    qo_indptr: torch.Tensor = None
-    kv_indptr: torch.Tensor = None
-    kv_indices: torch.Tensor = None
-    kv_last_page_len: torch.Tensor = None
-    flashinfer_prefill_wrapper_ragged: "BatchPrefillWithRaggedKVCacheWrapper" = None
-    flashinfer_prefill_wrapper_paged: "BatchPrefillWithPagedKVCacheWrapper" = None
-    flashinfer_decode_wrapper: "BatchDecodeWithPagedKVCacheWrapper" = None
-
-    def init_flashinfer_args(self, num_qo_heads, num_kv_heads, head_dim):
-        if (
-            self.forward_mode == ForwardMode.PREFILL
-            or self.forward_mode == ForwardMode.EXTEND
-        ):
-            paged_kernel_lens = self.prefix_lens
-            self.no_prefix = torch.all(self.prefix_lens == 0)
-        else:
-            paged_kernel_lens = self.seq_lens
-
-        self.kv_indptr = torch.zeros(
-            (self.batch_size + 1,), dtype=torch.int32, device="cuda"
-        )
-        self.kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0)
-        self.kv_last_page_len = torch.ones(
-            (self.batch_size,), dtype=torch.int32, device="cuda"
-        )
-        req_pool_indices_cpu = self.req_pool_indices.cpu().numpy()
-        paged_kernel_lens_cpu = paged_kernel_lens.cpu().numpy()
-        self.kv_indices = torch.cat(
-            [
-                self.req_to_token_pool.req_to_token[
-                    req_pool_indices_cpu[i], : paged_kernel_lens_cpu[i]
-                ]
-                for i in range(self.batch_size)
-            ],
-            dim=0,
-        ).contiguous()
-
-        if (
-            self.forward_mode == ForwardMode.PREFILL
-            or self.forward_mode == ForwardMode.EXTEND
-        ):
-            # extend part
-            self.qo_indptr = torch.zeros(
-                (self.batch_size + 1,), dtype=torch.int32, device="cuda"
-            )
-            self.qo_indptr[1:] = torch.cumsum(self.extend_seq_lens, dim=0)
-
-            self.flashinfer_prefill_wrapper_ragged.end_forward()
-            self.flashinfer_prefill_wrapper_ragged.begin_forward(
-                self.qo_indptr,
-                self.qo_indptr.clone(),
-                num_qo_heads,
-                num_kv_heads,
-                head_dim,
-            )
-
-            # cached part
-            self.flashinfer_prefill_wrapper_paged.end_forward()
-            self.flashinfer_prefill_wrapper_paged.begin_forward(
-                self.qo_indptr,
-                self.kv_indptr,
-                self.kv_indices,
-                self.kv_last_page_len,
-                num_qo_heads,
-                num_kv_heads,
-                head_dim,
-                1
-            )
-        else:
-            self.flashinfer_decode_wrapper.end_forward()
-            self.flashinfer_decode_wrapper.begin_forward(
-                self.kv_indptr,
-                self.kv_indices,
-                self.kv_last_page_len,
-                num_qo_heads,
-                num_kv_heads,
-                head_dim,
-                1,
-                pos_encoding_mode="NONE",
-                data_type=self.token_to_kv_pool.kv_data[0].dtype
-            )
-
-    def init_extend_args(self):
-        self.extend_seq_lens = self.seq_lens - self.prefix_lens
-        self.extend_start_loc = torch.zeros_like(self.seq_lens)
-        self.extend_start_loc[1:] = torch.cumsum(self.extend_seq_lens[:-1], dim=0)
-        self.max_extend_len = int(torch.max(self.extend_seq_lens))
-
-    @classmethod
-    def create(
-        cls,
-        model_runner,
-        tp_size,
-        forward_mode,
-        req_pool_indices,
-        seq_lens,
-        prefix_lens,
-        position_ids_offsets,
-        out_cache_loc,
-        out_cache_cont_start=None,
-        out_cache_cont_end=None,
-        top_logprobs_nums=None,
-        return_logprob=False,
-        flashinfer_prefill_wrapper_ragged=None,
-        flashinfer_prefill_wrapper_paged=None,
-        flashinfer_decode_wrapper=None,
-    ):
-        batch_size = len(req_pool_indices)
-        start_loc = torch.zeros((batch_size,), dtype=torch.int32, device="cuda")
-        start_loc[1:] = torch.cumsum(seq_lens[:-1], dim=0)
-        total_num_tokens = int(torch.sum(seq_lens))
-        max_seq_len = int(torch.max(seq_lens))
-
-        if forward_mode == ForwardMode.DECODE:
-            positions = ((seq_lens - 1) + position_ids_offsets).to(torch.int64)
-            other_kv_index = model_runner.req_to_token_pool.req_to_token[
-                req_pool_indices[0], seq_lens[0] - 1
-            ].item()
-        else:
-            seq_lens_cpu = seq_lens.cpu().numpy()
-            prefix_lens_cpu = prefix_lens.cpu().numpy()
-            position_ids_offsets_cpu = position_ids_offsets.cpu().numpy()
-            positions = torch.tensor(
-                np.concatenate(
-                    [
-                        np.arange(
-                            prefix_lens_cpu[i] + position_ids_offsets_cpu[i],
-                            seq_lens_cpu[i] + position_ids_offsets_cpu[i],
-                        )
-                        for i in range(batch_size)
-                    ],
-                    axis=0,
-                ),
-                device="cuda",
-            )
-            other_kv_index = None
-
-        ret = cls(
-            forward_mode=forward_mode,
-            batch_size=batch_size,
-            total_num_tokens=total_num_tokens,
-            max_seq_len=max_seq_len,
-            req_pool_indices=req_pool_indices,
-            start_loc=start_loc,
-            seq_lens=seq_lens,
-            prefix_lens=prefix_lens,
-            positions=positions,
-            req_to_token_pool=model_runner.req_to_token_pool,
-            token_to_kv_pool=model_runner.token_to_kv_pool,
-            out_cache_loc=out_cache_loc,
-            out_cache_cont_start=out_cache_cont_start,
-            out_cache_cont_end=out_cache_cont_end,
-            other_kv_index=other_kv_index,
-            return_logprob=return_logprob,
-            top_logprobs_nums=top_logprobs_nums,
-            flashinfer_prefill_wrapper_ragged=flashinfer_prefill_wrapper_ragged,
-            flashinfer_prefill_wrapper_paged=flashinfer_prefill_wrapper_paged,
-            flashinfer_decode_wrapper=flashinfer_decode_wrapper,
-        )
-
-        if forward_mode == ForwardMode.EXTEND:
-            ret.init_extend_args()
-
-        if not global_server_args_dict.get("disable_flashinfer", False):
-            ret.init_flashinfer_args(
-                model_runner.model_config.num_attention_heads // tp_size,
-                model_runner.model_config.get_num_kv_heads(tp_size),
-                model_runner.model_config.head_dim
-            )
-
-        return ret
-
 
 class ModelRunner:
     def __init__(
@@ -245,6 +40,7 @@ class ModelRunner:
         nccl_port: int,
         server_args: ServerArgs,
     ):
+        # Parse args
         self.model_config = model_config
         self.mem_fraction_static = mem_fraction_static
         self.gpu_id = gpu_id
@@ -256,10 +52,12 @@ class ModelRunner:
         monkey_patch_vllm_dummy_weight_loader()
 
         # Init torch distributed
-        logger.info(f"[gpu_id={self.gpu_id}] Set cuda device.")
         torch.cuda.set_device(self.gpu_id)
         logger.info(f"[gpu_id={self.gpu_id}] Init nccl begin.")
-        monkey_patch_vllm_p2p_access_check(self.gpu_id)
+
+        if not server_args.enable_p2p_check:
+            monkey_patch_vllm_p2p_access_check(self.gpu_id)
+
         if server_args.nccl_init_addr:
             nccl_init_method = f"tcp://{server_args.nccl_init_addr}"
         else:
@@ -269,7 +67,7 @@ class ModelRunner:
             world_size=self.tp_size,
             rank=self.tp_rank,
             local_rank=self.gpu_id,
-            distributed_init_method=nccl_init_method
+            distributed_init_method=nccl_init_method,
         )
         initialize_model_parallel(tensor_model_parallel_size=self.tp_size)
         total_gpu_memory = get_available_gpu_memory(
@@ -284,11 +82,8 @@ class ModelRunner:
                 )
 
         # Set some global args
-        global global_server_args_dict
-        global_server_args_dict = {
-            "disable_flashinfer": server_args.disable_flashinfer,
-            "attention_reduce_in_fp32": server_args.attention_reduce_in_fp32,
-        }
+        global_server_args_dict["disable_flashinfer"] = server_args.disable_flashinfer
+        global_server_args_dict["attention_reduce_in_fp32"] = server_args.attention_reduce_in_fp32
 
         # Load the model and create memory pool
         self.load_model()
@@ -296,6 +91,9 @@ class ModelRunner:
         self.init_cublas()
         self.init_flash_infer()
 
+        # Capture cuda graphs
+        self.init_cuda_graphs()
+
     def load_model(self):
         logger.info(
             f"[gpu_id={self.gpu_id}] Load weight begin. "
@@ -323,7 +121,7 @@ class ModelRunner:
             device_config=device_config,
             load_config=load_config,
             lora_config=None,
-            vision_language_config=None,
+            multimodal_config=None,
             parallel_config=None,
             scheduler_config=None,
             cache_config=None,
@@ -341,7 +139,13 @@ class ModelRunner:
         )
         head_dim = self.model_config.head_dim
         head_num = self.model_config.get_num_kv_heads(self.tp_size)
-        cell_size = head_num * head_dim * self.model_config.num_hidden_layers * 2 * torch._utils._element_size(self.dtype)
+        cell_size = (
+            head_num
+            * head_dim
+            * self.model_config.num_hidden_layers
+            * 2
+            * torch._utils._element_size(self.dtype)
+        )
         rest_memory = available_gpu_memory - total_gpu_memory * (
             1 - self.mem_fraction_static
         )
@@ -382,64 +186,60 @@ class ModelRunner:
         return c
 
     def init_flash_infer(self):
-        if not global_server_args_dict.get("disable_flashinfer", False):
-            from flashinfer import (
-                BatchPrefillWithRaggedKVCacheWrapper,
-                BatchPrefillWithPagedKVCacheWrapper,
-                BatchDecodeWithPagedKVCacheWrapper,
-            )
-            from flashinfer.decode import _grouped_size_compiled_for_decode_kernels
+        if self.server_args.disable_flashinfer:
+            self.flashinfer_prefill_wrapper_ragged = None
+            self.flashinfer_prefill_wrapper_paged = None
+            self.flashinfer_decode_wrapper = None
+            return
 
-            if not _grouped_size_compiled_for_decode_kernels(
-                self.model_config.num_attention_heads // self.tp_size,
-                self.model_config.get_num_kv_heads(self.tp_size)):
-                use_tensor_cores = True
-            else:
-                use_tensor_cores = False
+        from flashinfer import (
+            BatchDecodeWithPagedKVCacheWrapper,
+            BatchPrefillWithPagedKVCacheWrapper,
+            BatchPrefillWithRaggedKVCacheWrapper,
+        )
+        from flashinfer.decode import _grouped_size_compiled_for_decode_kernels
 
-            workspace_buffers = torch.empty(
-                3, 96 * 1024 * 1024, dtype=torch.uint8, device="cuda"
-            )
-            self.flashinfer_prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper(
-                workspace_buffers[0], "NHD"
-            )
-            self.flashinfer_prefill_wrapper_paged = BatchPrefillWithPagedKVCacheWrapper(
-                workspace_buffers[1], "NHD"
-            )
-            self.flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
-                workspace_buffers[2], "NHD", use_tensor_cores=use_tensor_cores
-            )
+        if not _grouped_size_compiled_for_decode_kernels(
+            self.model_config.num_attention_heads // self.tp_size,
+            self.model_config.get_num_kv_heads(self.tp_size),
+        ):
+            use_tensor_cores = True
         else:
-            self.flashinfer_prefill_wrapper_ragged = self.flashinfer_prefill_wrapper_paged = None
-            self.flashinfer_decode_wrapper = None
+            use_tensor_cores = False
 
-    @torch.inference_mode()
-    def forward_prefill(self, batch: Batch):
-        input_metadata = InputMetadata.create(
-            self,
-            forward_mode=ForwardMode.PREFILL,
-            tp_size=self.tp_size,
-            req_pool_indices=batch.req_pool_indices,
-            seq_lens=batch.seq_lens,
-            prefix_lens=batch.prefix_lens,
-            position_ids_offsets=batch.position_ids_offsets,
-            out_cache_loc=batch.out_cache_loc,
-            top_logprobs_nums=batch.top_logprobs_nums,
-            return_logprob=batch.return_logprob,
-            flashinfer_prefill_wrapper_ragged=self.flashinfer_prefill_wrapper_ragged,
-            flashinfer_prefill_wrapper_paged=self.flashinfer_prefill_wrapper_paged,
-            flashinfer_decode_wrapper=self.flashinfer_decode_wrapper,
+        self.flashinfer_workspace_buffers = torch.empty(
+            2, global_config.flashinfer_workspace_size, dtype=torch.uint8, device="cuda"
         )
-        return self.model.forward(
-            batch.input_ids, input_metadata.positions, input_metadata
+        self.flashinfer_prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper(
+            self.flashinfer_workspace_buffers[0], "NHD"
+        )
+        self.flashinfer_prefill_wrapper_paged = BatchPrefillWithPagedKVCacheWrapper(
+            self.flashinfer_workspace_buffers[1], "NHD"
         )
+        self.flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
+            self.flashinfer_workspace_buffers[0], "NHD", use_tensor_cores=use_tensor_cores
+        )
+
+    def init_cuda_graphs(self):
+        from sglang.srt.managers.controller.cuda_graph_runner import CudaGraphRunner
+
+        if self.server_args.disable_cuda_graph or self.server_args.disable_flashinfer:
+            self.cuda_graph_runner = None
+            return
+
+        logger.info(f"[gpu_id={self.gpu_id}] Capture cuda graph begin.")
+        batch_size_list = [1, 2, 4] + [i * 8 for i in range(1, 16)]
+        self.cuda_graph_runner = CudaGraphRunner(self, max_batch_size_to_capture=max(batch_size_list))
+        self.cuda_graph_runner.capture(batch_size_list)
 
     @torch.inference_mode()
-    def forward_extend(self, batch: Batch):
+    def forward_decode(self, batch: Batch):
+        if self.cuda_graph_runner and self.cuda_graph_runner.can_run(len(batch.reqs)):
+            return self.cuda_graph_runner.replay(batch)
+
         input_metadata = InputMetadata.create(
             self,
-            forward_mode=ForwardMode.EXTEND,
-            tp_size=self.tp_size,
+            forward_mode=ForwardMode.DECODE,
             req_pool_indices=batch.req_pool_indices,
             seq_lens=batch.seq_lens,
             prefix_lens=batch.prefix_lens,
@@ -447,32 +247,23 @@ class ModelRunner:
             out_cache_loc=batch.out_cache_loc,
             top_logprobs_nums=batch.top_logprobs_nums,
             return_logprob=batch.return_logprob,
-            flashinfer_prefill_wrapper_ragged=self.flashinfer_prefill_wrapper_ragged,
-            flashinfer_prefill_wrapper_paged=self.flashinfer_prefill_wrapper_paged,
-            flashinfer_decode_wrapper=self.flashinfer_decode_wrapper,
         )
         return self.model.forward(
             batch.input_ids, input_metadata.positions, input_metadata
         )
 
     @torch.inference_mode()
-    def forward_decode(self, batch: Batch):
+    def forward_extend(self, batch: Batch):
         input_metadata = InputMetadata.create(
             self,
-            forward_mode=ForwardMode.DECODE,
-            tp_size=self.tp_size,
+            forward_mode=ForwardMode.EXTEND,
             req_pool_indices=batch.req_pool_indices,
             seq_lens=batch.seq_lens,
             prefix_lens=batch.prefix_lens,
             position_ids_offsets=batch.position_ids_offsets,
             out_cache_loc=batch.out_cache_loc,
-            out_cache_cont_start=batch.out_cache_cont_start,
-            out_cache_cont_end=batch.out_cache_cont_end,
             top_logprobs_nums=batch.top_logprobs_nums,
             return_logprob=batch.return_logprob,
-            flashinfer_prefill_wrapper_ragged=self.flashinfer_prefill_wrapper_ragged,
-            flashinfer_prefill_wrapper_paged=self.flashinfer_prefill_wrapper_paged,
-            flashinfer_decode_wrapper=self.flashinfer_decode_wrapper,
         )
         return self.model.forward(
             batch.input_ids, input_metadata.positions, input_metadata
@@ -483,17 +274,13 @@ class ModelRunner:
         input_metadata = InputMetadata.create(
             self,
             forward_mode=ForwardMode.EXTEND,
-            tp_size=self.tp_size,
             req_pool_indices=batch.req_pool_indices,
             seq_lens=batch.seq_lens,
             prefix_lens=batch.prefix_lens,
             position_ids_offsets=batch.position_ids_offsets,
             out_cache_loc=batch.out_cache_loc,
-            top_logprobs_nums=batch.top_logprobs_nums,
             return_logprob=batch.return_logprob,
-            flashinfer_prefill_wrapper_ragged=self.flashinfer_prefill_wrapper_ragged,
-            flashinfer_prefill_wrapper_paged=self.flashinfer_prefill_wrapper_paged,
-            flashinfer_decode_wrapper=self.flashinfer_decode_wrapper,
+            top_logprobs_nums=batch.top_logprobs_nums,
         )
         return self.model.forward(
             batch.input_ids,
@@ -511,8 +298,6 @@ class ModelRunner:
             return self.forward_decode(batch)
         elif forward_mode == ForwardMode.EXTEND:
             return self.forward_extend(batch)
-        elif forward_mode == ForwardMode.PREFILL:
-            return self.forward_prefill(batch)
         else:
             raise ValueError(f"Invaid forward mode: {forward_mode}")
 

sglang/srt/managers/controller/tp_worker.py

@@ -34,11 +34,11 @@ from sglang.srt.managers.io_struct import (
 from sglang.srt.model_config import ModelConfig
 from sglang.srt.server_args import ModelPortArgs, ServerArgs
 from sglang.srt.utils import (
+    connect_rpyc_service,
     get_int_token_logit_bias,
     is_multimodal_model,
     set_random_seed,
     start_rpyc_service_process,
-    connect_rpyc_service,
     suppress_other_loggers,
 )
 from sglang.utils import get_exception_traceback
@@ -98,7 +98,7 @@ class ModelTpServer:
             )
         self.max_total_num_tokens = self.model_runner.max_total_num_tokens
         self.max_prefill_tokens = (
-            4096
+            8192
             if server_args.max_prefill_tokens is None
             else server_args.max_prefill_tokens
         )
@@ -314,11 +314,9 @@ class ModelTpServer:
         self.forward_queue.append(req)
 
     def get_new_fill_batch(self) -> Optional[Batch]:
-        if (
-            self.running_batch is not None
-            and len(self.running_batch.reqs) > self.max_running_requests
-        ):
-            return None
+        running_bs = len(self.running_batch.reqs) if self.running_batch is not None else 0
+        if running_bs >= self.max_running_requests:
+            return
 
         # Compute matched prefix length
         for req in self.forward_queue:
@@ -368,9 +366,11 @@ class ModelTpServer:
             if (
                 req.extend_input_len + req.max_new_tokens() + new_batch_total_tokens
                 < available_size
-                and (req.extend_input_len + new_batch_input_tokens
-                <= self.max_prefill_tokens
-                or len(can_run_list) == 0)
+                and (
+                    req.extend_input_len + new_batch_input_tokens
+                    <= self.max_prefill_tokens
+                    or len(can_run_list) == 0
+                )
             ):
                 delta = self.tree_cache.inc_lock_ref(req.last_node)
                 available_size += delta
@@ -392,6 +392,10 @@ class ModelTpServer:
                     new_batch_input_tokens += req.extend_input_len
             else:
                 break
+
+            if running_bs + len(can_run_list) >= self.max_running_requests:
+                break
+
         if len(can_run_list) == 0:
             return None
 
@@ -452,7 +456,9 @@ class ModelTpServer:
                     next_token_ids,
                 ].tolist()
                 output.prefill_token_logprobs = output.prefill_token_logprobs.tolist()
-                output.normalized_prompt_logprobs = output.normalized_prompt_logprobs.tolist()
+                output.normalized_prompt_logprobs = (
+                    output.normalized_prompt_logprobs.tolist()
+                )
 
             next_token_ids = next_token_ids.tolist()
         else:
@@ -582,7 +588,9 @@ class ModelTpServer:
             req.check_finished()
 
             if req.return_logprob:
-                req.decode_token_logprobs.append((next_token_logprobs[i], next_token_id))
+                req.decode_token_logprobs.append(
+                    (next_token_logprobs[i], next_token_id)
+                )
                 if req.top_logprobs_num > 0:
                     req.decode_top_logprobs.append(output.decode_top_logprobs[i])
 
@@ -759,16 +767,27 @@ class ModelTpClient:
             with ThreadPoolExecutor(self.tp_size) as executor:
                 # Launch model processes
                 if server_args.nnodes == 1:
-                    self.procs = list(executor.map(
-                        lambda args: start_rpyc_service_process(*args),
-                        [(ModelTpService, p) for p in model_port_args.model_tp_ports],
-                    ))
+                    self.procs = list(
+                        executor.map(
+                            lambda args: start_rpyc_service_process(*args),
+                            [
+                                (ModelTpService, p)
+                                for p in model_port_args.model_tp_ports
+                            ],
+                        )
+                    )
                     addrs = [("localhost", p) for p in model_port_args.model_tp_ports]
                 else:
-                    addrs = [(ip, port) for ip, port in zip(model_port_args.model_tp_ips, model_port_args.model_tp_ports)]
-
-                self.model_services = list(executor.map(
-                    lambda args: connect_rpyc_service(*args), addrs))
+                    addrs = [
+                        (ip, port)
+                        for ip, port in zip(
+                            model_port_args.model_tp_ips, model_port_args.model_tp_ports
+                        )
+                    ]
+
+                self.model_services = list(
+                    executor.map(lambda args: connect_rpyc_service(*args), addrs)
+                )
 
                 # Init model
                 def init_model(i):

sglang/srt/managers/detokenizer_manager.py

@@ -11,7 +11,7 @@ from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.managers.controller.infer_batch import FINISH_MATCHED_STR
 from sglang.srt.managers.io_struct import BatchStrOut, BatchTokenIDOut
 from sglang.srt.server_args import PortArgs, ServerArgs
-from sglang.utils import get_exception_traceback, graceful_registry
+from sglang.utils import find_printable_text, get_exception_traceback, graceful_registry
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
@@ -57,6 +57,8 @@ class DetokenizerManager:
             output_strs = []
             for i in range(len(recv_obj.rids)):
                 new_text = read_texts[i][len(surr_texts[i]) :]
+                if recv_obj.finished_reason[i] is None:
+                    new_text = find_printable_text(new_text)
                 output_strs.append(recv_obj.decoded_texts[i] + new_text)
 
                 if isinstance(recv_obj.finished_reason[i], FINISH_MATCHED_STR):
@@ -67,7 +69,7 @@ class DetokenizerManager:
             self.send_to_tokenizer.send_pyobj(
                 BatchStrOut(
                     rids=recv_obj.rids,
-                    output_str=output_strs,
+                    output_strs=output_strs,
                     meta_info=recv_obj.meta_info,
                     finished_reason=recv_obj.finished_reason,
                 )

sglang/srt/managers/io_struct.py

@@ -122,7 +122,7 @@ class BatchTokenIDOut:
 @dataclass
 class BatchStrOut:
     rids: List[str]
-    output_str: List[str]
+    output_strs: List[str]
     meta_info: List[Dict]
     finished_reason: List[BaseFinishReason]
 

sglang/srt/managers/tokenizer_manager.py

@@ -316,7 +316,7 @@ class TokenizerManager:
 
                 recv_obj.meta_info[i]["id"] = rid
                 out_dict = {
-                    "text": recv_obj.output_str[i],
+                    "text": recv_obj.output_strs[i],
                     "meta_info": recv_obj.meta_info[i],
                 }
                 state.out_list.append(out_dict)
@@ -333,17 +333,18 @@ class TokenizerManager:
             ret["meta_info"]["decode_token_logprobs"] = self.detokenize_logprob_tokens(
                 ret["meta_info"]["decode_token_logprobs"], return_text_in_logprobs
             )
-        if top_logprobs_num > 0:
-            ret["meta_info"][
-                "prefill_top_logprobs"
-            ] = self.detokenize_top_logprobs_tokens(
-                ret["meta_info"]["prefill_top_logprobs"], return_text_in_logprobs
-            )
-            ret["meta_info"][
-                "decode_top_logprobs"
-            ] = self.detokenize_top_logprobs_tokens(
-                ret["meta_info"]["decode_top_logprobs"], return_text_in_logprobs
-            )
+
+            if top_logprobs_num > 0:
+                ret["meta_info"][
+                    "prefill_top_logprobs"
+                ] = self.detokenize_top_logprobs_tokens(
+                    ret["meta_info"]["prefill_top_logprobs"], return_text_in_logprobs
+                )
+                ret["meta_info"][
+                    "decode_top_logprobs"
+                ] = self.detokenize_top_logprobs_tokens(
+                    ret["meta_info"]["decode_top_logprobs"], return_text_in_logprobs
+                )
         return ret
 
     def detokenize_logprob_tokens(self, token_logprobs, decode_to_text):
@@ -383,7 +384,7 @@ def get_pixel_values(
     try:
         processor = processor or global_processor
         image, image_size = load_image(image_data)
-        if image_size != None:
+        if image_size is not None:
             image_hash = hash(image_data)
             pixel_values = processor.image_processor(image)["pixel_values"]
             for _ in range(len(pixel_values)):