sglang 0.5.2rc2__py3-none-any.whl → 0.5.3__py3-none-any.whl

sglang/srt/managers/scheduler_input_blocker.py

@@ -17,7 +17,7 @@ from enum import Enum, auto
 from typing import Any, List, Optional
 
 from sglang.srt.managers.io_struct import BlockReqInput, BlockReqType
-from sglang.srt.poll_based_barrier import PollBasedBarrier
+from sglang.srt.utils.poll_based_barrier import PollBasedBarrier
 
 logger = logging.getLogger(__name__)
 

sglang/srt/managers/scheduler_metrics_mixin.py

@@ -12,7 +12,6 @@ from sglang.srt.disaggregation.utils import DisaggregationMode
 from sglang.srt.managers.io_struct import TokenizedGenerateReqInput
 from sglang.srt.managers.schedule_policy import PrefillAdder
 from sglang.srt.managers.scheduler import Req, ScheduleBatch
-from sglang.srt.managers.utils import DPBalanceMeta
 from sglang.srt.metrics.collector import SchedulerMetricsCollector, SchedulerStats
 from sglang.srt.utils import get_bool_env_var
 
@@ -47,8 +46,11 @@ class SchedulerMetricsMixin:
         self.spec_num_total_forward_ct = 0
         self.cum_spec_accept_length = 0
         self.cum_spec_accept_count = 0
-        self.total_retracted_reqs = 0
+        self.kv_transfer_speed_gb_s: float = 0.0
+        self.kv_transfer_latency_ms: float = 0.0
+
         self.stats = SchedulerStats()
+
         if self.enable_metrics:
             engine_type = "unified"
             labels = {
@@ -61,33 +63,30 @@ class SchedulerMetricsMixin:
                 labels["dp_rank"] = dp_rank
             self.metrics_collector = SchedulerMetricsCollector(labels=labels)
 
-    def init_dp_balance(self: Scheduler, dp_balance_meta: Optional[DPBalanceMeta]):
-        self.balance_meta = dp_balance_meta
-        if (
-            self.server_args.enable_dp_attention
-            and self.server_args.load_balance_method == "minimum_tokens"
-        ):
-            assert dp_balance_meta is not None
-
-        self.recv_dp_balance_id_this_term = []
-
     def init_kv_events(self: Scheduler, kv_events_config: Optional[str]):
         if self.enable_kv_cache_events:
             self.kv_event_publisher = EventPublisherFactory.create(
                 kv_events_config, self.attn_dp_rank
             )
 
+    def udpate_spec_metrics(self, bs: int, num_accepted_tokens: int):
+        self.spec_num_total_accepted_tokens += num_accepted_tokens + bs
+        self.spec_num_total_forward_ct += bs
+        self.num_generated_tokens += num_accepted_tokens
+
     def log_prefill_stats(
         self: Scheduler,
         adder: PrefillAdder,
         can_run_list: List[Req],
         running_bs: int,
+        running_bs_offline_batch: int,
     ):
         gap_latency = time.perf_counter() - self.last_prefill_stats_tic
         self.last_prefill_stats_tic = time.perf_counter()
         self.last_input_throughput = self.last_prefill_tokens / gap_latency
         self.last_prefill_tokens = adder.log_input_tokens
 
+        # TODO: generalize this for various memory pools
         if self.is_hybrid:
             (
                 full_num_used,
@@ -101,51 +100,53 @@ class SchedulerMetricsMixin:
             ) = self._get_swa_token_info()
             num_used = max(full_num_used, swa_num_used)
             token_usage = max(full_token_usage, swa_token_usage)
-            token_msg = (
+            token_usage_msg = (
                 f"full token usage: {full_token_usage:.2f}, "
                 f"swa token usage: {swa_token_usage:.2f}, "
             )
         else:
             num_used, token_usage, _, _ = self._get_token_info()
-            token_msg = f"token usage: {token_usage:.2f}, "
+            token_usage_msg = f"token usage: {token_usage:.2f}, "
 
-        num_new_seq = len(can_run_list)
         f = (
             f"Prefill batch. "
-            f"#new-seq: {num_new_seq}, "
+            f"#new-seq: {len(can_run_list)}, "
             f"#new-token: {adder.log_input_tokens}, "
             f"#cached-token: {adder.log_hit_tokens}, "
-            f"{token_msg}"
+            f"{token_usage_msg}"
+            f"#running-req: {running_bs}, "
+            f"#queue-req: {len(self.waiting_queue)}, "
         )
 
         if self.disaggregation_mode == DisaggregationMode.PREFILL:
-            f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
-            f += f"#queue-req: {len(self.waiting_queue)}, "
-            f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
-            f += f"input throughput (token/s): {self.last_input_throughput:.2f}, "
-        else:
-            f += f"#running-req: {running_bs}, "
-            f += f"#queue-req: {len(self.waiting_queue)}, "
+            f += f"#prealloc-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
+            f += f"#inflight-req: {len(self.disagg_prefill_inflight_queue)}, "
 
         logger.info(f)
 
         if self.enable_metrics:
+            # Basics
             total_tokens = adder.log_input_tokens + adder.log_hit_tokens
-
             cache_hit_rate = (
                 adder.log_hit_tokens / total_tokens if total_tokens > 0 else 0.0
             )
+
             self.stats.num_running_reqs = running_bs
+            self.stats.num_running_reqs_offline_batch = running_bs_offline_batch
             self.stats.num_used_tokens = num_used
-            self.stats.token_usage = round(token_usage, 2)
+            self.stats.token_usage = token_usage
+            if self.is_hybrid:
+                self.stats.swa_token_usage = swa_token_usage
             self.stats.num_queue_reqs = len(self.waiting_queue)
+            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
             self.stats.cache_hit_rate = cache_hit_rate
 
-            total_queue_latency = 0
-            for req in can_run_list:
-                total_queue_latency += req.queue_time_end - req.queue_time_start
-            self.stats.avg_request_queue_latency = total_queue_latency / num_new_seq
+            # Retract
+            self.stats.num_retracted_reqs = self.num_retracted_reqs
+            self.stats.num_paused_reqs = self.num_paused_reqs
+            self.num_retracted_reqs = self.num_paused_reqs = 0
 
+            # PD disaggregation
             if self.disaggregation_mode == DisaggregationMode.PREFILL:
                 self.stats.num_prefill_prealloc_queue_reqs = len(
                     self.disagg_prefill_bootstrap_queue.queue
@@ -153,7 +154,18 @@ class SchedulerMetricsMixin:
                 self.stats.num_prefill_inflight_queue_reqs = len(
                     self.disagg_prefill_inflight_queue
                 )
+                self.stats.kv_transfer_speed_gb_s = self.kv_transfer_speed_gb_s
+                self.stats.kv_transfer_latency_ms = self.kv_transfer_latency_ms
+            elif self.disaggregation_mode == DisaggregationMode.DECODE:
+                self.stats.num_decode_prealloc_queue_reqs = len(
+                    self.disagg_decode_prealloc_queue.queue
+                )
+                self.stats.num_decode_transfer_queue_reqs = len(
+                    self.disagg_decode_transfer_queue.queue
+                )
 
+            # Others
+            self.calculate_utilization()
             self.metrics_collector.log_stats(self.stats)
             self._emit_kv_metrics()
         self._publish_kv_events()
@@ -166,8 +178,12 @@ class SchedulerMetricsMixin:
         gap_latency = time.perf_counter() - self.last_decode_stats_tic
         self.last_decode_stats_tic = time.perf_counter()
         self.last_gen_throughput = self.num_generated_tokens / gap_latency
+
         self.num_generated_tokens = 0
         num_running_reqs = len(batch.reqs)
+        num_running_reqs_offline_batch = 0
+
+        # TODO: generalize this for various memory pools
         if self.is_hybrid:
             (
                 full_num_used,
@@ -181,7 +197,7 @@ class SchedulerMetricsMixin:
             ) = self._get_swa_token_info()
             num_used = max(full_num_used, swa_num_used)
             token_usage = max(full_token_usage, swa_token_usage)
-            token_msg = (
+            token_usage_msg = (
                 f"#full token: {full_num_used}, "
                 f"full token usage: {full_token_usage:.2f}, "
                 f"#swa token: {swa_num_used}, "
@@ -189,14 +205,14 @@ class SchedulerMetricsMixin:
             )
         else:
             num_used, token_usage, _, _ = self._get_token_info()
-            token_msg = f"#token: {num_used}, " f"token usage: {token_usage:.2f}, "
+            token_usage_msg = f"#token: {num_used}, token usage: {token_usage:.2f}, "
 
         if RECORD_STEP_TIME:
             self.step_time_dict[num_running_reqs].append(
                 gap_latency / self.server_args.decode_log_interval
             )
 
-        msg = f"Decode batch. #running-req: {num_running_reqs}, {token_msg}"
+        msg = f"Decode batch. #running-req: {num_running_reqs}, {token_usage_msg}"
 
         if self.spec_algorithm.is_none():
             spec_accept_length = 0
@@ -208,40 +224,66 @@ class SchedulerMetricsMixin:
             self.cum_spec_accept_count += self.spec_num_total_forward_ct
             self.spec_num_total_accepted_tokens = self.spec_num_total_forward_ct = 0
             msg += f"accept len: {spec_accept_length:.2f}, "
+        cache_hit_rate = 0.0
 
         if self.disaggregation_mode == DisaggregationMode.DECODE:
             msg += f"pre-allocated usage: {self.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.max_total_num_tokens:.2f}, "
+            msg += f"#prealloc-req: {len(self.disagg_decode_prealloc_queue.queue)}, "
+            msg += f"#transfer-req: {len(self.disagg_decode_transfer_queue.queue)}, "
             msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
 
         msg += (
-            f"cuda graph: {can_run_cuda_graph}, "
+            f"{'cuda graph' if self.device == 'cuda' else 'cpu graph'}: {can_run_cuda_graph}, "
             f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
             f"#queue-req: {len(self.waiting_queue)}, "
         )
 
         logger.info(msg)
         if self.enable_metrics:
+            # Basics
             self.stats.num_running_reqs = num_running_reqs
+            self.stats.num_running_reqs_offline_batch = num_running_reqs_offline_batch
             self.stats.num_used_tokens = num_used
-            self.stats.token_usage = round(token_usage, 2)
-            self.stats.cache_hit_rate = 0.0
+            self.stats.token_usage = token_usage
+            if self.is_hybrid:
+                self.stats.swa_token_usage = swa_token_usage
             self.stats.gen_throughput = self.last_gen_throughput
             self.stats.num_queue_reqs = len(self.waiting_queue)
             self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
+            self.stats.cache_hit_rate = cache_hit_rate
             self.stats.spec_accept_length = spec_accept_length
-            self.stats.total_retracted_reqs = self.total_retracted_reqs
-            self.metrics_collector.log_stats(self.stats)
-            if self.disaggregation_mode == DisaggregationMode.DECODE:
+
+            # Retract
+            self.stats.num_retracted_reqs = self.num_retracted_reqs
+            self.stats.num_paused_reqs = self.num_paused_reqs
+            self.num_retracted_reqs = self.num_paused_reqs = 0
+
+            # PD disaggregation
+            if self.disaggregation_mode == DisaggregationMode.PREFILL:
+                self.stats.num_prefill_prealloc_queue_reqs = len(
+                    self.disagg_prefill_bootstrap_queue.queue
+                )
+                self.stats.num_prefill_inflight_queue_reqs = len(
+                    self.disagg_prefill_inflight_queue
+                )
+            elif self.disaggregation_mode == DisaggregationMode.DECODE:
                 self.stats.num_decode_prealloc_queue_reqs = len(
                     self.disagg_decode_prealloc_queue.queue
                 )
                 self.stats.num_decode_transfer_queue_reqs = len(
                     self.disagg_decode_transfer_queue.queue
                 )
+
+            # Others
+            self.calculate_utilization()
+            self.metrics_collector.log_stats(self.stats)
             self._emit_kv_metrics()
         self._publish_kv_events()
 
     def _emit_kv_metrics(self: Scheduler):
+        if not self.enable_kv_cache_events:
+            return
+
         kv_metrics = KvMetrics()
         kv_metrics.request_active_slots = self.stats.num_running_reqs
         kv_metrics.request_total_slots = self.max_running_requests
@@ -258,93 +300,24 @@ class SchedulerMetricsMixin:
             self.send_metrics_from_scheduler.send_pyobj(kv_metrics)
 
     def _publish_kv_events(self: Scheduler):
-        if self.enable_kv_cache_events:
-            events = self.tree_cache.take_events()
-            if events:
-                batch = KVEventBatch(ts=time.time(), events=events)
-                self.kv_event_publisher.publish(batch)
-
-    def maybe_update_dp_balance_data(
-        self: Scheduler, recv_req: TokenizedGenerateReqInput
-    ):
-        if (
-            self.server_args.enable_dp_attention
-            and self.server_args.load_balance_method == "minimum_tokens"
-        ):
-            self.recv_dp_balance_id_this_term.append(recv_req.dp_balance_id)
-
-    def maybe_handle_dp_balance_data(self: Scheduler):
-        if (
-            self.server_args.load_balance_method == "minimum_tokens"
-            and self.forward_ct % 40 == 0
-        ):
-            holding_tokens = self.get_load()
-
-            new_recv_dp_balance_id_list, holding_token_list = (
-                self.gather_dp_balance_info(holding_tokens)
-            )
+        if not self.enable_kv_cache_events:
+            return
 
-            self.recv_dp_balance_id_this_term.clear()
-            if self.tp_rank == 0:  # only first worker write info
-                self.write_shared_dp_balance_info(
-                    new_recv_dp_balance_id_list, holding_token_list
-                )
+        events = self.tree_cache.take_events()
+        if events:
+            batch = KVEventBatch(ts=time.time(), events=events)
+            self.kv_event_publisher.publish(batch)
 
-    def gather_dp_balance_info(
-        self: Scheduler, holding_tokens_list
-    ) -> Union[None, List[List[int]]]:
-        """gather recv_dp_balance_id_this_term and holding tokens per worker for dp balance"""
-        recv_list = self.recv_dp_balance_id_this_term
-        assert len(recv_list) <= 511, (
-            "The number of requests received this round is too large. "
-            "Please increase gather_tensor_size and onfly_info_size."
-        )
-        # The maximum size of the tensor used for gathering data from all workers.
-        gather_tensor_size = 512
-
-        # recv_tensor: | holding_tokens | len(recv_dp_balance_id) | recv_dp_balance_ids
-        recv_tensor = torch.zeros(gather_tensor_size, dtype=torch.int32)
-        recv_tensor[0] = holding_tokens_list
-        recv_tensor[1] = len(recv_list)  # The first element is the length of the list.
-        recv_tensor[2 : len(recv_list) + 2] = torch.tensor(recv_list, dtype=torch.int32)
-
-        if self.tp_rank == 0:
-            gathered_list = [
-                torch.zeros(gather_tensor_size, dtype=torch.int32)
-                for _ in range(self.balance_meta.num_workers)
-            ]
+    def calculate_utilization(self):
+        if self.disaggregation_mode == DisaggregationMode.PREFILL:
+            self.stats.utilization = -1
         else:
-            gathered_list = None
-
-        torch.distributed.gather(recv_tensor, gathered_list, group=self.tp_cpu_group)
-
-        gathered_id_list_per_worker = None
-        if self.tp_rank == 0:
-            gathered_id_list_per_worker = []
-            holding_tokens_list = []
-            for tensor in gathered_list:
-                holding_tokens_list.append(tensor[0].item())
-                list_length = tensor[1].item()
-                gathered_id_list_per_worker.append(tensor[2 : list_length + 2].tolist())
-
-        return gathered_id_list_per_worker, holding_tokens_list
-
-    def write_shared_dp_balance_info(self: Scheduler, new_recv_rid_lists, local_tokens):
-        meta = self.balance_meta
-
-        with meta.mutex:
-            onfly_list: List[Dict[int, int]] = meta.get_shared_onfly()
-            assert len(new_recv_rid_lists) == len(onfly_list), "num_worker not equal"
-            # 1.Check if the rid received by each worker this round is present in onfly.
-            #   If it is, remove the corresponding onfly item.
-            worker_id = 0
-            for new_recv_rids, on_fly_reqs in zip(new_recv_rid_lists, onfly_list):
-                for new_recv_rid in new_recv_rids:
-                    assert (
-                        new_recv_rid in on_fly_reqs
-                    ), f"{new_recv_rid=} not in {worker_id=} {on_fly_reqs=}, data consistency is wrong"
-                    del on_fly_reqs[new_recv_rid]
-                worker_id += 1
-            # 2. Atomically write local_tokens and onfly into shm under the mutex
-            meta.set_shared_onfly_info(onfly_list)
-            meta.set_shared_local_tokens(local_tokens)
+            if (
+                self.stats.max_running_requests_under_SLO is not None
+                and self.stats.max_running_requests_under_SLO > 0
+            ):
+                self.stats.utilization = max(
+                    self.stats.num_running_reqs
+                    / self.stats.max_running_requests_under_SLO,
+                    self.stats.token_usage / 0.9,
+                )

sglang/srt/managers/scheduler_output_processor_mixin.py

@@ -5,9 +5,15 @@ import threading
 import time
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
+import torch
+
 from sglang.srt.disaggregation.utils import DisaggregationMode
 from sglang.srt.layers.logits_processor import LogitsProcessorOutput
-from sglang.srt.managers.io_struct import AbortReq, BatchEmbeddingOut, BatchTokenIDOut
+from sglang.srt.managers.io_struct import (
+    AbortReq,
+    BatchEmbeddingOutput,
+    BatchTokenIDOutput,
+)
 from sglang.srt.managers.schedule_batch import BaseFinishReason, Req, ScheduleBatch
 
 if TYPE_CHECKING:
@@ -71,6 +77,7 @@ class SchedulerOutputProcessorMixin:
 
             # Check finish conditions
             logprob_pt = 0
+
             for i, (req, next_token_id) in enumerate(zip(batch.reqs, next_token_ids)):
                 if req.is_retracted:
                     continue
@@ -88,7 +95,7 @@ class SchedulerOutputProcessorMixin:
 
                     if req.finished():
                         self.tree_cache.cache_finished_req(req)
-                        req.time_stats.completion_time = time.time()
+                        req.time_stats.completion_time = time.perf_counter()
                     elif not batch.decoding_reqs or req not in batch.decoding_reqs:
                         # This updates radix so others can match
                         self.tree_cache.cache_unfinished_req(req)
@@ -99,6 +106,7 @@ class SchedulerOutputProcessorMixin:
                         extend_logprob_start_len = extend_logprob_start_len_per_req[i]
                         extend_input_len = extend_input_len_per_req[i]
                         num_input_logprobs = extend_input_len - extend_logprob_start_len
+
                         if req.return_logprob:
                             self.add_logprob_return_values(
                                 i,
@@ -136,7 +144,7 @@ class SchedulerOutputProcessorMixin:
                             logger.error(
                                 f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
                             )
-                            self.abort_request(AbortReq(req.rid))
+                            self.abort_request(AbortReq(rid=req.rid))
                         req.grammar.finished = req.finished()
                 else:
                     # being chunked reqs' prefill is not finished
@@ -169,8 +177,7 @@ class SchedulerOutputProcessorMixin:
             self.set_next_batch_sampling_info_done(batch)
 
         else:  # embedding or reward model
-            embeddings, bid = result.embeddings, result.bid
-            embeddings = embeddings.tolist()
+            embeddings = result.embeddings.tolist()
 
             # Check finish conditions
             for i, req in enumerate(batch.reqs):
@@ -246,8 +253,14 @@ class SchedulerOutputProcessorMixin:
 
             req.check_finished()
             if req.finished():
-                self.tree_cache.cache_finished_req(req)
-                req.time_stats.completion_time = time.time()
+                if self.server_args.disaggregation_decode_enable_offload_kvcache:
+                    # Asynchronously offload KV cache; cache_finished_req will be called after Device->Host transfer completes
+                    if not self.decode_offload_manager.offload_kv_cache(req):
+                        self.tree_cache.cache_finished_req(req)
+                else:
+                    self.tree_cache.cache_finished_req(req)
+
+                req.time_stats.completion_time = time.perf_counter()
 
             if req.return_logprob and batch.spec_algorithm.is_none():
                 # speculative worker handles logprob in speculative decoding
@@ -283,7 +296,7 @@ class SchedulerOutputProcessorMixin:
                     logger.error(
                         f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
                     )
-                    self.abort_request(AbortReq(req.rid))
+                    self.abort_request(AbortReq(rid=req.rid))
                 req.grammar.finished = req.finished()
 
         self.set_next_batch_sampling_info_done(batch)
@@ -441,27 +454,59 @@ class SchedulerOutputProcessorMixin:
         output: LogitsProcessorOutput,
     ):
         """Attach logprobs to the return values."""
-        req.output_token_logprobs_val.append(output.next_token_logprobs[i])
-        req.output_token_logprobs_idx.append(next_token_ids[i])
-
-        self.add_input_logprob_return_values(
-            i, req, output, pt, num_input_logprobs, last_prefill_chunk=True
-        )
+        if output.next_token_logprobs is not None:
+            req.output_token_logprobs_val.append(output.next_token_logprobs[i])
+            req.output_token_logprobs_idx.append(next_token_ids[i])
+
+        # Only add input logprobs if there are input tokens to process
+        # Note: For prefill-only requests with default logprob_start_len, this will be 0,
+        # meaning we only compute output logprobs (which is the intended behavior)
+        if num_input_logprobs > 0:
+            self.add_input_logprob_return_values(
+                i, req, output, pt, num_input_logprobs, last_prefill_chunk=True
+            )
+        else:
+            self._initialize_empty_logprob_containers(req)
 
         if req.top_logprobs_num > 0:
             req.output_top_logprobs_val.append(output.next_token_top_logprobs_val[i])
             req.output_top_logprobs_idx.append(output.next_token_top_logprobs_idx[i])
 
-        if req.token_ids_logprob is not None:
-            req.output_token_ids_logprobs_val.append(
-                output.next_token_token_ids_logprobs_val[i]
-            )
+        if (
+            req.token_ids_logprob is not None
+            and output.next_token_token_ids_logprobs_val is not None
+        ):
+            # Convert GPU tensor to list if needed
+            logprobs_val = output.next_token_token_ids_logprobs_val[i]
+            if isinstance(logprobs_val, torch.Tensor):
+                logprobs_val = logprobs_val.tolist()
+            req.output_token_ids_logprobs_val.append(logprobs_val)
             req.output_token_ids_logprobs_idx.append(
                 output.next_token_token_ids_logprobs_idx[i]
             )
 
         return num_input_logprobs
 
+    def _initialize_empty_logprob_containers(self, req: Req) -> None:
+        """
+        Initialize logprob fields to empty lists if unset.
+
+        This is needed for prefill-only requests where the normal initialization
+        flow might be bypassed, but downstream code expects these fields to be lists.
+        """
+        if req.input_token_logprobs_val is None:
+            req.input_token_logprobs_val = []
+        if req.input_token_logprobs_idx is None:
+            req.input_token_logprobs_idx = []
+        if req.input_top_logprobs_val is None:
+            req.input_top_logprobs_val = []
+        if req.input_top_logprobs_idx is None:
+            req.input_top_logprobs_idx = []
+        if req.input_token_ids_logprobs_val is None:
+            req.input_token_ids_logprobs_val = []
+        if req.input_token_ids_logprobs_idx is None:
+            req.input_token_ids_logprobs_idx = []
+
     def stream_output(
         self: Scheduler,
         reqs: List[Req],
@@ -673,8 +718,7 @@ class SchedulerOutputProcessorMixin:
                 return
 
             self.send_to_detokenizer.send_pyobj(
-                BatchTokenIDOut(
-                    rids,
+                BatchTokenIDOutput(
                     finished_reasons,
                     decoded_texts,
                     decode_ids_list,
@@ -700,6 +744,9 @@ class SchedulerOutputProcessorMixin:
                     output_token_ids_logprobs_val,
                     output_token_ids_logprobs_idx,
                     output_hidden_states,
+                    rids=rids,
+                    placeholder_tokens_idx=None,
+                    placeholder_tokens_val=None,
                 )
             )
 
@@ -718,7 +765,13 @@ class SchedulerOutputProcessorMixin:
                 prompt_tokens.append(len(req.origin_input_ids))
                 cached_tokens.append(req.cached_tokens)
         self.send_to_detokenizer.send_pyobj(
-            BatchEmbeddingOut(
-                rids, finished_reasons, embeddings, prompt_tokens, cached_tokens
+            BatchEmbeddingOutput(
+                finished_reasons,
+                embeddings,
+                prompt_tokens,
+                cached_tokens,
+                rids=rids,
+                placeholder_tokens_idx=None,
+                placeholder_tokens_val=None,
             )
         )

sglang/srt/managers/scheduler_profiler_mixin.py

@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 class SchedulerProfilerMixin:
 
-    def init_profier(self):
+    def init_profiler(self):
         self.torch_profiler = None
         self.torch_profiler_output_dir: Optional[str] = None
         self.profiler_activities: Optional[List[str]] = None
@@ -97,7 +97,7 @@ class SchedulerProfilerMixin:
     def start_profile(
         self, stage: Optional[ForwardMode] = None
     ) -> ProfileReqOutput | None:
-        stage_str = f" for {stage.__str__()}" if stage else ""
+        stage_str = f" for {stage.name}" if stage else ""
         logger.info(
             f"Profiling starts{stage_str}. Traces will be saved to: {self.torch_profiler_output_dir} (with profile id: {self.profile_id})",
         )
@@ -181,7 +181,7 @@ class SchedulerProfilerMixin:
         if not Path(self.torch_profiler_output_dir).exists():
             Path(self.torch_profiler_output_dir).mkdir(parents=True, exist_ok=True)
 
-        stage_suffix = f"-{stage.__str__()}" if stage else ""
+        stage_suffix = f"-{stage.name}" if stage else ""
         logger.info("Stop profiling" + stage_suffix + "...")
         if self.torch_profiler is not None:
             self.torch_profiler.stop()
@@ -204,7 +204,7 @@ class SchedulerProfilerMixin:
 
             torch.distributed.barrier(self.tp_cpu_group)
             if self.tp_rank == 0:
-                from sglang.srt.utils import rpd_to_chrome_trace
+                from sglang.srt.utils.rpd_utils import rpd_to_chrome_trace
 
                 rpd_to_chrome_trace("trace.rpd", self.rpd_profile_path)
             self.rpd_profiler = None
@@ -247,7 +247,7 @@ class SchedulerProfilerMixin:
                 if self.profiler_decode_ct == 0:
                     if self.profile_in_progress:
                         # force trace flush
-                        self.stop_profile(ForwardMode.EXTEND)
+                        self.stop_profile(stage=ForwardMode.EXTEND)
                     self.start_profile(batch.forward_mode)
                 self.profiler_decode_ct += 1
                 if self.profiler_decode_ct > self.profiler_target_decode_ct:
@@ -294,6 +294,6 @@ class SchedulerProfilerMixin:
                     recv_req.profile_by_stage,
                     recv_req.profile_id,
                 )
-                return self.start_profile(True)
+                return self.start_profile()
         else:
             return self.stop_profile()

sglang/srt/managers/scheduler_update_weights_mixin.py

@@ -5,6 +5,8 @@ import torch
 
 from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS
 from sglang.srt.managers.io_struct import (
+    DestroyWeightsUpdateGroupReqInput,
+    DestroyWeightsUpdateGroupReqOutput,
     GetWeightsByNameReqInput,
     GetWeightsByNameReqOutput,
     InitWeightsUpdateGroupReqInput,
@@ -41,6 +43,11 @@ class SchedulerUpdateWeightsMixin:
         success, message = self.tp_worker.init_weights_update_group(recv_req)
         return InitWeightsUpdateGroupReqOutput(success, message)
 
+    def destroy_weights_update_group(self, recv_req: DestroyWeightsUpdateGroupReqInput):
+        """Destroy the online model parameter update group."""
+        success, message = self.tp_worker.destroy_weights_update_group(recv_req)
+        return DestroyWeightsUpdateGroupReqOutput(success, message)
+
     def update_weights_from_distributed(
         self,
         recv_req: UpdateWeightsFromDistributedReqInput,