sglang 0.3.3.post1__py3-none-any.whl → 0.3.4__py3-none-any.whl

sglang/srt/openai_api/adapter.py

@@ -25,7 +25,7 @@ from http import HTTPStatus
 from typing import Dict, List
 
 from fastapi import HTTPException, Request, UploadFile
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import ORJSONResponse, StreamingResponse
 from pydantic import ValidationError
 
 try:
@@ -101,7 +101,7 @@ def create_error_response(
     status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
 ):
     error = ErrorResponse(message=message, type=err_type, code=status_code.value)
-    return JSONResponse(content=error.model_dump(), status_code=error.code)
+    return ORJSONResponse(content=error.model_dump(), status_code=error.code)
 
 
 def create_streaming_error_response(
@@ -302,7 +302,12 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
             if not isinstance(ret, list):
                 ret = [ret]
             if end_point == "/v1/chat/completions":
-                responses = v1_chat_generate_response(request, ret, to_file=True)
+                responses = v1_chat_generate_response(
+                    request,
+                    ret,
+                    to_file=True,
+                    cache_report=tokenizer_manager.server_args.enable_cache_report,
+                )
             else:
                 responses = v1_generate_response(
                     request, ret, tokenizer_manager, to_file=True
@@ -493,23 +498,38 @@ def v1_generate_request(
         top_logprobs_nums.append(
             request.logprobs if request.logprobs is not None else 0
         )
-        sampling_params_list.append(
-            {
-                "temperature": request.temperature,
-                "max_new_tokens": request.max_tokens,
-                "min_new_tokens": request.min_tokens,
-                "stop": request.stop,
-                "stop_token_ids": request.stop_token_ids,
-                "top_p": request.top_p,
-                "presence_penalty": request.presence_penalty,
-                "frequency_penalty": request.frequency_penalty,
-                "repetition_penalty": request.repetition_penalty,
-                "regex": request.regex,
-                "json_schema": request.json_schema,
-                "n": request.n,
-                "ignore_eos": request.ignore_eos,
-            }
-        )
+        sampling_params = []
+        if isinstance(request.no_stop_trim, list):
+            num_reqs = len(request.prompt)
+        else:
+            num_reqs = 1
+        for i in range(num_reqs):
+            sampling_params.append(
+                {
+                    "temperature": request.temperature,
+                    "max_new_tokens": request.max_tokens,
+                    "min_new_tokens": request.min_tokens,
+                    "stop": request.stop,
+                    "stop_token_ids": request.stop_token_ids,
+                    "top_p": request.top_p,
+                    "presence_penalty": request.presence_penalty,
+                    "frequency_penalty": request.frequency_penalty,
+                    "repetition_penalty": request.repetition_penalty,
+                    "regex": request.regex,
+                    "json_schema": request.json_schema,
+                    "n": request.n,
+                    "ignore_eos": request.ignore_eos,
+                    "no_stop_trim": (
+                        request.no_stop_trim
+                        if not isinstance(request.no_stop_trim, list)
+                        else request.no_stop_trim[i]
+                    ),
+                }
+            )
+        if num_reqs == 1:
+            sampling_params_list.append(sampling_params[0])
+        else:
+            sampling_params_list.append(sampling_params)
 
     if len(all_requests) == 1:
         prompt = prompts[0]
@@ -601,16 +621,19 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
         else:
             logprobs = None
 
+        finish_reason = ret_item["meta_info"]["finish_reason"]
+
         if to_file:
             # to make the choise data json serializable
             choice_data = {
                 "index": 0,
                 "text": text,
                 "logprobs": logprobs,
-                "finish_reason": (
-                    ret_item["meta_info"]["finish_reason"]["type"]
-                    if ret_item["meta_info"]["finish_reason"]
-                    else ""
+                "finish_reason": (finish_reason["type"] if finish_reason else ""),
+                "matched_stop": (
+                    finish_reason["matched"]
+                    if finish_reason and "matched" in finish_reason
+                    else None
                 ),
             }
         else:
@@ -618,10 +641,11 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
                 index=idx,
                 text=text,
                 logprobs=logprobs,
-                finish_reason=(
-                    ret_item["meta_info"]["finish_reason"]["type"]
-                    if ret_item["meta_info"]["finish_reason"]
-                    else ""
+                finish_reason=(finish_reason["type"] if finish_reason else ""),
+                matched_stop=(
+                    finish_reason["matched"]
+                    if finish_reason and "matched" in finish_reason
+                    else None
                 ),
             )
 
@@ -751,14 +775,16 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
 
                     delta = text[len(stream_buffer) :]
                     stream_buffer = stream_buffer + delta
+                    finish_reason = content["meta_info"]["finish_reason"]
                     choice_data = CompletionResponseStreamChoice(
                         index=index,
                         text=delta,
                         logprobs=logprobs,
-                        finish_reason=(
-                            content["meta_info"]["finish_reason"]["type"]
-                            if content["meta_info"]["finish_reason"]
-                            else ""
+                        finish_reason=(finish_reason["type"] if finish_reason else ""),
+                        matched_stop=(
+                            finish_reason["matched"]
+                            if finish_reason and "matched" in finish_reason
+                            else None
                         ),
                     )
                     chunk = CompletionStreamResponse(
@@ -910,6 +936,7 @@ def v1_chat_generate_request(
             "repetition_penalty": request.repetition_penalty,
             "regex": request.regex,
             "n": request.n,
+            "ignore_eos": request.ignore_eos,
         }
         if request.response_format and request.response_format.type == "json_schema":
             sampling_params["json_schema"] = convert_json_schema_to_str(
@@ -954,7 +981,7 @@ def v1_chat_generate_request(
     return adapted_request, all_requests
 
 
-def v1_chat_generate_response(request, ret, to_file=False):
+def v1_chat_generate_response(request, ret, to_file=False, cache_report=False):
     choices = []
 
     for idx, ret_item in enumerate(ret):
@@ -995,16 +1022,19 @@ def v1_chat_generate_response(request, ret, to_file=False):
         else:
             choice_logprobs = None
 
+        finish_reason = ret_item["meta_info"]["finish_reason"]
+
         if to_file:
             # to make the choice data json serializable
             choice_data = {
                 "index": 0,
                 "message": {"role": "assistant", "content": ret_item["text"]},
                 "logprobs": choice_logprobs,
-                "finish_reason": (
-                    ret_item["meta_info"]["finish_reason"]["type"]
-                    if ret_item["meta_info"]["finish_reason"]
-                    else ""
+                "finish_reason": (finish_reason["type"] if finish_reason else ""),
+                "matched_stop": (
+                    finish_reason["matched"]
+                    if finish_reason and "matched" in finish_reason
+                    else None
                 ),
             }
         else:
@@ -1012,10 +1042,11 @@ def v1_chat_generate_response(request, ret, to_file=False):
                 index=idx,
                 message=ChatMessage(role="assistant", content=ret_item["text"]),
                 logprobs=choice_logprobs,
-                finish_reason=(
-                    ret_item["meta_info"]["finish_reason"]["type"]
-                    if ret_item["meta_info"]["finish_reason"]
-                    else ""
+                finish_reason=(finish_reason["type"] if finish_reason else ""),
+                matched_stop=(
+                    finish_reason["matched"]
+                    if finish_reason and "matched" in finish_reason
+                    else None
                 ),
             )
 
@@ -1051,6 +1082,7 @@ def v1_chat_generate_response(request, ret, to_file=False):
             ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
         )
         completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
+        cached_tokens = sum(item["meta_info"].get("cached_tokens", 0) for item in ret)
         response = ChatCompletionResponse(
             id=ret[0]["meta_info"]["id"],
             model=request.model,
@@ -1059,6 +1091,9 @@ def v1_chat_generate_response(request, ret, to_file=False):
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=prompt_tokens + completion_tokens,
+                prompt_tokens_details=(
+                    {"cached_tokens": cached_tokens} if cache_report else None
+                ),
             ),
         )
         return response
@@ -1134,6 +1169,8 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                     else:
                         choice_logprobs = None
 
+                    finish_reason = content["meta_info"]["finish_reason"]
+
                     if is_first:
                         # First chunk with role
                         is_first = False
@@ -1141,9 +1178,12 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             index=index,
                             delta=DeltaMessage(role="assistant"),
                             finish_reason=(
-                                content["meta_info"]["finish_reason"]["type"]
-                                if content["meta_info"]["finish_reason"]
-                                else ""
+                                finish_reason["type"] if finish_reason else ""
+                            ),
+                            matched_stop=(
+                                finish_reason["matched"]
+                                if finish_reason and "matched" in finish_reason
+                                else None
                             ),
                             logprobs=choice_logprobs,
                         )
@@ -1160,10 +1200,11 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                     choice_data = ChatCompletionResponseStreamChoice(
                         index=index,
                         delta=DeltaMessage(content=delta),
-                        finish_reason=(
-                            content["meta_info"]["finish_reason"]["type"]
-                            if content["meta_info"]["finish_reason"]
-                            else ""
+                        finish_reason=(finish_reason["type"] if finish_reason else ""),
+                        matched_stop=(
+                            finish_reason["matched"]
+                            if finish_reason and "matched" in finish_reason
+                            else None
                         ),
                         logprobs=choice_logprobs,
                     )
@@ -1224,7 +1265,9 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
     if not isinstance(ret, list):
         ret = [ret]
 
-    response = v1_chat_generate_response(request, ret)
+    response = v1_chat_generate_response(
+        request, ret, cache_report=tokenizer_manager.server_args.enable_cache_report
+    )
 
     return response
 

sglang/srt/openai_api/protocol.py

@@ -76,6 +76,8 @@ class UsageInfo(BaseModel):
     prompt_tokens: int = 0
     total_tokens: int = 0
     completion_tokens: Optional[int] = 0
+    # only used to return cached tokens when --enable-cache-report is set
+    prompt_tokens_details: Optional[Dict[str, int]] = None
 
 
 class StreamOptions(BaseModel):
@@ -170,10 +172,11 @@ class CompletionRequest(BaseModel):
     # Extra parameters for SRT backend only and will be ignored by OpenAI models.
     regex: Optional[str] = None
     json_schema: Optional[str] = None
-    ignore_eos: Optional[bool] = False
-    min_tokens: Optional[int] = 0
+    ignore_eos: bool = False
+    min_tokens: int = 0
     repetition_penalty: Optional[float] = 1.0
     stop_token_ids: Optional[List[int]] = Field(default_factory=list)
+    no_stop_trim: Union[bool, List[bool]] = False
 
 
 class CompletionResponseChoice(BaseModel):
@@ -181,6 +184,7 @@ class CompletionResponseChoice(BaseModel):
     text: str
     logprobs: Optional[LogProbs] = None
     finish_reason: Optional[str] = None
+    matched_stop: Union[None, int, str] = None
 
 
 class CompletionResponse(BaseModel):
@@ -197,6 +201,7 @@ class CompletionResponseStreamChoice(BaseModel):
     text: str
     logprobs: Optional[LogProbs] = None
     finish_reason: Optional[str] = None
+    matched_stop: Union[None, int, str] = None
 
 
 class CompletionStreamResponse(BaseModel):
@@ -275,6 +280,7 @@ class ChatCompletionRequest(BaseModel):
     min_tokens: Optional[int] = 0
     repetition_penalty: Optional[float] = 1.0
     stop_token_ids: Optional[List[int]] = Field(default_factory=list)
+    ignore_eos: bool = False
 
 
 class ChatMessage(BaseModel):
@@ -287,6 +293,7 @@ class ChatCompletionResponseChoice(BaseModel):
     message: ChatMessage
     logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None
     finish_reason: str
+    matched_stop: Union[None, int, str] = None
 
 
 class ChatCompletionResponse(BaseModel):
@@ -308,6 +315,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
     delta: DeltaMessage
     logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None
     finish_reason: Optional[str] = None
+    matched_stop: Union[None, int, str] = None
 
 
 class ChatCompletionStreamResponse(BaseModel):

sglang/srt/sampling/penaltylib/orchestrator.py

@@ -37,12 +37,16 @@ class BatchedPenalizerOrchestrator:
 
         self.penalizers = {Penalizer: Penalizer(self) for Penalizer in Penalizers}
 
+        is_required = False
         for penalizer in self.penalizers.values():
-            penalizer.prepare_if_required()
+            pen_is_required = penalizer.prepare_if_required()
+            is_required |= pen_is_required
+        self.is_required = is_required
 
-        self.cumulate_input_tokens(
-            input_ids=[req.origin_input_ids for req in self.reqs()]
-        )
+        if self.is_required:
+            self.cumulate_input_tokens(
+                input_ids=[req.origin_input_ids for req in self.reqs()]
+            )
 
     def reqs(self):
         return self.batch.reqs
@@ -79,6 +83,9 @@ class BatchedPenalizerOrchestrator:
         Args:
             output_ids (typing.Union[typing.List[torch.Tensor], typing.List[typing.List[int]]]): The output tokens.
         """
+        if not self.is_required:
+            return
+
         token_ids = _TokenIDs(orchestrator=self, token_ids=output_ids)
 
         for penalizer in self.penalizers.values():
@@ -95,6 +102,9 @@ class BatchedPenalizerOrchestrator:
         Returns:
             torch.Tensor: The logits after applying the penalizers.
         """
+        if not self.is_required:
+            return
+
         for penalizer in self.penalizers.values():
             logits = penalizer.apply(logits)
 
@@ -112,10 +122,16 @@ class BatchedPenalizerOrchestrator:
             indices_to_keep (typing.List[int]): List of indices to keep in the batch.
             indices_tensor_to_keep (torch.Tensor = None): Tensor of indices to keep in the batch. If not None, it will be used instead of converting indices_to_keep to a tensor.
         """
+        if not self.is_required:
+            return
+
         empty_indices = len(indices_to_keep) == 0
 
+        is_required = False
         for penalizer in self.penalizers.values():
-            if not penalizer.is_required() or empty_indices:
+            tmp_is_required = penalizer.is_required()
+            is_required = is_required or tmp_is_required
+            if not tmp_is_required or empty_indices:
                 penalizer.teardown()
             else:
                 # create tensor index only when it's needed
@@ -128,6 +144,7 @@ class BatchedPenalizerOrchestrator:
                     indices_to_keep=indices_to_keep,
                     indices_tensor_to_keep=indices_tensor_to_keep,
                 )
+        self.is_required = is_required
 
     def merge(self, their: "BatchedPenalizerOrchestrator"):
         """
@@ -140,11 +157,10 @@ class BatchedPenalizerOrchestrator:
         Args:
             their (BatchedPenalizerOrchestrator): The orchestrator to merge into this one.
         """
-        if self.vocab_size != their.vocab_size:
-            raise ValueError(
-                f"vocab_size mismatch: {self.vocab_size} != {their.vocab_size}"
-            )
+        if not self.is_required and not their.is_required:
+            return
 
+        self.is_required |= their.is_required
         for Penalizer, their_penalizer in their.penalizers.items():
             if Penalizer not in self.penalizers:
                 raise ValueError(f"Penalizer {Penalizer} not found in self.penalizers")
@@ -250,6 +266,9 @@ class _BatchedPenalizer(abc.ABC):
     def prepare_if_required(self):
         if self.is_required():
             self.prepare()
+            return True
+        else:
+            return False
 
     def teardown(self):
         if self.is_prepared():

sglang/srt/sampling/sampling_batch_info.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import dataclasses
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, Optional
 
 import torch
 
@@ -20,6 +20,9 @@ class SamplingBatchInfo:
     top_ks: torch.Tensor
     min_ps: torch.Tensor
 
+    # All requests use greedy sampling
+    is_all_greedy: bool
+
     # Dispatch in CUDA graph
     need_min_p_sampling: bool
 
@@ -33,30 +36,39 @@ class SamplingBatchInfo:
     regex_fsm_states: List[int] = None
 
     # Penalizer
-    penalizer_orchestrator: penaltylib.BatchedPenalizerOrchestrator = None
-    linear_penalties: torch.Tensor = None
-    scaling_penalties: torch.Tensor = None
+    penalizer_orchestrator: Optional[penaltylib.BatchedPenalizerOrchestrator] = None
+    linear_penalties: Optional[torch.Tensor] = None
+    scaling_penalties: Optional[torch.Tensor] = None
 
     # Device
     device: str = "cuda"
 
     @classmethod
-    def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int):
+    def from_schedule_batch(
+        cls,
+        batch: ScheduleBatch,
+        vocab_size: int,
+        disable_penalizer: bool,
+    ):
         reqs = batch.reqs
-        with batch.input_ids.device:
-            temperatures = torch.tensor(
+        device = batch.input_ids.device
+        temperatures = (
+            torch.tensor(
                 [r.sampling_params.temperature for r in reqs],
                 dtype=torch.float,
-            ).view(-1, 1)
-            top_ps = torch.tensor(
-                [r.sampling_params.top_p for r in reqs], dtype=torch.float
-            )
-            top_ks = torch.tensor(
-                [r.sampling_params.top_k for r in reqs], dtype=torch.int
-            )
-            min_ps = torch.tensor(
-                [r.sampling_params.min_p for r in reqs], dtype=torch.float
             )
+            .view(-1, 1)
+            .to(device, non_blocking=True)
+        )
+        top_ps = torch.tensor(
+            [r.sampling_params.top_p for r in reqs], dtype=torch.float
+        ).to(device, non_blocking=True)
+        top_ks = torch.tensor(
+            [r.sampling_params.top_k for r in reqs], dtype=torch.int32
+        ).to(device, non_blocking=True)
+        min_ps = torch.tensor(
+            [r.sampling_params.min_p for r in reqs], dtype=torch.float
+        ).to(device, non_blocking=True)
 
         ret = cls(
             temperatures=temperatures,
@@ -64,6 +76,7 @@ class SamplingBatchInfo:
             top_ks=top_ks,
             min_ps=min_ps,
             need_min_p_sampling=any(r.sampling_params.min_p > 0 for r in reqs),
+            is_all_greedy=top_ks.max().item() <= 1,
             vocab_size=vocab_size,
             device=batch.input_ids.device,
         )
@@ -75,18 +88,21 @@ class SamplingBatchInfo:
         #
         # While we choose not to even create the class instances if they are not required, this
         # could add additional complexity to the {ScheduleBatch} class, especially we need to
-        # handle {filter_batch()} and {merge()} cases as well.
-        ret.penalizer_orchestrator = penaltylib.BatchedPenalizerOrchestrator(
-            vocab_size=vocab_size,
-            batch=batch,
-            device=batch.input_ids.device,
-            Penalizers={
-                penaltylib.BatchedFrequencyPenalizer,
-                penaltylib.BatchedMinNewTokensPenalizer,
-                penaltylib.BatchedPresencePenalizer,
-                penaltylib.BatchedRepetitionPenalizer,
-            },
-        )
+        # handle {filter_batch()} and {merge_batch()} cases as well.
+        if disable_penalizer:
+            ret.penalizer_orchestrator = None
+        else:
+            ret.penalizer_orchestrator = penaltylib.BatchedPenalizerOrchestrator(
+                vocab_size=vocab_size,
+                batch=batch,
+                device=batch.input_ids.device,
+                Penalizers={
+                    penaltylib.BatchedFrequencyPenalizer,
+                    penaltylib.BatchedMinNewTokensPenalizer,
+                    penaltylib.BatchedPresencePenalizer,
+                    penaltylib.BatchedRepetitionPenalizer,
+                },
+            )
 
         # Handle logit bias but only allocate when needed
         ret.logit_bias = None
@@ -97,46 +113,50 @@ class SamplingBatchInfo:
         return len(self.temperatures)
 
     def update_penalties(self):
+        if not self.penalizer_orchestrator:
+            return
+
         self.scaling_penalties = None
         self.linear_penalties = None
 
         for penalizer in self.penalizer_orchestrator.penalizers.values():
+            if not penalizer.is_prepared():
+                continue
+
             if isinstance(penalizer, penaltylib.BatchedRepetitionPenalizer):
-                if penalizer.is_prepared():
-                    self.scaling_penalties = penalizer.cumulated_repetition_penalties
+                self.scaling_penalties = penalizer.cumulated_repetition_penalties
             else:
-                if penalizer.is_prepared():
-                    if self.linear_penalties is None:
-                        bs = self.penalizer_orchestrator.batch.batch_size()
-                        self.linear_penalties = torch.zeros(
-                            (bs, self.vocab_size),
-                            dtype=torch.float32,
-                            device=self.device,
-                        )
-                    self.linear_penalties = penalizer.apply(self.linear_penalties)
+                if self.linear_penalties is None:
+                    bs = self.penalizer_orchestrator.batch.batch_size()
+                    self.linear_penalties = torch.zeros(
+                        (bs, self.vocab_size),
+                        dtype=torch.float32,
+                        device=self.device,
+                    )
+                self.linear_penalties = penalizer.apply(self.linear_penalties)
 
     def update_regex_vocab_mask(self):
         has_regex = self.regex_fsms and any(regex_fsm for regex_fsm in self.regex_fsms)
-
-        # Reset the vocab mask
-        self.vocab_mask = None
-
-        if has_regex:
-            self.vocab_mask = torch.zeros(
-                len(self.temperatures),
-                self.vocab_size,
-                dtype=torch.bool,
-                device=self.device,
-            )
-            for i, regex_fsm in enumerate(self.regex_fsms):
-                if regex_fsm is not None:
-                    self.vocab_mask[i].fill_(1)
-                    self.vocab_mask[i][
-                        regex_fsm.get_next_instruction(self.regex_fsm_states[i]).tokens
-                    ] = 0
+        if not has_regex:
+            self.vocab_mask = None
+            return
+
+        self.vocab_mask = torch.zeros(
+            len(self.temperatures),
+            self.vocab_size,
+            dtype=torch.bool,
+            device=self.device,
+        )
+        for i, regex_fsm in enumerate(self.regex_fsms):
+            if regex_fsm is not None:
+                self.vocab_mask[i].fill_(1)
+                self.vocab_mask[i][
+                    regex_fsm.get_next_instruction(self.regex_fsm_states[i]).tokens
+                ] = 0
 
     def filter_batch(self, unfinished_indices: List[int], new_indices: torch.Tensor):
-        self.penalizer_orchestrator.filter(unfinished_indices, new_indices)
+        if self.penalizer_orchestrator:
+            self.penalizer_orchestrator.filter(unfinished_indices, new_indices)
 
         for item in [
             "temperatures",
@@ -175,7 +195,8 @@ class SamplingBatchInfo:
         return None
 
     def merge_batch(self, other: "SamplingBatchInfo"):
-        self.penalizer_orchestrator.merge(other.penalizer_orchestrator)
+        if self.penalizer_orchestrator:
+            self.penalizer_orchestrator.merge(other.penalizer_orchestrator)
 
         for item in [
             "temperatures",
@@ -187,6 +208,19 @@ class SamplingBatchInfo:
             other_val = getattr(other, item, None)
             setattr(self, item, torch.concat([self_val, other_val]))
 
+        self.is_all_greedy = self.is_all_greedy and other.is_all_greedy
         self.logit_bias = SamplingBatchInfo.merge_bias_tensor(
             self.logit_bias, other.logit_bias, len(self), len(other), self.device
         )
+
+    def copy(self):
+        return SamplingBatchInfo(
+            temperatures=self.temperatures,
+            top_ps=self.top_ps,
+            top_ks=self.top_ks,
+            min_ps=self.min_ps,
+            is_all_greedy=self.is_all_greedy,
+            need_min_p_sampling=self.need_min_p_sampling,
+            vocab_size=self.vocab_size,
+            device=self.device,
+        )

sglang/srt/sampling/sampling_params.py

@@ -40,6 +40,7 @@ class SamplingParams:
         regex: Optional[str] = None,
         n: int = 1,
         json_schema: Optional[str] = None,
+        no_stop_trim: bool = False,
     ) -> None:
         self.temperature = temperature
         self.top_p = top_p
@@ -60,6 +61,7 @@ class SamplingParams:
         self.regex = regex
         self.n = n
         self.json_schema = json_schema
+        self.no_stop_trim = no_stop_trim
 
         # Process some special cases
         if self.temperature < _SAMPLING_EPS: