sglang 0.4.4.post1__py3-none-any.whl → 0.4.4.post2__py3-none-any.whl

sglang/srt/openai_api/adapter.py

@@ -20,19 +20,16 @@ import os
 import time
 import uuid
 from http import HTTPStatus
-from typing import Dict, List
+from typing import Any, Dict, List, Set
 
 from fastapi import HTTPException, Request, UploadFile
 from fastapi.responses import ORJSONResponse, StreamingResponse
 from pydantic import ValidationError
 
-try:
-    from outlines.fsm.json_schema import convert_json_schema_to_str
-except ImportError:
-    # Before outlines 0.0.47, convert_json_schema_to_str is under
-    # outlines.integrations.utils
-    from outlines.integrations.utils import convert_json_schema_to_str
-
+from sglang.srt.code_completion_parser import (
+    generate_completion_prompt_from_request,
+    is_completion_template_defined,
+)
 from sglang.srt.conversation import (
     Conversation,
     SeparatorStyle,
@@ -41,7 +38,7 @@ from sglang.srt.conversation import (
     generate_embedding_convs,
     register_conv_template,
 )
-from sglang.srt.function_call_parser import TOOLS_TAG_LIST, FunctionCallParser
+from sglang.srt.function_call_parser import FunctionCallParser
 from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
 from sglang.srt.openai_api.protocol import (
     BatchRequest,
@@ -75,7 +72,7 @@ from sglang.srt.openai_api.protocol import (
     UsageInfo,
 )
 from sglang.srt.reasoning_parser import ReasoningParser
-from sglang.utils import get_exception_traceback
+from sglang.utils import convert_json_schema_to_str, get_exception_traceback
 
 logger = logging.getLogger(__name__)
 
@@ -310,6 +307,7 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
             )
 
         try:
+            created = int(time.time())
             ret = await tokenizer_manager.generate_request(adapted_request).__anext__()
             if not isinstance(ret, list):
                 ret = [ret]
@@ -317,13 +315,19 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
                 responses = v1_chat_generate_response(
                     request,
                     ret,
+                    created,
                     to_file=True,
                     cache_report=tokenizer_manager.server_args.enable_cache_report,
                     tool_call_parser=tokenizer_manager.server_args.tool_call_parser,
                 )
             else:
                 responses = v1_generate_response(
-                    request, ret, tokenizer_manager, to_file=True
+                    request,
+                    ret,
+                    tokenizer_manager,
+                    created,
+                    to_file=True,
+                    cache_report=tokenizer_manager.server_args.enable_cache_report,
                 )
 
         except Exception as e:
@@ -504,7 +508,11 @@ def v1_generate_request(
                 "To compute logprobs of input prompt, please use the native /generate API."
             )
 
-        prompts.append(request.prompt)
+        prompt = request.prompt
+        if is_completion_template_defined():
+            prompt = generate_completion_prompt_from_request(request)
+        prompts.append(prompt)
+
         lora_paths.append(request.lora_path)
         if request.echo and request.logprobs:
             current_logprob_start_len = 0
@@ -569,7 +577,9 @@ def v1_generate_request(
     return adapted_request, all_requests if len(all_requests) > 1 else all_requests[0]
 
 
-def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
+def v1_generate_response(
+    request, ret, tokenizer_manager, created, to_file=False, cache_report=False
+):
     choices = []
     echo = False
 
@@ -667,7 +677,7 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
                     # remain the same but if needed we can change that
                     "id": ret[i]["meta_info"]["id"],
                     "object": "text_completion",
-                    "created": int(time.time()),
+                    "created": created,
                     "model": request[i].model,
                     "choices": choice,
                     "usage": {
@@ -686,14 +696,19 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
             ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
         )
         completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
+        cached_tokens = sum(item["meta_info"].get("cached_tokens", 0) for item in ret)
         response = CompletionResponse(
             id=ret[0]["meta_info"]["id"],
             model=request.model,
+            created=created,
             choices=choices,
             usage=UsageInfo(
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=prompt_tokens + completion_tokens,
+                prompt_tokens_details=(
+                    {"cached_tokens": cached_tokens} if cache_report else None
+                ),
             ),
         )
     return response
@@ -702,6 +717,7 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
 async def v1_completions(tokenizer_manager, raw_request: Request):
     request_json = await raw_request.json()
     all_requests = [CompletionRequest(**request_json)]
+    created = int(time.time())
     adapted_request, request = v1_generate_request(all_requests)
 
     if adapted_request.stream:
@@ -711,6 +727,8 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
             n_prev_tokens = {}
             prompt_tokens = {}
             completion_tokens = {}
+            cached_tokens = {}
+
             try:
                 async for content in tokenizer_manager.generate_request(
                     adapted_request, raw_request
@@ -723,6 +741,7 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                     text = content["text"]
                     prompt_tokens[index] = content["meta_info"]["prompt_tokens"]
                     completion_tokens[index] = content["meta_info"]["completion_tokens"]
+                    cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
 
                     if not stream_buffer:  # The first chunk
                         if request.echo:
@@ -795,6 +814,7 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                     )
                     chunk = CompletionStreamResponse(
                         id=content["meta_info"]["id"],
+                        created=created,
                         object="text_completion",
                         choices=[choice_data],
                         model=request.model,
@@ -813,14 +833,24 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                     total_completion_tokens = sum(
                         tokens for tokens in completion_tokens.values()
                     )
+                    cache_report = tokenizer_manager.server_args.enable_cache_report
+                    if cache_report:
+                        cached_tokens_sum = sum(
+                            tokens for tokens in cached_tokens.values()
+                        )
+                        prompt_tokens_details = {"cached_tokens": cached_tokens_sum}
+                    else:
+                        prompt_tokens_details = None
                     usage = UsageInfo(
                         prompt_tokens=total_prompt_tokens,
                         completion_tokens=total_completion_tokens,
                         total_tokens=total_prompt_tokens + total_completion_tokens,
+                        prompt_tokens_details=prompt_tokens_details,
                     )
 
                     final_usage_chunk = CompletionStreamResponse(
                         id=content["meta_info"]["id"],
+                        created=created,
                         choices=[],
                         model=request.model,
                         usage=usage,
@@ -851,7 +881,13 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
     if not isinstance(ret, list):
         ret = [ret]
 
-    response = v1_generate_response(request, ret, tokenizer_manager)
+    response = v1_generate_response(
+        request,
+        ret,
+        tokenizer_manager,
+        created,
+        cache_report=tokenizer_manager.server_args.enable_cache_report,
+    )
     return response
 
 
@@ -863,6 +899,7 @@ def v1_chat_generate_request(
     input_ids = []
     sampling_params_list = []
     image_data_list = []
+    audio_data_list = []
     return_logprobs = []
     logprob_start_lens = []
     top_logprobs_nums = []
@@ -876,7 +913,9 @@ def v1_chat_generate_request(
         #  - prompt: The full prompt string.
         #  - stop: Custom stop tokens.
         #  - image_data: None or a list of image strings (URLs or base64 strings).
+        #  - audio_data: None or a list of audio strings (URLs).
         #    None skips any image processing in GenerateReqInput.
+        strict_tag = None
         if not isinstance(request.messages, str):
             # Apply chat template and its stop strings.
             tools = None
@@ -891,6 +930,10 @@ def v1_chat_generate_request(
                 else:
                     tools = [item.function.model_dump() for item in request.tools]
 
+                tool_call_parser = tokenizer_manager.server_args.tool_call_parser
+                parser = FunctionCallParser(request.tools, tool_call_parser)
+                strict_tag = parser.get_structure_tag()
+
             if chat_template_name is None:
                 openai_compatible_messages = []
                 for message in request.messages:
@@ -920,7 +963,7 @@ def v1_chat_generate_request(
                     )
                 except:
                     #  This except branch will be triggered when the chosen model
-                    #  has a different tools input format that is not compatiable
+                    #  has a different tools input format that is not compatible
                     #  with openAI's apply_chat_template tool_call format, like Mistral.
                     tools = [t if "function" in t else {"function": t} for t in tools]
                     prompt_ids = tokenizer_manager.tokenizer.apply_chat_template(
@@ -940,11 +983,13 @@ def v1_chat_generate_request(
                     prompt_ids += encoded
                 stop = request.stop
                 image_data = None
+                audio_data = None
                 modalities = []
             else:
                 conv = generate_chat_conv(request, chat_template_name)
                 prompt = conv.get_prompt()
                 image_data = conv.image_data
+                audio_data = conv.audio_data
                 modalities = conv.modalities
                 stop = conv.stop_str or []
                 if request.stop:
@@ -958,6 +1003,7 @@ def v1_chat_generate_request(
             prompt_ids = request.messages
             stop = request.stop
             image_data = None
+            audio_data = None
             modalities = []
         input_ids.append(prompt_ids)
         return_logprobs.append(request.logprobs)
@@ -995,9 +1041,26 @@ def v1_chat_generate_request(
             sampling_params["structural_tag"] = convert_json_schema_to_str(
                 request.response_format.model_dump(by_alias=True)
             )
+
+        if strict_tag is not None:
+            if (
+                sampling_params.get("regex")
+                or sampling_params.get("ebnf")
+                or sampling_params.get("structural_tag")
+                or sampling_params.get("json_schema")
+            ):
+                logger.warning(
+                    "Constrained decoding is not compatible with tool calls."
+                )
+            else:
+                sampling_params["structural_tag"] = convert_json_schema_to_str(
+                    strict_tag.model_dump(by_alias=True)
+                )
+
         sampling_params_list.append(sampling_params)
 
         image_data_list.append(image_data)
+        audio_data_list.append(audio_data)
         modalities_list.append(modalities)
     if len(all_requests) == 1:
         if isinstance(input_ids[0], str):
@@ -1006,6 +1069,7 @@ def v1_chat_generate_request(
             prompt_kwargs = {"input_ids": input_ids[0]}
         sampling_params_list = sampling_params_list[0]
         image_data_list = image_data_list[0]
+        audio_data_list = audio_data_list[0]
         return_logprobs = return_logprobs[0]
         logprob_start_lens = logprob_start_lens[0]
         top_logprobs_nums = top_logprobs_nums[0]
@@ -1020,6 +1084,7 @@ def v1_chat_generate_request(
     adapted_request = GenerateReqInput(
         **prompt_kwargs,
         image_data=image_data_list,
+        audio_data=audio_data_list,
         sampling_params=sampling_params_list,
         return_logprob=return_logprobs,
         logprob_start_len=logprob_start_lens,
@@ -1037,6 +1102,7 @@ def v1_chat_generate_request(
 def v1_chat_generate_response(
     request,
     ret,
+    created,
     to_file=False,
     cache_report=False,
     tool_call_parser=None,
@@ -1122,7 +1188,7 @@ def v1_chat_generate_response(
                     finish_reason["type"] = "tool_calls"
                     finish_reason["matched"] = None
                 try:
-                    full_normal_text, call_info_list = parser.parse_non_stream(text)
+                    text, call_info_list = parser.parse_non_stream(text)
                     tool_calls = [
                         ToolCall(
                             id=str(call_info.tool_index),
@@ -1145,9 +1211,9 @@ def v1_chat_generate_response(
                 "index": 0,
                 "message": {
                     "role": "assistant",
-                    "content": text if tool_calls is None else None,
+                    "content": text if text else None,
                     "tool_calls": tool_calls,
-                    "reasoning_content": reasoning_text,
+                    "reasoning_content": reasoning_text if reasoning_text else None,
                 },
                 "logprobs": choice_logprobs.model_dump() if choice_logprobs else None,
                 "finish_reason": (finish_reason["type"] if finish_reason else ""),
@@ -1162,9 +1228,9 @@ def v1_chat_generate_response(
                 index=idx,
                 message=ChatMessage(
                     role="assistant",
-                    content=text if tool_calls is None else None,
+                    content=text if text else None,
                     tool_calls=tool_calls,
-                    reasoning_content=reasoning_text,
+                    reasoning_content=reasoning_text if reasoning_text else None,
                 ),
                 logprobs=choice_logprobs,
                 finish_reason=(finish_reason["type"] if finish_reason else ""),
@@ -1188,7 +1254,7 @@ def v1_chat_generate_response(
                     # remain the same but if needed we can change that
                     "id": ret[i]["meta_info"]["id"],
                     "object": "chat.completion",
-                    "created": int(time.time()),
+                    "created": created,
                     "model": request[i].model,
                     "choices": choice,
                     "usage": {
@@ -1210,6 +1276,7 @@ def v1_chat_generate_response(
         cached_tokens = sum(item["meta_info"].get("cached_tokens", 0) for item in ret)
         response = ChatCompletionResponse(
             id=ret[0]["meta_info"]["id"],
+            created=created,
             model=request.model,
             choices=choices,
             usage=UsageInfo(
@@ -1224,9 +1291,12 @@ def v1_chat_generate_response(
         return response
 
 
-async def v1_chat_completions(tokenizer_manager, raw_request: Request):
+async def v1_chat_completions(
+    tokenizer_manager, raw_request: Request, cache_report=False
+):
     request_json = await raw_request.json()
     all_requests = [ChatCompletionRequest(**request_json)]
+    created = int(time.time())
     adapted_request, request = v1_chat_generate_request(all_requests, tokenizer_manager)
 
     if adapted_request.stream:
@@ -1239,6 +1309,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
             n_prev_tokens = {}
             prompt_tokens = {}
             completion_tokens = {}
+            cached_tokens = {}
             try:
                 async for content in tokenizer_manager.generate_request(
                     adapted_request, raw_request
@@ -1252,6 +1323,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
 
                     prompt_tokens[index] = content["meta_info"]["prompt_tokens"]
                     completion_tokens[index] = content["meta_info"]["completion_tokens"]
+                    cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
                     if request.logprobs:
                         logprobs = to_openai_style_logprobs(
                             output_token_logprobs=content["meta_info"][
@@ -1309,9 +1381,11 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             tokenizer_manager.server_args.reasoning_parser
                             and request.separate_reasoning
                         ):
-                            delta = DeltaMessage(role="assistant", reasoning_content="")
+                            delta = DeltaMessage(
+                                role="assistant", reasoning_content=None
+                            )
                         else:
-                            delta = DeltaMessage(role="assistant", content="")
+                            delta = DeltaMessage(role="assistant", content=None)
                         choice_data = ChatCompletionResponseStreamChoice(
                             index=index,
                             delta=delta,
@@ -1329,6 +1403,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         )
                         chunk = ChatCompletionStreamResponse(
                             id=content["meta_info"]["id"],
+                            created=created,
                             choices=[choice_data],
                             model=request.model,
                         )
@@ -1354,7 +1429,11 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         if reasoning_text:
                             choice_data = ChatCompletionResponseStreamChoice(
                                 index=index,
-                                delta=DeltaMessage(reasoning_content=reasoning_text),
+                                delta=DeltaMessage(
+                                    reasoning_content=(
+                                        reasoning_text if reasoning_text else None
+                                    )
+                                ),
                                 finish_reason=(
                                     None
                                     if finish_reason_type
@@ -1364,6 +1443,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             )
                             chunk = ChatCompletionStreamResponse(
                                 id=content["meta_info"]["id"],
+                                created=created,
                                 choices=[choice_data],
                                 model=request.model,
                             )
@@ -1388,7 +1468,9 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         if normal_text:
                             choice_data = ChatCompletionResponseStreamChoice(
                                 index=index,
-                                delta=DeltaMessage(content=normal_text),
+                                delta=DeltaMessage(
+                                    content=normal_text if normal_text else None
+                                ),
                                 finish_reason=(
                                     None
                                     if finish_reason_type
@@ -1398,6 +1480,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             )
                             chunk = ChatCompletionStreamResponse(
                                 id=content["meta_info"]["id"],
+                                created=created,
                                 choices=[choice_data],
                                 model=request.model,
                             )
@@ -1448,6 +1531,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             )
                             chunk = ChatCompletionStreamResponse(
                                 id=content["meta_info"]["id"],
+                                created=created,
                                 choices=[choice_data],
                                 model=request.model,
                             )
@@ -1460,7 +1544,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         # No tool calls => just treat this as normal text
                         choice_data = ChatCompletionResponseStreamChoice(
                             index=index,
-                            delta=DeltaMessage(content=delta),
+                            delta=DeltaMessage(content=delta if delta else None),
                             finish_reason=(
                                 None
                                 if finish_reason_type and len(finish_reason_type) == 0
@@ -1475,6 +1559,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         )
                         chunk = ChatCompletionStreamResponse(
                             id=content["meta_info"]["id"],
+                            created=created,
                             choices=[choice_data],
                             model=request.model,
                         )
@@ -1490,14 +1575,24 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                     total_completion_tokens = sum(
                         tokens for tokens in completion_tokens.values()
                     )
+                    cache_report = tokenizer_manager.server_args.enable_cache_report
+                    if cache_report:
+                        cached_tokens_sum = sum(
+                            tokens for tokens in cached_tokens.values()
+                        )
+                        prompt_tokens_details = {"cached_tokens": cached_tokens_sum}
+                    else:
+                        prompt_tokens_details = None
                     usage = UsageInfo(
                         prompt_tokens=total_prompt_tokens,
                         completion_tokens=total_completion_tokens,
                         total_tokens=total_prompt_tokens + total_completion_tokens,
+                        prompt_tokens_details=prompt_tokens_details,
                     )
 
                     final_usage_chunk = ChatCompletionStreamResponse(
                         id=content["meta_info"]["id"],
+                        created=created,
                         choices=[],
                         model=request.model,
                         usage=usage,
@@ -1530,6 +1625,7 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
     response = v1_chat_generate_response(
         request,
         ret,
+        created,
         cache_report=tokenizer_manager.server_args.enable_cache_report,
         tool_call_parser=tokenizer_manager.server_args.tool_call_parser,
         reasoning_parser=tokenizer_manager.server_args.reasoning_parser,

sglang/srt/openai_api/protocol.py

@@ -16,7 +16,7 @@
 import time
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, root_validator
 from typing_extensions import Literal
 
 
@@ -227,14 +227,25 @@ class ChatCompletionMessageContentImageURL(BaseModel):
     detail: Optional[Literal["auto", "low", "high"]] = "auto"
 
 
+class ChatCompletionMessageContentAudioURL(BaseModel):
+    url: str
+
+
 class ChatCompletionMessageContentImagePart(BaseModel):
     type: Literal["image_url"]
     image_url: ChatCompletionMessageContentImageURL
     modalities: Optional[Literal["image", "multi-images", "video"]] = "image"
 
 
+class ChatCompletionMessageContentAudioPart(BaseModel):
+    type: Literal["audio_url"]
+    audio_url: ChatCompletionMessageContentAudioURL
+
+
 ChatCompletionMessageContentPart = Union[
-    ChatCompletionMessageContentTextPart, ChatCompletionMessageContentImagePart
+    ChatCompletionMessageContentTextPart,
+    ChatCompletionMessageContentImagePart,
+    ChatCompletionMessageContentAudioPart,
 ]
 
 
@@ -276,6 +287,7 @@ class Function(BaseModel):
     description: Optional[str] = Field(default=None, examples=[None])
     name: Optional[str] = None
     parameters: Optional[object] = None
+    strict: bool = False
 
 
 class Tool(BaseModel):
@@ -323,6 +335,15 @@ class ChatCompletionRequest(BaseModel):
         default="auto", examples=["none"]
     )  # noqa
 
+    @root_validator(pre=True)
+    def set_tool_choice_default(cls, values):
+        if values.get("tool_choice") is None:
+            if values.get("tools") is None:
+                values["tool_choice"] = "none"
+            else:
+                values["tool_choice"] = "auto"
+        return values
+
     # Extra parameters for SRT backend only and will be ignored by OpenAI models.
     top_k: int = -1
     min_p: float = 0.0

sglang/srt/sampling/sampling_batch_info.py

@@ -306,7 +306,7 @@ class SamplingBatchInfo:
         ]:
             self_val = getattr(self, item, None)
             other_val = getattr(other, item, None)
-            setattr(self, item, torch.concat([self_val, other_val]))
+            setattr(self, item, torch.cat([self_val, other_val]))
 
         self.is_all_greedy |= other.is_all_greedy
         self.need_min_p_sampling |= other.need_min_p_sampling

sglang/srt/sampling/sampling_params.py

@@ -77,7 +77,7 @@ class SamplingParams:
         self.custom_params = custom_params
 
         # Process some special cases
-        if self.temperature < _SAMPLING_EPS:
+        if 0 <= self.temperature < _SAMPLING_EPS:
             # top_k = 1 means greedy sampling
             self.temperature = 1.0
             self.top_k = 1
@@ -93,9 +93,9 @@ class SamplingParams:
             raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.")
         if not 0.0 <= self.min_p <= 1.0:
             raise ValueError(f"min_p must be in [0, 1], got {self.min_p}.")
-        if self.top_k < -1 or self.top_k == 0:
+        if self.top_k < 1 or self.top_k == -1:
             raise ValueError(
-                f"top_k must be -1 (disable), or at least 1, " f"got {self.top_k}."
+                f"top_k must be -1 (disable) or at least 1, got {self.top_k}."
             )
         if not -2.0 <= self.frequency_penalty <= 2.0:
             raise ValueError(
@@ -108,12 +108,12 @@ class SamplingParams:
             )
         if not 0.0 <= self.repetition_penalty <= 2.0:
             raise ValueError(
-                "repetition_penalty must be in (0, 2], got "
+                "repetition_penalty must be in [0, 2], got "
                 f"{self.repetition_penalty}."
             )
         if not 0 <= self.min_new_tokens:
             raise ValueError(
-                f"min_new_tokens must be in (0, max_new_tokens], got "
+                f"min_new_tokens must be in [0, max_new_tokens], got "
                 f"{self.min_new_tokens}."
             )
         if self.max_new_tokens is not None:
@@ -123,7 +123,7 @@ class SamplingParams:
                 )
             if not self.min_new_tokens <= self.max_new_tokens:
                 raise ValueError(
-                    f"min_new_tokens must be in (0, max_new_tokens({self.max_new_tokens})], got "
+                    f"min_new_tokens must be in [0, max_new_tokens({self.max_new_tokens})], got "
                     f"{self.min_new_tokens}."
                 )
         grammars = [