sglang 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

sglang/srt/models/qwen2_moe.py

@@ -28,9 +28,7 @@ from vllm.distributed import (
     get_tensor_model_parallel_world_size,
     tensor_model_parallel_all_reduce,
 )
-from vllm.model_executor.layers.activation import SiluAndMul
 from vllm.model_executor.layers.fused_moe import FusedMoE
-from vllm.model_executor.layers.layernorm import RMSNorm
 from vllm.model_executor.layers.linear import (
     MergedColumnParallelLinear,
     QKVParallelLinear,
@@ -46,9 +44,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
     VocabParallelEmbedding,
 )
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.model_executor.sampling_metadata import SamplingMetadata
-from vllm.sequence import IntermediateTensors, SamplerOutput
 
+from sglang.srt.layers.activation import SiluAndMul
+from sglang.srt.layers.layernorm import RMSNorm
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
 from sglang.srt.model_executor.forward_batch_info import InputMetadata
@@ -368,7 +366,6 @@ class Qwen2MoeForCausalLM(nn.Module):
             config.vocab_size, config.hidden_size, quant_config=quant_config
         )
         self.logits_processor = LogitsProcessor(config)
-        self.sampler = Sampler()
 
     @torch.no_grad()
     def forward(
@@ -394,14 +391,6 @@ class Qwen2MoeForCausalLM(nn.Module):
         )
         return logits
 
-    def sample(
-        self,
-        logits: Optional[torch.Tensor],
-        sampling_metadata: SamplingMetadata,
-    ) -> Optional[SamplerOutput]:
-        next_tokens = self.sampler(logits, sampling_metadata)
-        return next_tokens
-
     def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
         stacked_params_mapping = [
             # (param_name, shard_name, shard_id)

sglang/srt/models/stablelm.py

@@ -24,7 +24,6 @@ from torch import nn
 from transformers import PretrainedConfig
 from vllm.config import CacheConfig
 from vllm.distributed import get_tensor_model_parallel_world_size
-from vllm.model_executor.layers.activation import SiluAndMul
 from vllm.model_executor.layers.linear import (
     MergedColumnParallelLinear,
     QKVParallelLinear,
@@ -38,6 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
 )
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
+from sglang.srt.layers.activation import SiluAndMul
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
 from sglang.srt.model_executor.forward_batch_info import InputMetadata

sglang/srt/openai_api/adapter.py

@@ -34,7 +34,7 @@ from sglang.srt.conversation import (
     generate_chat_conv,
     register_conv_template,
 )
-from sglang.srt.managers.io_struct import GenerateReqInput
+from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
 from sglang.srt.openai_api.protocol import (
     BatchRequest,
     BatchResponse,
@@ -52,6 +52,9 @@ from sglang.srt.openai_api.protocol import (
     CompletionResponseStreamChoice,
     CompletionStreamResponse,
     DeltaMessage,
+    EmbeddingObject,
+    EmbeddingRequest,
+    EmbeddingResponse,
     ErrorResponse,
     FileDeleteResponse,
     FileRequest,
@@ -74,7 +77,7 @@ class FileMetadata:
 batch_storage: Dict[str, BatchResponse] = {}
 file_id_request: Dict[str, FileMetadata] = {}
 file_id_response: Dict[str, FileResponse] = {}
-# map file id to file path in SGlang backend
+# map file id to file path in SGLang backend
 file_id_storage: Dict[str, str] = {}
 
 
@@ -82,6 +85,19 @@ file_id_storage: Dict[str, str] = {}
 storage_dir = None
 
 
+def format_finish_reason(finish_reason) -> Optional[str]:
+    if finish_reason.startswith("None"):
+        return None
+    elif finish_reason.startswith("FINISH_MATCHED"):
+        return "stop"
+    elif finish_reason.startswith("FINISH_LENGTH"):
+        return "length"
+    elif finish_reason.startswith("FINISH_ABORT"):
+        return "abort"
+    else:
+        return "unknown"
+
+
 def create_error_response(
     message: str,
     err_type: str = "BadRequestError",
@@ -101,7 +117,7 @@ def create_streaming_error_response(
     return json_str
 
 
-def load_chat_template_for_openai_api(chat_template_arg):
+def load_chat_template_for_openai_api(tokenizer_manager, chat_template_arg):
     global chat_template_name
 
     print(f"Use chat template: {chat_template_arg}")
@@ -111,27 +127,38 @@ def load_chat_template_for_openai_api(chat_template_arg):
                 f"Chat template {chat_template_arg} is not a built-in template name "
                 "or a valid chat template file path."
             )
-        with open(chat_template_arg, "r") as filep:
-            template = json.load(filep)
-            try:
-                sep_style = SeparatorStyle[template["sep_style"]]
-            except KeyError:
-                raise ValueError(
-                    f"Unknown separator style: {template['sep_style']}"
-                ) from None
-            register_conv_template(
-                Conversation(
-                    name=template["name"],
-                    system_template=template["system"] + "\n{system_message}",
-                    system_message=template.get("system_message", ""),
-                    roles=(template["user"], template["assistant"]),
-                    sep_style=sep_style,
-                    sep=template.get("sep", "\n"),
-                    stop_str=template["stop_str"],
-                ),
-                override=True,
+        if chat_template_arg.endswith(".jinja"):
+            with open(chat_template_arg, "r") as f:
+                chat_template = "".join(f.readlines()).strip("\n")
+            tokenizer_manager.tokenizer.chat_template = chat_template.replace(
+                "\\n", "\n"
             )
-        chat_template_name = template["name"]
+            chat_template_name = None
+        else:
+            assert chat_template_arg.endswith(
+                ".json"
+            ), "unrecognized format of chat template file"
+            with open(chat_template_arg, "r") as filep:
+                template = json.load(filep)
+                try:
+                    sep_style = SeparatorStyle[template["sep_style"]]
+                except KeyError:
+                    raise ValueError(
+                        f"Unknown separator style: {template['sep_style']}"
+                    ) from None
+                register_conv_template(
+                    Conversation(
+                        name=template["name"],
+                        system_template=template["system"] + "\n{system_message}",
+                        system_message=template.get("system_message", ""),
+                        roles=(template["user"], template["assistant"]),
+                        sep_style=sep_style,
+                        sep=template.get("sep", "\n"),
+                        stop_str=template["stop_str"],
+                    ),
+                    override=True,
+                )
+            chat_template_name = template["name"]
     else:
         chat_template_name = chat_template_arg
 
@@ -319,7 +346,7 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
         }
 
     except Exception as e:
-        print("error in SGlang:", e)
+        print("error in SGLang:", e)
         # Update batch status to "failed"
         retrieve_batch = batch_storage[batch_id]
         retrieve_batch.status = "failed"
@@ -357,7 +384,6 @@ async def v1_retrieve_file_content(file_id: str):
 
 
 def v1_generate_request(all_requests):
-
     prompts = []
     sampling_params_list = []
     return_logprobs = []
@@ -378,10 +404,13 @@ def v1_generate_request(all_requests):
             {
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
+                "min_new_tokens": request.min_tokens,
                 "stop": request.stop,
+                "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
+                "repetition_penalty": request.repetition_penalty,
                 "regex": request.regex,
                 "n": request.n,
                 "ignore_eos": request.ignore_eos,
@@ -485,14 +514,18 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
                 "index": 0,
                 "text": text,
                 "logprobs": logprobs,
-                "finish_reason": ret_item["meta_info"]["finish_reason"],
+                "finish_reason": format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             }
         else:
             choice_data = CompletionResponseChoice(
                 index=idx,
                 text=text,
                 logprobs=logprobs,
-                finish_reason=ret_item["meta_info"]["finish_reason"],
+                finish_reason=format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             )
 
         choices.append(choice_data)
@@ -607,20 +640,34 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                         index=0,
                         text=delta,
                         logprobs=logprobs,
-                        finish_reason=content["meta_info"]["finish_reason"],
+                        finish_reason=format_finish_reason(
+                            content["meta_info"]["finish_reason"]
+                        ),
                     )
                     chunk = CompletionStreamResponse(
                         id=content["meta_info"]["id"],
                         object="text_completion",
                         choices=[choice_data],
                         model=request.model,
-                        usage=UsageInfo(
-                            prompt_tokens=prompt_tokens,
-                            completion_tokens=completion_tokens,
-                            total_tokens=prompt_tokens + completion_tokens,
-                        ),
                     )
                     yield f"data: {chunk.model_dump_json()}\n\n"
+                if request.stream_options and request.stream_options.include_usage:
+                    usage = UsageInfo(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=prompt_tokens + completion_tokens,
+                    )
+
+                    final_usage_chunk = CompletionStreamResponse(
+                        id=str(uuid.uuid4().hex),
+                        choices=[],
+                        model=request.model,
+                        usage=usage,
+                    )
+                    final_usage_data = final_usage_chunk.model_dump_json(
+                        exclude_unset=True, exclude_none=True
+                    )
+                    yield f"data: {final_usage_data}\n\n"
             except ValueError as e:
                 error = create_streaming_error_response(str(e))
                 yield f"data: {error}\n\n"
@@ -648,7 +695,6 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
 
 
 def v1_chat_generate_request(all_requests, tokenizer_manager):
-
     input_ids = []
     sampling_params_list = []
     image_data_list = []
@@ -691,10 +737,13 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
             {
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
+                "min_new_tokens": request.min_tokens,
                 "stop": stop,
+                "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
+                "repetition_penalty": request.repetition_penalty,
                 "regex": request.regex,
                 "n": request.n,
             }
@@ -776,14 +825,18 @@ def v1_chat_generate_response(request, ret, to_file=False):
                 "index": 0,
                 "message": {"role": "assistant", "content": ret_item["text"]},
                 "logprobs": choice_logprobs,
-                "finish_reason": ret_item["meta_info"]["finish_reason"],
+                "finish_reason": format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             }
         else:
             choice_data = ChatCompletionResponseChoice(
                 index=idx,
                 message=ChatMessage(role="assistant", content=ret_item["text"]),
                 logprobs=choice_logprobs,
-                finish_reason=ret_item["meta_info"]["finish_reason"],
+                finish_reason=format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             )
 
         choices.append(choice_data)
@@ -900,18 +953,15 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         choice_data = ChatCompletionResponseStreamChoice(
                             index=0,
                             delta=DeltaMessage(role="assistant"),
-                            finish_reason=content["meta_info"]["finish_reason"],
+                            finish_reason=format_finish_reason(
+                                content["meta_info"]["finish_reason"]
+                            ),
                             logprobs=choice_logprobs,
                         )
                         chunk = ChatCompletionStreamResponse(
                             id=content["meta_info"]["id"],
                             choices=[choice_data],
                             model=request.model,
-                            usage=UsageInfo(
-                                prompt_tokens=prompt_tokens,
-                                completion_tokens=completion_tokens,
-                                total_tokens=prompt_tokens + completion_tokens,
-                            ),
                         )
                         yield f"data: {chunk.model_dump_json()}\n\n"
 
@@ -921,20 +971,34 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                     choice_data = ChatCompletionResponseStreamChoice(
                         index=0,
                         delta=DeltaMessage(content=delta),
-                        finish_reason=content["meta_info"]["finish_reason"],
+                        finish_reason=format_finish_reason(
+                            content["meta_info"]["finish_reason"]
+                        ),
                         logprobs=choice_logprobs,
                     )
                     chunk = ChatCompletionStreamResponse(
                         id=content["meta_info"]["id"],
                         choices=[choice_data],
                         model=request.model,
-                        usage=UsageInfo(
-                            prompt_tokens=prompt_tokens,
-                            completion_tokens=completion_tokens,
-                            total_tokens=prompt_tokens + completion_tokens,
-                        ),
                     )
                     yield f"data: {chunk.model_dump_json()}\n\n"
+                if request.stream_options and request.stream_options.include_usage:
+                    usage = UsageInfo(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=prompt_tokens + completion_tokens,
+                    )
+
+                    final_usage_chunk = ChatCompletionStreamResponse(
+                        id=str(uuid.uuid4().hex),
+                        choices=[],
+                        model=request.model,
+                        usage=usage,
+                    )
+                    final_usage_data = final_usage_chunk.model_dump_json(
+                        exclude_unset=True, exclude_none=True
+                    )
+                    yield f"data: {final_usage_data}\n\n"
             except ValueError as e:
                 error = create_streaming_error_response(str(e))
                 yield f"data: {error}\n\n"
@@ -961,6 +1025,81 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
     return response
 
 
+def v1_embedding_request(all_requests, tokenizer_manager):
+    prompts = []
+    sampling_params_list = []
+    first_prompt_type = type(all_requests[0].input)
+
+    for request in all_requests:
+        prompt = request.input
+        assert (
+            type(prompt) == first_prompt_type
+        ), "All prompts must be of the same type in file input settings"
+        prompts.append(prompt)
+
+    if len(all_requests) == 1:
+        prompt = prompts[0]
+        if isinstance(prompt, str) or isinstance(prompt[0], str):
+            prompt_kwargs = {"text": prompt}
+        else:
+            prompt_kwargs = {"input_ids": prompt}
+    else:
+        if isinstance(prompts[0], str) or isinstance(propmt[0][0], str):
+            prompt_kwargs = {"text": prompts}
+        else:
+            prompt_kwargs = {"input_ids": prompts}
+
+    adapted_request = EmbeddingReqInput(
+        **prompt_kwargs,
+    )
+
+    if len(all_requests) == 1:
+        return adapted_request, all_requests[0]
+    return adapted_request, all_requests
+
+
+def v1_embedding_response(ret, model_path, to_file=False):
+    embedding_objects = []
+    prompt_tokens = 0
+    for idx, ret_item in enumerate(ret):
+        embedding_objects.append(
+            EmbeddingObject(
+                embedding=ret[idx]["embedding"],
+                index=idx,
+            )
+        )
+        prompt_tokens += ret[idx]["meta_info"]["prompt_tokens"]
+
+    return EmbeddingResponse(
+        data=embedding_objects,
+        model=model_path,
+        usage=UsageInfo(
+            prompt_tokens=prompt_tokens,
+            total_tokens=prompt_tokens,
+        ),
+    )
+
+
+async def v1_embeddings(tokenizer_manager, raw_request: Request):
+    request_json = await raw_request.json()
+    all_requests = [EmbeddingRequest(**request_json)]
+    adapted_request, request = v1_embedding_request(all_requests, tokenizer_manager)
+
+    try:
+        ret = await tokenizer_manager.generate_request(
+            adapted_request, raw_request
+        ).__anext__()
+    except ValueError as e:
+        return create_error_response(str(e))
+
+    if not isinstance(ret, list):
+        ret = [ret]
+
+    response = v1_embedding_response(ret, tokenizer_manager.model_path)
+
+    return response
+
+
 def to_openai_style_logprobs(
     input_token_logprobs=None,
     output_token_logprobs=None,

sglang/srt/openai_api/protocol.py

@@ -78,6 +78,10 @@ class UsageInfo(BaseModel):
     completion_tokens: Optional[int] = 0
 
 
+class StreamOptions(BaseModel):
+    include_usage: Optional[bool] = False
+
+
 class FileRequest(BaseModel):
     # https://platform.openai.com/docs/api-reference/files/create
     file: bytes  # The File object (not file name) to be uploaded
@@ -149,6 +153,7 @@ class CompletionRequest(BaseModel):
     seed: Optional[int] = None
     stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
     stream: Optional[bool] = False
+    stream_options: Optional[StreamOptions] = None
     suffix: Optional[str] = None
     temperature: Optional[float] = 1.0
     top_p: Optional[float] = 1.0
@@ -157,6 +162,9 @@ class CompletionRequest(BaseModel):
     # Extra parameters for SRT backend only and will be ignored by OpenAI models.
     regex: Optional[str] = None
     ignore_eos: Optional[bool] = False
+    min_tokens: Optional[int] = 0
+    repetition_penalty: Optional[float] = 1.0
+    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
 
 
 class CompletionResponseChoice(BaseModel):
@@ -188,7 +196,7 @@ class CompletionStreamResponse(BaseModel):
     created: int = Field(default_factory=lambda: int(time.time()))
     model: str
     choices: List[CompletionResponseStreamChoice]
-    usage: UsageInfo
+    usage: Optional[UsageInfo] = None
 
 
 class ChatCompletionMessageGenericParam(BaseModel):
@@ -247,12 +255,16 @@ class ChatCompletionRequest(BaseModel):
     seed: Optional[int] = None
     stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
     stream: Optional[bool] = False
+    stream_options: Optional[StreamOptions] = None
     temperature: Optional[float] = 0.7
     top_p: Optional[float] = 1.0
     user: Optional[str] = None
 
     # Extra parameters for SRT backend only and will be ignored by OpenAI models.
     regex: Optional[str] = None
+    min_tokens: Optional[int] = 0
+    repetition_penalty: Optional[float] = 1.0
+    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
 
 
 class ChatMessage(BaseModel):
@@ -294,3 +306,27 @@ class ChatCompletionStreamResponse(BaseModel):
     created: int = Field(default_factory=lambda: int(time.time()))
     model: str
     choices: List[ChatCompletionResponseStreamChoice]
+    usage: Optional[UsageInfo] = None
+
+
+class EmbeddingRequest(BaseModel):
+    # Ordered by official OpenAI API documentation
+    # https://platform.openai.com/docs/api-reference/embeddings/create
+    input: Union[List[int], List[List[int]], str, List[str]]
+    model: str
+    encoding_format: str = "float"
+    dimensions: int = None
+    user: Optional[str] = None
+
+
+class EmbeddingObject(BaseModel):
+    embedding: List[float]
+    index: int
+    object: str = "embedding"
+
+
+class EmbeddingResponse(BaseModel):
+    data: List[EmbeddingObject]
+    model: str
+    object: str = "list"
+    usage: Optional[UsageInfo] = None

sglang/srt/sampling/penaltylib/__init__.py

@@ -0,0 +1,13 @@
+from .orchestrator import BatchedPenalizerOrchestrator
+from .penalizers.frequency_penalty import BatchedFrequencyPenalizer
+from .penalizers.min_new_tokens import BatchedMinNewTokensPenalizer
+from .penalizers.presence_penalty import BatchedPresencePenalizer
+from .penalizers.repetition_penalty import BatchedRepetitionPenalizer
+
+__all__ = [
+    "BatchedFrequencyPenalizer",
+    "BatchedMinNewTokensPenalizer",
+    "BatchedPresencePenalizer",
+    "BatchedRepetitionPenalizer",
+    "BatchedPenalizerOrchestrator",
+]