sglang 0.2.9.post1__py3-none-any.whl → 0.2.11__py3-none-any.whl

sglang/srt/models/gpt_bigcode.py

@@ -35,7 +35,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.managers.schedule_batch import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class GPTBigCodeAttention(nn.Module):

sglang/srt/models/grok.py

@@ -52,7 +52,7 @@ from vllm.utils import print_warning_once
 from sglang.srt.layers.fused_moe import fused_moe
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 use_fused = True
 

sglang/srt/models/internlm2.py

@@ -40,7 +40,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class InternLM2MLP(nn.Module):

sglang/srt/models/llama2.py

@@ -41,7 +41,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class LlamaMLP(nn.Module):

sglang/srt/models/llama_classification.py

@@ -25,7 +25,7 @@ from vllm.model_executor.layers.quantization.base_config import QuantizationConf
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitProcessorOutput
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 from sglang.srt.models.llama2 import LlamaModel
 
 

sglang/srt/models/llava.py

@@ -32,13 +32,12 @@ from vllm.config import CacheConfig
 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
-from sglang.srt.managers.schedule_batch import ForwardMode
 from sglang.srt.mm_utils import (
     get_anyres_image_grid_shape,
     unpad_image,
     unpad_image_shape,
 )
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import ForwardMode, InputMetadata
 from sglang.srt.models.llama2 import LlamaForCausalLM
 from sglang.srt.models.mistral import MistralForCausalLM
 from sglang.srt.models.qwen2 import Qwen2ForCausalLM

sglang/srt/models/llavavid.py

@@ -26,13 +26,12 @@ from vllm.config import CacheConfig
 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
-from sglang.srt.managers.schedule_batch import ForwardMode
 from sglang.srt.mm_utils import (
     get_anyres_image_grid_shape,
     unpad_image,
     unpad_image_shape,
 )
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import ForwardMode, InputMetadata
 from sglang.srt.models.llama2 import LlamaForCausalLM
 
 

sglang/srt/models/minicpm.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MiniCPMMLP(nn.Module):

sglang/srt/models/mixtral.py

@@ -50,7 +50,7 @@ from vllm.utils import print_warning_once
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MixtralMoE(nn.Module):

sglang/srt/models/mixtral_quant.py

@@ -45,7 +45,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MixtralMLP(nn.Module):

sglang/srt/models/qwen.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class QWenMLP(nn.Module):

sglang/srt/models/qwen2.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 Qwen2Config = None
 

sglang/srt/models/qwen2_moe.py

@@ -51,7 +51,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class Qwen2MoeMLP(nn.Module):

sglang/srt/models/stablelm.py

@@ -40,7 +40,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class StablelmMLP(nn.Module):

sglang/srt/openai_api/adapter.py

@@ -53,6 +53,7 @@ from sglang.srt.openai_api.protocol import (
     CompletionStreamResponse,
     DeltaMessage,
     ErrorResponse,
+    FileDeleteResponse,
     FileRequest,
     FileResponse,
     LogProbs,
@@ -174,6 +175,20 @@ async def v1_files_create(file: UploadFile, purpose: str, file_storage_pth: str
         return {"error": "Invalid input", "details": e.errors()}
 
 
+async def v1_delete_file(file_id: str):
+    # Retrieve the file job from the in-memory storage
+    file_response = file_id_response.get(file_id)
+    if file_response is None:
+        raise HTTPException(status_code=404, detail="File not found")
+    file_path = file_id_storage.get(file_id)
+    if file_path is None:
+        raise HTTPException(status_code=404, detail="File not found")
+    os.remove(file_path)
+    del file_id_response[file_id]
+    del file_id_storage[file_id]
+    return FileDeleteResponse(id=file_id, deleted=True)
+
+
 async def v1_batches(tokenizer_manager, raw_request: Request):
     try:
         body = await raw_request.json()
@@ -251,7 +266,9 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
             if end_point == "/v1/chat/completions":
                 responses = v1_chat_generate_response(request, ret, to_file=True)
             else:
-                responses = v1_generate_response(request, ret, to_file=True)
+                responses = v1_generate_response(
+                    request, ret, tokenizer_manager, to_file=True
+                )
 
         except Exception as e:
             error_json = {
@@ -285,6 +302,13 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
         retrieve_batch = batch_storage[batch_id]
         retrieve_batch.output_file_id = output_file_id
         file_id_storage[output_file_id] = output_file_path
+        file_id_response[output_file_id] = FileResponse(
+            id=output_file_id,
+            bytes=os.path.getsize(output_file_path),
+            created_at=int(time.time()),
+            filename=f"{output_file_id}.jsonl",
+            purpose="batch_result",
+        )
         # Update batch status to "completed"
         retrieve_batch.status = "completed"
         retrieve_batch.completed_at = int(time.time())
@@ -339,6 +363,7 @@ def v1_generate_request(all_requests):
     return_logprobs = []
     top_logprobs_nums = []
     first_prompt_type = type(all_requests[0].prompt)
+
     for request in all_requests:
         prompt = request.prompt
         assert (
@@ -364,7 +389,7 @@ def v1_generate_request(all_requests):
         )
         if len(all_requests) > 1 and request.n > 1:
             raise ValueError(
-                "Batch operation is not supported for completions from files"
+                "Parallel sampling is not supported for completions from files"
             )
 
     if len(all_requests) == 1:
@@ -381,6 +406,7 @@ def v1_generate_request(all_requests):
             prompt_kwargs = {"text": prompts}
         else:
             prompt_kwargs = {"input_ids": prompts}
+
     adapted_request = GenerateReqInput(
         **prompt_kwargs,
         sampling_params=sampling_params_list,
@@ -389,35 +415,52 @@ def v1_generate_request(all_requests):
         return_text_in_logprobs=True,
         stream=all_requests[0].stream,
     )
+
     if len(all_requests) == 1:
         return adapted_request, all_requests[0]
     return adapted_request, all_requests
 
 
-def v1_generate_response(request, ret, to_file=False):
+def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
     choices = []
     echo = False
 
-    if (not isinstance(request, List)) and request.echo:
+    if (not isinstance(request, list)) and request.echo:
         # TODO: handle the case propmt is token ids
-        if isinstance(request.prompt, list):
+        if isinstance(request.prompt, list) and isinstance(request.prompt[0], str):
+            # for the case of multiple str prompts
             prompts = request.prompt
+        elif isinstance(request.prompt, list) and isinstance(request.prompt[0], list):
+            # for the case of multiple token ids prompts
+            prompts = [
+                tokenizer_manager.tokenizer.decode(prompt, skip_special_tokens=True)
+                for prompt in request.prompt
+            ]
+        elif isinstance(request.prompt, list) and isinstance(request.prompt[0], int):
+            # for the case of single token ids prompt
+            prompts = [
+                tokenizer_manager.tokenizer.decode(
+                    request.prompt, skip_special_tokens=True
+                )
+            ]
         else:
+            # for the case of single str prompt
             prompts = [request.prompt]
         echo = True
 
     for idx, ret_item in enumerate(ret):
         text = ret_item["text"]
-        if isinstance(request, List) and request[idx].echo:
+        if isinstance(request, list) and request[idx].echo:
             echo = True
             text = request[idx].prompt + text
-        if (not isinstance(request, List)) and echo:
-            text = prompts[idx] + text
+        if (not isinstance(request, list)) and echo:
+            prompt_index = idx // request.n
+            text = prompts[prompt_index] + text
 
         logprobs = False
-        if isinstance(request, List) and request[idx].logprobs:
+        if isinstance(request, list) and request[idx].logprobs:
             logprobs = True
-        elif (not isinstance(request, List)) and request.logprobs:
+        elif (not isinstance(request, list)) and request.logprobs:
             logprobs = True
         if logprobs:
             if echo:
@@ -479,15 +522,18 @@ def v1_generate_response(request, ret, to_file=False):
             responses.append(response)
         return responses
     else:
+        prompt_tokens = sum(
+            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
+        )
         completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
         response = CompletionResponse(
             id=ret[0]["meta_info"]["id"],
             model=request.model,
             choices=choices,
             usage=UsageInfo(
-                prompt_tokens=ret[0]["meta_info"]["prompt_tokens"],
+                prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
-                total_tokens=ret[0]["meta_info"]["prompt_tokens"] + completion_tokens,
+                total_tokens=prompt_tokens + completion_tokens,
             ),
         )
     return response
@@ -513,8 +559,18 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
 
                     if not stream_buffer:  # The first chunk
                         if request.echo:
+                            if isinstance(request.prompt, str):
+                                # for the case of single str prompts
+                                prompts = request.prompt
+                            elif isinstance(request.prompt, list) and isinstance(
+                                request.prompt[0], int
+                            ):
+                                prompts = tokenizer_manager.tokenizer.decode(
+                                    request.prompt, skip_special_tokens=True
+                                )
+
                             # Prepend prompt in response text.
-                            text = request.prompt + text
+                            text = prompts + text
 
                     if request.logprobs:
                         # The first chunk and echo is enabled.
@@ -539,7 +595,6 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                                 "output_top_logprobs"
                             ][n_prev_token:],
                         )
-
                         n_prev_token = len(
                             content["meta_info"]["output_token_logprobs"]
                         )
@@ -588,7 +643,7 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
     if not isinstance(ret, list):
         ret = [ret]
 
-    response = v1_generate_response(request, ret)
+    response = v1_generate_response(request, ret, tokenizer_manager)
     return response
 
 
@@ -626,7 +681,7 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
                 prompt_ids = tokenizer_manager.tokenizer.encode(prompt)
         else:
             # Use the raw prompt and stop strings if the messages is already a string.
-            prompt = request.messages
+            prompt_ids = request.messages
             stop = request.stop
             image_data = None
         input_ids.append(prompt_ids)
@@ -647,12 +702,21 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
         image_data_list.append(image_data)
     if len(all_requests) == 1:
         input_ids = input_ids[0]
+        if isinstance(input_ids, str):
+            prompt_kwargs = {"text": input_ids}
+        else:
+            prompt_kwargs = {"input_ids": input_ids}
         sampling_params_list = sampling_params_list[0]
         image_data = image_data_list[0]
         return_logprobs = return_logprobs[0]
         top_logprobs_nums = top_logprobs_nums[0]
+    else:
+        if isinstance(input_ids[0], str):
+            prompt_kwargs = {"text": input_ids}
+        else:
+            prompt_kwargs = {"input_ids": input_ids}
     adapted_request = GenerateReqInput(
-        input_ids=input_ids,
+        **prompt_kwargs,
         image_data=image_data,
         sampling_params=sampling_params_list,
         return_logprob=return_logprobs,
@@ -667,14 +731,12 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
 
 def v1_chat_generate_response(request, ret, to_file=False):
     choices = []
-    total_prompt_tokens = 0
-    total_completion_tokens = 0
 
     for idx, ret_item in enumerate(ret):
         logprobs = False
-        if isinstance(request, List) and request[idx].logprobs:
+        if isinstance(request, list) and request[idx].logprobs:
             logprobs = True
-        elif (not isinstance(request, List)) and request.logprobs:
+        elif (not isinstance(request, list)) and request.logprobs:
             logprobs = True
         if logprobs:
             logprobs = to_openai_style_logprobs(
@@ -707,8 +769,6 @@ def v1_chat_generate_response(request, ret, to_file=False):
             choice_logprobs = ChoiceLogprobs(content=token_logprobs)
         else:
             choice_logprobs = None
-        prompt_tokens = ret_item["meta_info"]["prompt_tokens"]
-        completion_tokens = ret_item["meta_info"]["completion_tokens"]
 
         if to_file:
             # to make the choice data json serializable
@@ -727,8 +787,7 @@ def v1_chat_generate_response(request, ret, to_file=False):
             )
 
         choices.append(choice_data)
-        total_prompt_tokens += prompt_tokens
-        total_completion_tokens += completion_tokens
+
     if to_file:
         responses = []
 
@@ -755,14 +814,18 @@ def v1_chat_generate_response(request, ret, to_file=False):
             responses.append(response)
         return responses
     else:
+        prompt_tokens = sum(
+            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
+        )
+        completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
         response = ChatCompletionResponse(
             id=ret[0]["meta_info"]["id"],
             model=request.model,
             choices=choices,
             usage=UsageInfo(
-                prompt_tokens=total_prompt_tokens,
-                completion_tokens=total_completion_tokens,
-                total_tokens=total_prompt_tokens + total_completion_tokens,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=prompt_tokens + completion_tokens,
             ),
         )
         return response
@@ -779,10 +842,58 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
             is_first = True
 
             stream_buffer = ""
+            n_prev_token = 0
             try:
                 async for content in tokenizer_manager.generate_request(
                     adapted_request, raw_request
                 ):
+                    prompt_tokens = content["meta_info"]["prompt_tokens"]
+                    completion_tokens = content["meta_info"]["completion_tokens"]
+                    if request.logprobs:
+                        logprobs = to_openai_style_logprobs(
+                            output_token_logprobs=content["meta_info"][
+                                "output_token_logprobs"
+                            ][n_prev_token:],
+                            output_top_logprobs=content["meta_info"][
+                                "output_top_logprobs"
+                            ][n_prev_token:],
+                        )
+
+                        n_prev_token = len(
+                            content["meta_info"]["output_token_logprobs"]
+                        )
+                        token_logprobs = []
+                        for token, logprob in zip(
+                            logprobs.tokens, logprobs.token_logprobs
+                        ):
+                            token_bytes = list(token.encode("utf-8"))
+                            top_logprobs = []
+                            if logprobs.top_logprobs:
+                                for top_token, top_logprob in logprobs.top_logprobs[
+                                    0
+                                ].items():
+                                    top_token_bytes = list(top_token.encode("utf-8"))
+                                    top_logprobs.append(
+                                        TopLogprob(
+                                            token=top_token,
+                                            bytes=top_token_bytes,
+                                            logprob=top_logprob,
+                                        )
+                                    )
+                            token_logprobs.append(
+                                ChatCompletionTokenLogprob(
+                                    token=token,
+                                    bytes=token_bytes,
+                                    logprob=logprob,
+                                    top_logprobs=top_logprobs,
+                                )
+                            )
+
+                        choice_logprobs = ChoiceLogprobs(content=token_logprobs)
+
+                    else:
+                        choice_logprobs = None
+
                     if is_first:
                         # First chunk with role
                         is_first = False
@@ -790,11 +901,17 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                             index=0,
                             delta=DeltaMessage(role="assistant"),
                             finish_reason=content["meta_info"]["finish_reason"],
+                            logprobs=choice_logprobs,
                         )
                         chunk = ChatCompletionStreamResponse(
                             id=content["meta_info"]["id"],
                             choices=[choice_data],
                             model=request.model,
+                            usage=UsageInfo(
+                                prompt_tokens=prompt_tokens,
+                                completion_tokens=completion_tokens,
+                                total_tokens=prompt_tokens + completion_tokens,
+                            ),
                         )
                         yield f"data: {chunk.model_dump_json()}\n\n"
 
@@ -805,11 +922,17 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         index=0,
                         delta=DeltaMessage(content=delta),
                         finish_reason=content["meta_info"]["finish_reason"],
+                        logprobs=choice_logprobs,
                     )
                     chunk = ChatCompletionStreamResponse(
                         id=content["meta_info"]["id"],
                         choices=[choice_data],
                         model=request.model,
+                        usage=UsageInfo(
+                            prompt_tokens=prompt_tokens,
+                            completion_tokens=completion_tokens,
+                            total_tokens=prompt_tokens + completion_tokens,
+                        ),
                     )
                     yield f"data: {chunk.model_dump_json()}\n\n"
             except ValueError as e:
@@ -830,7 +953,6 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
         ).__anext__()
     except ValueError as e:
         return create_error_response(str(e))
-
     if not isinstance(ret, list):
         ret = [ret]
 

sglang/srt/openai_api/protocol.py

@@ -95,6 +95,12 @@ class FileResponse(BaseModel):
     purpose: str
 
 
+class FileDeleteResponse(BaseModel):
+    id: str
+    object: str = "file"
+    deleted: bool
+
+
 class BatchRequest(BaseModel):
     input_file_id: (
         str  # The ID of an uploaded file that contains requests for the new batch
@@ -278,7 +284,7 @@ class DeltaMessage(BaseModel):
 class ChatCompletionResponseStreamChoice(BaseModel):
     index: int
     delta: DeltaMessage
-    logprobs: Optional[LogProbs] = None
+    logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None
     finish_reason: Optional[str] = None