sglang 0.5.3rc0__py3-none-any.whl → 0.5.4__py3-none-any.whl

sglang/srt/grpc/sglang_scheduler_pb2.pyi

@@ -3,7 +3,6 @@ import datetime
 from google.protobuf import timestamp_pb2 as _timestamp_pb2
 from google.protobuf import struct_pb2 as _struct_pb2
 from google.protobuf.internal import containers as _containers
-from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from collections.abc import Iterable as _Iterable, Mapping as _Mapping
@@ -12,7 +11,7 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class SamplingParams(_message.Message):
-    __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "lora_path", "n", "token_healing", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "structural_tag", "custom_params")
+    __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "structural_tag", "n", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "custom_params")
     class LogitBiasEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -35,15 +34,13 @@ class SamplingParams(_message.Message):
     REGEX_FIELD_NUMBER: _ClassVar[int]
     JSON_SCHEMA_FIELD_NUMBER: _ClassVar[int]
     EBNF_GRAMMAR_FIELD_NUMBER: _ClassVar[int]
-    LORA_PATH_FIELD_NUMBER: _ClassVar[int]
+    STRUCTURAL_TAG_FIELD_NUMBER: _ClassVar[int]
     N_FIELD_NUMBER: _ClassVar[int]
-    TOKEN_HEALING_FIELD_NUMBER: _ClassVar[int]
     MIN_NEW_TOKENS_FIELD_NUMBER: _ClassVar[int]
     IGNORE_EOS_FIELD_NUMBER: _ClassVar[int]
     NO_STOP_TRIM_FIELD_NUMBER: _ClassVar[int]
     STREAM_INTERVAL_FIELD_NUMBER: _ClassVar[int]
     LOGIT_BIAS_FIELD_NUMBER: _ClassVar[int]
-    STRUCTURAL_TAG_FIELD_NUMBER: _ClassVar[int]
     CUSTOM_PARAMS_FIELD_NUMBER: _ClassVar[int]
     temperature: float
     top_p: float
@@ -60,17 +57,15 @@ class SamplingParams(_message.Message):
     regex: str
     json_schema: str
     ebnf_grammar: str
-    lora_path: str
+    structural_tag: str
     n: int
-    token_healing: bool
     min_new_tokens: int
     ignore_eos: bool
     no_stop_trim: bool
     stream_interval: int
     logit_bias: _containers.ScalarMap[str, float]
-    structural_tag: str
     custom_params: _struct_pb2.Struct
-    def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., lora_path: _Optional[str] = ..., n: _Optional[int] = ..., token_healing: bool = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., structural_tag: _Optional[str] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ...
+    def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., structural_tag: _Optional[str] = ..., n: _Optional[int] = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ...
 
 class DisaggregatedParams(_message.Message):
     __slots__ = ("bootstrap_host", "bootstrap_port", "bootstrap_room")
@@ -83,7 +78,7 @@ class DisaggregatedParams(_message.Message):
     def __init__(self, bootstrap_host: _Optional[str] = ..., bootstrap_port: _Optional[int] = ..., bootstrap_room: _Optional[int] = ...) -> None: ...
 
 class GenerateRequest(_message.Message):
-    __slots__ = ("request_id", "tokenized", "mm_inputs", "sampling_params", "return_logprob", "logprob_start_len", "top_logprobs_num", "token_ids_logprob", "return_hidden_states", "disaggregated_params", "custom_logit_processor", "timestamp", "log_metrics", "input_embeds", "lora_id", "data_parallel_rank", "dp_balance_id")
+    __slots__ = ("request_id", "tokenized", "mm_inputs", "sampling_params", "return_logprob", "logprob_start_len", "top_logprobs_num", "token_ids_logprob", "return_hidden_states", "disaggregated_params", "custom_logit_processor", "timestamp", "log_metrics", "input_embeds", "lora_id", "data_parallel_rank", "stream")
     REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
     TOKENIZED_FIELD_NUMBER: _ClassVar[int]
     MM_INPUTS_FIELD_NUMBER: _ClassVar[int]
@@ -100,7 +95,7 @@ class GenerateRequest(_message.Message):
     INPUT_EMBEDS_FIELD_NUMBER: _ClassVar[int]
     LORA_ID_FIELD_NUMBER: _ClassVar[int]
     DATA_PARALLEL_RANK_FIELD_NUMBER: _ClassVar[int]
-    DP_BALANCE_ID_FIELD_NUMBER: _ClassVar[int]
+    STREAM_FIELD_NUMBER: _ClassVar[int]
     request_id: str
     tokenized: TokenizedInput
     mm_inputs: MultimodalInputs
@@ -117,8 +112,8 @@ class GenerateRequest(_message.Message):
     input_embeds: _containers.RepeatedScalarFieldContainer[float]
     lora_id: str
     data_parallel_rank: int
-    dp_balance_id: int
-    def __init__(self, request_id: _Optional[str] = ..., tokenized: _Optional[_Union[TokenizedInput, _Mapping]] = ..., mm_inputs: _Optional[_Union[MultimodalInputs, _Mapping]] = ..., sampling_params: _Optional[_Union[SamplingParams, _Mapping]] = ..., return_logprob: bool = ..., logprob_start_len: _Optional[int] = ..., top_logprobs_num: _Optional[int] = ..., token_ids_logprob: _Optional[_Iterable[int]] = ..., return_hidden_states: bool = ..., disaggregated_params: _Optional[_Union[DisaggregatedParams, _Mapping]] = ..., custom_logit_processor: _Optional[str] = ..., timestamp: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., log_metrics: bool = ..., input_embeds: _Optional[_Iterable[float]] = ..., lora_id: _Optional[str] = ..., data_parallel_rank: _Optional[int] = ..., dp_balance_id: _Optional[int] = ...) -> None: ...
+    stream: bool
+    def __init__(self, request_id: _Optional[str] = ..., tokenized: _Optional[_Union[TokenizedInput, _Mapping]] = ..., mm_inputs: _Optional[_Union[MultimodalInputs, _Mapping]] = ..., sampling_params: _Optional[_Union[SamplingParams, _Mapping]] = ..., return_logprob: bool = ..., logprob_start_len: _Optional[int] = ..., top_logprobs_num: _Optional[int] = ..., token_ids_logprob: _Optional[_Iterable[int]] = ..., return_hidden_states: bool = ..., disaggregated_params: _Optional[_Union[DisaggregatedParams, _Mapping]] = ..., custom_logit_processor: _Optional[str] = ..., timestamp: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., log_metrics: bool = ..., input_embeds: _Optional[_Iterable[float]] = ..., lora_id: _Optional[str] = ..., data_parallel_rank: _Optional[int] = ..., stream: bool = ...) -> None: ...
 
 class TokenizedInput(_message.Message):
     __slots__ = ("original_text", "input_ids")
@@ -161,52 +156,50 @@ class GenerateResponse(_message.Message):
     def __init__(self, request_id: _Optional[str] = ..., chunk: _Optional[_Union[GenerateStreamChunk, _Mapping]] = ..., complete: _Optional[_Union[GenerateComplete, _Mapping]] = ..., error: _Optional[_Union[GenerateError, _Mapping]] = ...) -> None: ...
 
 class GenerateStreamChunk(_message.Message):
-    __slots__ = ("token_id", "text", "prompt_tokens", "completion_tokens", "cached_tokens", "logprobs", "hidden_states", "generation_time", "queue_time")
-    TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
-    TEXT_FIELD_NUMBER: _ClassVar[int]
+    __slots__ = ("token_ids", "prompt_tokens", "completion_tokens", "cached_tokens", "output_logprobs", "hidden_states", "input_logprobs", "index")
+    TOKEN_IDS_FIELD_NUMBER: _ClassVar[int]
     PROMPT_TOKENS_FIELD_NUMBER: _ClassVar[int]
     COMPLETION_TOKENS_FIELD_NUMBER: _ClassVar[int]
     CACHED_TOKENS_FIELD_NUMBER: _ClassVar[int]
-    LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    OUTPUT_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
     HIDDEN_STATES_FIELD_NUMBER: _ClassVar[int]
-    GENERATION_TIME_FIELD_NUMBER: _ClassVar[int]
-    QUEUE_TIME_FIELD_NUMBER: _ClassVar[int]
-    token_id: int
-    text: str
+    INPUT_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    INDEX_FIELD_NUMBER: _ClassVar[int]
+    token_ids: _containers.RepeatedScalarFieldContainer[int]
     prompt_tokens: int
     completion_tokens: int
     cached_tokens: int
-    logprobs: LogProbs
+    output_logprobs: OutputLogProbs
     hidden_states: _containers.RepeatedScalarFieldContainer[float]
-    generation_time: float
-    queue_time: int
-    def __init__(self, token_id: _Optional[int] = ..., text: _Optional[str] = ..., prompt_tokens: _Optional[int] = ..., completion_tokens: _Optional[int] = ..., cached_tokens: _Optional[int] = ..., logprobs: _Optional[_Union[LogProbs, _Mapping]] = ..., hidden_states: _Optional[_Iterable[float]] = ..., generation_time: _Optional[float] = ..., queue_time: _Optional[int] = ...) -> None: ...
+    input_logprobs: InputLogProbs
+    index: int
+    def __init__(self, token_ids: _Optional[_Iterable[int]] = ..., prompt_tokens: _Optional[int] = ..., completion_tokens: _Optional[int] = ..., cached_tokens: _Optional[int] = ..., output_logprobs: _Optional[_Union[OutputLogProbs, _Mapping]] = ..., hidden_states: _Optional[_Iterable[float]] = ..., input_logprobs: _Optional[_Union[InputLogProbs, _Mapping]] = ..., index: _Optional[int] = ...) -> None: ...
 
 class GenerateComplete(_message.Message):
-    __slots__ = ("output_ids", "output_text", "finish_reason", "all_logprobs", "all_hidden_states")
-    class FinishReason(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
-        __slots__ = ()
-        STOP: _ClassVar[GenerateComplete.FinishReason]
-        LENGTH: _ClassVar[GenerateComplete.FinishReason]
-        EOS_TOKEN: _ClassVar[GenerateComplete.FinishReason]
-        STOP_STR: _ClassVar[GenerateComplete.FinishReason]
-        ABORT: _ClassVar[GenerateComplete.FinishReason]
-    STOP: GenerateComplete.FinishReason
-    LENGTH: GenerateComplete.FinishReason
-    EOS_TOKEN: GenerateComplete.FinishReason
-    STOP_STR: GenerateComplete.FinishReason
-    ABORT: GenerateComplete.FinishReason
+    __slots__ = ("output_ids", "finish_reason", "prompt_tokens", "completion_tokens", "cached_tokens", "output_logprobs", "all_hidden_states", "matched_token_id", "matched_stop_str", "input_logprobs", "index")
     OUTPUT_IDS_FIELD_NUMBER: _ClassVar[int]
-    OUTPUT_TEXT_FIELD_NUMBER: _ClassVar[int]
     FINISH_REASON_FIELD_NUMBER: _ClassVar[int]
-    ALL_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    PROMPT_TOKENS_FIELD_NUMBER: _ClassVar[int]
+    COMPLETION_TOKENS_FIELD_NUMBER: _ClassVar[int]
+    CACHED_TOKENS_FIELD_NUMBER: _ClassVar[int]
+    OUTPUT_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
     ALL_HIDDEN_STATES_FIELD_NUMBER: _ClassVar[int]
+    MATCHED_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
+    MATCHED_STOP_STR_FIELD_NUMBER: _ClassVar[int]
+    INPUT_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    INDEX_FIELD_NUMBER: _ClassVar[int]
     output_ids: _containers.RepeatedScalarFieldContainer[int]
-    output_text: str
-    finish_reason: GenerateComplete.FinishReason
-    all_logprobs: _containers.RepeatedCompositeFieldContainer[LogProbs]
+    finish_reason: str
+    prompt_tokens: int
+    completion_tokens: int
+    cached_tokens: int
+    output_logprobs: OutputLogProbs
     all_hidden_states: _containers.RepeatedCompositeFieldContainer[HiddenStates]
-    def __init__(self, output_ids: _Optional[_Iterable[int]] = ..., output_text: _Optional[str] = ..., finish_reason: _Optional[_Union[GenerateComplete.FinishReason, str]] = ..., all_logprobs: _Optional[_Iterable[_Union[LogProbs, _Mapping]]] = ..., all_hidden_states: _Optional[_Iterable[_Union[HiddenStates, _Mapping]]] = ...) -> None: ...
+    matched_token_id: int
+    matched_stop_str: str
+    input_logprobs: InputLogProbs
+    index: int
+    def __init__(self, output_ids: _Optional[_Iterable[int]] = ..., finish_reason: _Optional[str] = ..., prompt_tokens: _Optional[int] = ..., completion_tokens: _Optional[int] = ..., cached_tokens: _Optional[int] = ..., output_logprobs: _Optional[_Union[OutputLogProbs, _Mapping]] = ..., all_hidden_states: _Optional[_Iterable[_Union[HiddenStates, _Mapping]]] = ..., matched_token_id: _Optional[int] = ..., matched_stop_str: _Optional[str] = ..., input_logprobs: _Optional[_Union[InputLogProbs, _Mapping]] = ..., index: _Optional[int] = ...) -> None: ...
 
 class GenerateError(_message.Message):
     __slots__ = ("message", "http_status_code", "details")
@@ -218,27 +211,39 @@ class GenerateError(_message.Message):
     details: str
     def __init__(self, message: _Optional[str] = ..., http_status_code: _Optional[str] = ..., details: _Optional[str] = ...) -> None: ...
 
-class LogProbs(_message.Message):
-    __slots__ = ("token_logprobs", "token_ids", "top_logprobs", "token_texts")
+class OutputLogProbs(_message.Message):
+    __slots__ = ("token_logprobs", "token_ids", "top_logprobs")
     TOKEN_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
     TOKEN_IDS_FIELD_NUMBER: _ClassVar[int]
     TOP_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
-    TOKEN_TEXTS_FIELD_NUMBER: _ClassVar[int]
     token_logprobs: _containers.RepeatedScalarFieldContainer[float]
     token_ids: _containers.RepeatedScalarFieldContainer[int]
     top_logprobs: _containers.RepeatedCompositeFieldContainer[TopLogProbs]
-    token_texts: _containers.RepeatedScalarFieldContainer[str]
-    def __init__(self, token_logprobs: _Optional[_Iterable[float]] = ..., token_ids: _Optional[_Iterable[int]] = ..., top_logprobs: _Optional[_Iterable[_Union[TopLogProbs, _Mapping]]] = ..., token_texts: _Optional[_Iterable[str]] = ...) -> None: ...
+    def __init__(self, token_logprobs: _Optional[_Iterable[float]] = ..., token_ids: _Optional[_Iterable[int]] = ..., top_logprobs: _Optional[_Iterable[_Union[TopLogProbs, _Mapping]]] = ...) -> None: ...
+
+class InputLogProbs(_message.Message):
+    __slots__ = ("token_logprobs", "token_ids", "top_logprobs")
+    TOKEN_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    TOKEN_IDS_FIELD_NUMBER: _ClassVar[int]
+    TOP_LOGPROBS_FIELD_NUMBER: _ClassVar[int]
+    token_logprobs: _containers.RepeatedCompositeFieldContainer[InputTokenLogProb]
+    token_ids: _containers.RepeatedScalarFieldContainer[int]
+    top_logprobs: _containers.RepeatedCompositeFieldContainer[TopLogProbs]
+    def __init__(self, token_logprobs: _Optional[_Iterable[_Union[InputTokenLogProb, _Mapping]]] = ..., token_ids: _Optional[_Iterable[int]] = ..., top_logprobs: _Optional[_Iterable[_Union[TopLogProbs, _Mapping]]] = ...) -> None: ...
+
+class InputTokenLogProb(_message.Message):
+    __slots__ = ("value",)
+    VALUE_FIELD_NUMBER: _ClassVar[int]
+    value: float
+    def __init__(self, value: _Optional[float] = ...) -> None: ...
 
 class TopLogProbs(_message.Message):
-    __slots__ = ("values", "token_ids", "token_texts")
+    __slots__ = ("values", "token_ids")
     VALUES_FIELD_NUMBER: _ClassVar[int]
     TOKEN_IDS_FIELD_NUMBER: _ClassVar[int]
-    TOKEN_TEXTS_FIELD_NUMBER: _ClassVar[int]
     values: _containers.RepeatedScalarFieldContainer[float]
     token_ids: _containers.RepeatedScalarFieldContainer[int]
-    token_texts: _containers.RepeatedScalarFieldContainer[str]
-    def __init__(self, values: _Optional[_Iterable[float]] = ..., token_ids: _Optional[_Iterable[int]] = ..., token_texts: _Optional[_Iterable[str]] = ...) -> None: ...
+    def __init__(self, values: _Optional[_Iterable[float]] = ..., token_ids: _Optional[_Iterable[int]] = ...) -> None: ...
 
 class HiddenStates(_message.Message):
     __slots__ = ("values", "layer", "position")
@@ -283,20 +288,18 @@ class EmbedResponse(_message.Message):
     def __init__(self, request_id: _Optional[str] = ..., complete: _Optional[_Union[EmbedComplete, _Mapping]] = ..., error: _Optional[_Union[EmbedError, _Mapping]] = ...) -> None: ...
 
 class EmbedComplete(_message.Message):
-    __slots__ = ("embedding", "prompt_tokens", "cached_tokens", "embedding_dim", "generation_time", "batch_embeddings")
+    __slots__ = ("embedding", "prompt_tokens", "cached_tokens", "embedding_dim", "batch_embeddings")
     EMBEDDING_FIELD_NUMBER: _ClassVar[int]
     PROMPT_TOKENS_FIELD_NUMBER: _ClassVar[int]
     CACHED_TOKENS_FIELD_NUMBER: _ClassVar[int]
     EMBEDDING_DIM_FIELD_NUMBER: _ClassVar[int]
-    GENERATION_TIME_FIELD_NUMBER: _ClassVar[int]
     BATCH_EMBEDDINGS_FIELD_NUMBER: _ClassVar[int]
     embedding: _containers.RepeatedScalarFieldContainer[float]
     prompt_tokens: int
     cached_tokens: int
     embedding_dim: int
-    generation_time: float
     batch_embeddings: _containers.RepeatedCompositeFieldContainer[Embedding]
-    def __init__(self, embedding: _Optional[_Iterable[float]] = ..., prompt_tokens: _Optional[int] = ..., cached_tokens: _Optional[int] = ..., embedding_dim: _Optional[int] = ..., generation_time: _Optional[float] = ..., batch_embeddings: _Optional[_Iterable[_Union[Embedding, _Mapping]]] = ...) -> None: ...
+    def __init__(self, embedding: _Optional[_Iterable[float]] = ..., prompt_tokens: _Optional[int] = ..., cached_tokens: _Optional[int] = ..., embedding_dim: _Optional[int] = ..., batch_embeddings: _Optional[_Iterable[_Union[Embedding, _Mapping]]] = ...) -> None: ...
 
 class Embedding(_message.Message):
     __slots__ = ("values", "index")
@@ -317,10 +320,8 @@ class EmbedError(_message.Message):
     def __init__(self, message: _Optional[str] = ..., code: _Optional[str] = ..., details: _Optional[str] = ...) -> None: ...
 
 class HealthCheckRequest(_message.Message):
-    __slots__ = ("tokenized",)
-    TOKENIZED_FIELD_NUMBER: _ClassVar[int]
-    tokenized: TokenizedInput
-    def __init__(self, tokenized: _Optional[_Union[TokenizedInput, _Mapping]] = ...) -> None: ...
+    __slots__ = ()
+    def __init__(self) -> None: ...
 
 class HealthCheckResponse(_message.Message):
     __slots__ = ("healthy", "message")
@@ -425,3 +426,65 @@ class SetInternalStateResponse(_message.Message):
     success: bool
     message: str
     def __init__(self, success: bool = ..., message: _Optional[str] = ...) -> None: ...
+
+class GetModelInfoRequest(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
+class GetModelInfoResponse(_message.Message):
+    __slots__ = ("model_path", "tokenizer_path", "is_generation", "preferred_sampling_params", "weight_version", "served_model_name", "max_context_length", "vocab_size", "supports_vision", "model_type", "eos_token_ids", "pad_token_id", "bos_token_id", "max_req_input_len")
+    MODEL_PATH_FIELD_NUMBER: _ClassVar[int]
+    TOKENIZER_PATH_FIELD_NUMBER: _ClassVar[int]
+    IS_GENERATION_FIELD_NUMBER: _ClassVar[int]
+    PREFERRED_SAMPLING_PARAMS_FIELD_NUMBER: _ClassVar[int]
+    WEIGHT_VERSION_FIELD_NUMBER: _ClassVar[int]
+    SERVED_MODEL_NAME_FIELD_NUMBER: _ClassVar[int]
+    MAX_CONTEXT_LENGTH_FIELD_NUMBER: _ClassVar[int]
+    VOCAB_SIZE_FIELD_NUMBER: _ClassVar[int]
+    SUPPORTS_VISION_FIELD_NUMBER: _ClassVar[int]
+    MODEL_TYPE_FIELD_NUMBER: _ClassVar[int]
+    EOS_TOKEN_IDS_FIELD_NUMBER: _ClassVar[int]
+    PAD_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
+    BOS_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
+    MAX_REQ_INPUT_LEN_FIELD_NUMBER: _ClassVar[int]
+    model_path: str
+    tokenizer_path: str
+    is_generation: bool
+    preferred_sampling_params: str
+    weight_version: str
+    served_model_name: str
+    max_context_length: int
+    vocab_size: int
+    supports_vision: bool
+    model_type: str
+    eos_token_ids: _containers.RepeatedScalarFieldContainer[int]
+    pad_token_id: int
+    bos_token_id: int
+    max_req_input_len: int
+    def __init__(self, model_path: _Optional[str] = ..., tokenizer_path: _Optional[str] = ..., is_generation: bool = ..., preferred_sampling_params: _Optional[str] = ..., weight_version: _Optional[str] = ..., served_model_name: _Optional[str] = ..., max_context_length: _Optional[int] = ..., vocab_size: _Optional[int] = ..., supports_vision: bool = ..., model_type: _Optional[str] = ..., eos_token_ids: _Optional[_Iterable[int]] = ..., pad_token_id: _Optional[int] = ..., bos_token_id: _Optional[int] = ..., max_req_input_len: _Optional[int] = ...) -> None: ...
+
+class GetServerInfoRequest(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
+class GetServerInfoResponse(_message.Message):
+    __slots__ = ("server_args", "scheduler_info", "active_requests", "is_paused", "last_receive_timestamp", "uptime_seconds", "sglang_version", "server_type", "start_time")
+    SERVER_ARGS_FIELD_NUMBER: _ClassVar[int]
+    SCHEDULER_INFO_FIELD_NUMBER: _ClassVar[int]
+    ACTIVE_REQUESTS_FIELD_NUMBER: _ClassVar[int]
+    IS_PAUSED_FIELD_NUMBER: _ClassVar[int]
+    LAST_RECEIVE_TIMESTAMP_FIELD_NUMBER: _ClassVar[int]
+    UPTIME_SECONDS_FIELD_NUMBER: _ClassVar[int]
+    SGLANG_VERSION_FIELD_NUMBER: _ClassVar[int]
+    SERVER_TYPE_FIELD_NUMBER: _ClassVar[int]
+    START_TIME_FIELD_NUMBER: _ClassVar[int]
+    server_args: _struct_pb2.Struct
+    scheduler_info: _struct_pb2.Struct
+    active_requests: int
+    is_paused: bool
+    last_receive_timestamp: float
+    uptime_seconds: float
+    sglang_version: str
+    server_type: str
+    start_time: _timestamp_pb2.Timestamp
+    def __init__(self, server_args: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., scheduler_info: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., active_requests: _Optional[int] = ..., is_paused: bool = ..., last_receive_timestamp: _Optional[float] = ..., uptime_seconds: _Optional[float] = ..., sglang_version: _Optional[str] = ..., server_type: _Optional[str] = ..., start_time: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...

sglang/srt/grpc/sglang_scheduler_pb2_grpc.py

@@ -1,3 +1,6 @@
+# This file is auto-generated. Do not edit manually.
+# Regenerate with: python compile_proto.py
+
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
@@ -5,7 +8,7 @@ import warnings
 
 from . import sglang_scheduler_pb2 as sglang__scheduler__pb2
 
-GRPC_GENERATED_VERSION = '1.74.0'
+GRPC_GENERATED_VERSION = '1.75.1'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 
@@ -56,6 +59,16 @@ class SglangSchedulerStub(object):
                 request_serializer=sglang__scheduler__pb2.AbortRequest.SerializeToString,
                 response_deserializer=sglang__scheduler__pb2.AbortResponse.FromString,
                 _registered_method=True)
+        self.GetModelInfo = channel.unary_unary(
+                '/sglang.grpc.scheduler.SglangScheduler/GetModelInfo',
+                request_serializer=sglang__scheduler__pb2.GetModelInfoRequest.SerializeToString,
+                response_deserializer=sglang__scheduler__pb2.GetModelInfoResponse.FromString,
+                _registered_method=True)
+        self.GetServerInfo = channel.unary_unary(
+                '/sglang.grpc.scheduler.SglangScheduler/GetServerInfo',
+                request_serializer=sglang__scheduler__pb2.GetServerInfoRequest.SerializeToString,
+                response_deserializer=sglang__scheduler__pb2.GetServerInfoResponse.FromString,
+                _registered_method=True)
 
 
 class SglangSchedulerServicer(object):
@@ -91,6 +104,20 @@ class SglangSchedulerServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def GetModelInfo(self, request, context):
+        """Get model information
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def GetServerInfo(self, request, context):
+        """Get server information
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_SglangSchedulerServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -114,6 +141,16 @@ def add_SglangSchedulerServicer_to_server(servicer, server):
                     request_deserializer=sglang__scheduler__pb2.AbortRequest.FromString,
                     response_serializer=sglang__scheduler__pb2.AbortResponse.SerializeToString,
             ),
+            'GetModelInfo': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetModelInfo,
+                    request_deserializer=sglang__scheduler__pb2.GetModelInfoRequest.FromString,
+                    response_serializer=sglang__scheduler__pb2.GetModelInfoResponse.SerializeToString,
+            ),
+            'GetServerInfo': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetServerInfo,
+                    request_deserializer=sglang__scheduler__pb2.GetServerInfoRequest.FromString,
+                    response_serializer=sglang__scheduler__pb2.GetServerInfoResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'sglang.grpc.scheduler.SglangScheduler', rpc_method_handlers)
@@ -234,3 +271,57 @@ class SglangScheduler(object):
             timeout,
             metadata,
             _registered_method=True)
+
+    @staticmethod
+    def GetModelInfo(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/sglang.grpc.scheduler.SglangScheduler/GetModelInfo',
+            sglang__scheduler__pb2.GetModelInfoRequest.SerializeToString,
+            sglang__scheduler__pb2.GetModelInfoResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def GetServerInfo(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/sglang.grpc.scheduler.SglangScheduler/GetServerInfo',
+            sglang__scheduler__pb2.GetServerInfoRequest.SerializeToString,
+            sglang__scheduler__pb2.GetServerInfoResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)

sglang/srt/layers/activation.py

@@ -224,12 +224,13 @@ class XIELU(CustomOp):
                 self._xielu_cuda_fn = self._xielu_cuda
             logger.warning_once(msg)
         except Exception as err:
-            logger.warning_once(
-                "CUDA-fused xIELU not available (%s) –"
-                " falling back to a Python version.\n"
-                "For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`",
-                str(err),
-            )
+            pass
+            # logger.warning_once(
+            #     "CUDA-fused xIELU not available (%s) –"
+            #     " falling back to a Python version.\n"
+            #     "For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`",
+            #     str(err),
+            # )
 
     def _xielu_python(self, x: torch.Tensor) -> torch.Tensor:
         alpha_p = nn.functional.softplus(self.alpha_p)
@@ -379,4 +380,7 @@ if not (
     logger.info(
         "sgl-kernel is not available on Non-NV, Non-AMD platforms or Non-AMX CPUs. Fallback to other kernel libraries."
     )
-    from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
+    from vllm.model_executor.layers.activation import (  # noqa: F401
+        GeluAndMul,
+        SiluAndMul,
+    )

sglang/srt/layers/attention/aiter_backend.py

@@ -4,18 +4,13 @@ from __future__ import annotations
 end to end attention solution with aiter kernels
 """
 
-import math
-import os
 from dataclasses import dataclass
 from enum import Enum, auto
-from functools import partial
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
 import torch
 import triton
-import triton.language as tl
 
-from sglang.global_config import global_config
 from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
 from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton
 from sglang.srt.layers.dp_attention import (
@@ -27,7 +22,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo
 if TYPE_CHECKING:
     from sglang.srt.layers.radix_attention import RadixAttention
     from sglang.srt.model_executor.model_runner import ModelRunner
-    from sglang.srt.speculative.spec_info import SpecInfo
+    from sglang.srt.speculative.spec_info import SpecInput
 
 try:
     from aiter import (
@@ -374,7 +369,7 @@ class AiterAttnBackend(AttentionBackend):
         seq_lens: torch.Tensor,
         encoder_lens: Optional[torch.Tensor],
         forward_mode: ForwardMode,
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
     ):
         if forward_mode.is_decode_or_idle():
             qo_indptr = None
@@ -509,7 +504,7 @@ class AiterAttnBackend(AttentionBackend):
         seq_lens_sum: int,
         encoder_lens: Optional[torch.Tensor],
         forward_mode: ForwardMode,
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
         seq_lens_cpu: Optional[torch.Tensor],
     ):
         if forward_mode.is_decode_or_idle():
@@ -619,7 +614,11 @@ class AiterAttnBackend(AttentionBackend):
             assert len(k.shape) == 3
             assert len(v.shape) == 3
 
-            if forward_batch.forward_mode.is_extend():
+            if (
+                forward_batch.forward_mode.is_extend()
+                and not forward_batch.forward_mode.is_target_verify()
+                and not forward_batch.forward_mode.is_draft_extend()
+            ):
                 if kv_indices.shape[0] == 0:
                     o = flash_attn_varlen_func(
                         q,
@@ -884,7 +883,7 @@ class AiterIndicesUpdaterPrefill:
         seq_lens_sum: int,
         prefix_lens: torch.Tensor,
         encoder_lens: Optional[torch.Tensor],
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
     ):
         # Keep the signature for type checking. It will be assigned during runtime.
         raise NotImplementedError()
@@ -896,7 +895,7 @@ class AiterIndicesUpdaterPrefill:
         seq_lens_sum: int,
         prefix_lens: torch.Tensor,
         encoder_lens: Optional[torch.Tensor],
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
     ):
 
         kv_start_idx = None
@@ -980,7 +979,7 @@ class AiterMlaIndicesUpdaterPrefill:
         extend_lens: torch.Tensor,
         max_q_len: int,
         max_kv_len: int,
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
     ):
         # Keep the signature for type checking. It will be assigned during runtime.
         raise NotImplementedError()
@@ -993,7 +992,7 @@ class AiterMlaIndicesUpdaterPrefill:
         extend_lens: torch.Tensor,
         max_q_len: int,
         max_kv_len: int,
-        spec_info: Optional[SpecInfo],
+        spec_info: Optional[SpecInput],
     ):
         bs = len(req_pool_indices)
 
@@ -1050,7 +1049,7 @@ class AiterMultiStepDraftBackend:
         topk: int,
         speculative_num_steps: int,
     ):
-        from sglang.srt.speculative.eagle_utils import generate_draft_decode_kv_indices
+        from sglang.srt.speculative.spec_utils import generate_draft_decode_kv_indices
 
         self.topk = topk
         self.speculative_num_steps = speculative_num_steps
@@ -1065,7 +1064,7 @@ class AiterMultiStepDraftBackend:
             device=model_runner.device,
         )
         self.attn_backends = []
-        for i in range(self.speculative_num_steps):
+        for i in range(self.speculative_num_steps - 1):
             self.attn_backends.append(
                 AiterAttnBackend(
                     model_runner,
@@ -1108,7 +1107,7 @@ class AiterMultiStepDraftBackend:
             self.page_size,
         )
 
-        for i in range(self.speculative_num_steps):
+        for i in range(self.speculative_num_steps - 1):
             forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1]
             forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
                 : seq_lens_sum * self.topk + bs * (i + 1)
@@ -1142,7 +1141,7 @@ class AiterMultiStepDraftBackend:
             dtype=torch.int32,
             device=self.device,
         )
-        for i in range(self.speculative_num_steps):
+        for i in range(self.speculative_num_steps - 1):
             self.attn_backends[i].init_cuda_graph_state(
                 max_bs, max_num_tokens, kv_indices_buf=self.cuda_graph_kv_indices[i]
             )