sglang 0.5.3rc0__py3-none-any.whl → 0.5.4__py3-none-any.whl

sglang/srt/grpc/health_servicer.py

@@ -0,0 +1,189 @@
+"""
+Standard gRPC health check service implementation for Kubernetes probes.
+
+This module implements the grpc.health.v1.Health service protocol, enabling
+native Kubernetes gRPC health probes for liveness and readiness checks.
+"""
+
+import logging
+import time
+from typing import AsyncIterator
+
+import grpc
+from grpc_health.v1 import health_pb2, health_pb2_grpc
+
+logger = logging.getLogger(__name__)
+
+
+class SGLangHealthServicer(health_pb2_grpc.HealthServicer):
+    """
+    Standard gRPC health check service implementation for Kubernetes probes.
+    Implements grpc.health.v1.Health protocol.
+
+    Supports two service levels:
+    1. Overall server health (service="") - for liveness probes
+    2. SGLang service health (service="sglang.grpc.scheduler.SglangScheduler") - for readiness probes
+
+    Health status lifecycle:
+    - NOT_SERVING: Initial state, model loading, or shutting down
+    - SERVING: Model loaded and ready to serve requests
+    """
+
+    # Service names we support
+    OVERALL_SERVER = ""  # Empty string for overall server health
+    SGLANG_SERVICE = "sglang.grpc.scheduler.SglangScheduler"
+
+    def __init__(self, request_manager, scheduler_info: dict):
+        """
+        Initialize health servicer.
+
+        Args:
+            request_manager: GrpcRequestManager instance for checking server state
+            scheduler_info: Dict containing scheduler metadata
+        """
+        self.request_manager = request_manager
+        self.scheduler_info = scheduler_info
+        self._serving_status = {}
+
+        # Initially set to NOT_SERVING until model is loaded
+        self._serving_status[self.OVERALL_SERVER] = (
+            health_pb2.HealthCheckResponse.NOT_SERVING
+        )
+        self._serving_status[self.SGLANG_SERVICE] = (
+            health_pb2.HealthCheckResponse.NOT_SERVING
+        )
+
+        logger.info("Standard gRPC health service initialized")
+
+    def set_serving(self):
+        """Mark services as SERVING - call this after model is loaded."""
+        self._serving_status[self.OVERALL_SERVER] = (
+            health_pb2.HealthCheckResponse.SERVING
+        )
+        self._serving_status[self.SGLANG_SERVICE] = (
+            health_pb2.HealthCheckResponse.SERVING
+        )
+        logger.info("Health service status set to SERVING")
+
+    def set_not_serving(self):
+        """Mark services as NOT_SERVING - call this during shutdown."""
+        self._serving_status[self.OVERALL_SERVER] = (
+            health_pb2.HealthCheckResponse.NOT_SERVING
+        )
+        self._serving_status[self.SGLANG_SERVICE] = (
+            health_pb2.HealthCheckResponse.NOT_SERVING
+        )
+        logger.info("Health service status set to NOT_SERVING")
+
+    async def Check(
+        self,
+        request: health_pb2.HealthCheckRequest,
+        context: grpc.aio.ServicerContext,
+    ) -> health_pb2.HealthCheckResponse:
+        """
+        Standard health check for Kubernetes probes.
+
+        Args:
+            request: Contains service name ("" for overall, or specific service)
+            context: gRPC context
+
+        Returns:
+            HealthCheckResponse with SERVING/NOT_SERVING/SERVICE_UNKNOWN status
+        """
+        service_name = request.service
+        logger.debug(f"Health check request for service: '{service_name}'")
+
+        # Check if shutting down
+        if self.request_manager.gracefully_exit:
+            logger.debug("Health check: Server is shutting down")
+            return health_pb2.HealthCheckResponse(
+                status=health_pb2.HealthCheckResponse.NOT_SERVING
+            )
+
+        # Overall server health - just check if process is alive
+        if service_name == self.OVERALL_SERVER:
+            status = self._serving_status.get(
+                self.OVERALL_SERVER, health_pb2.HealthCheckResponse.NOT_SERVING
+            )
+            logger.debug(
+                f"Overall health check: {health_pb2.HealthCheckResponse.ServingStatus.Name(status)}"
+            )
+            return health_pb2.HealthCheckResponse(status=status)
+
+        # Specific service health - check if ready to serve
+        elif service_name == self.SGLANG_SERVICE:
+            # Additional checks for service readiness
+
+            # Check base status first
+            base_status = self._serving_status.get(
+                self.SGLANG_SERVICE, health_pb2.HealthCheckResponse.NOT_SERVING
+            )
+
+            if base_status != health_pb2.HealthCheckResponse.SERVING:
+                logger.debug("Service health check: NOT_SERVING (base status)")
+                return health_pb2.HealthCheckResponse(status=base_status)
+
+            # Check if scheduler is responsive (received data recently)
+            time_since_last_receive = (
+                time.time() - self.request_manager.last_receive_tstamp
+            )
+
+            # If no recent activity and we have active requests, might be stuck
+            # NOTE: 30s timeout is hardcoded. This is more conservative than
+            # HEALTH_CHECK_TIMEOUT (20s) used for custom HealthCheck RPC.
+            # Consider making this configurable via environment variable in the future
+            # if different workloads need different responsiveness thresholds.
+            if (
+                time_since_last_receive > 30
+                and len(self.request_manager.rid_to_state) > 0
+            ):
+                logger.warning(
+                    f"Service health check: Scheduler not responsive "
+                    f"({time_since_last_receive:.1f}s since last receive, "
+                    f"{len(self.request_manager.rid_to_state)} pending requests)"
+                )
+                return health_pb2.HealthCheckResponse(
+                    status=health_pb2.HealthCheckResponse.NOT_SERVING
+                )
+
+            logger.debug("Service health check: SERVING")
+            return health_pb2.HealthCheckResponse(
+                status=health_pb2.HealthCheckResponse.SERVING
+            )
+
+        # Unknown service
+        else:
+            logger.debug(f"Health check for unknown service: '{service_name}'")
+            context.set_code(grpc.StatusCode.NOT_FOUND)
+            context.set_details(f"Unknown service: {service_name}")
+            return health_pb2.HealthCheckResponse(
+                status=health_pb2.HealthCheckResponse.SERVICE_UNKNOWN
+            )
+
+    async def Watch(
+        self,
+        request: health_pb2.HealthCheckRequest,
+        context: grpc.aio.ServicerContext,
+    ) -> AsyncIterator[health_pb2.HealthCheckResponse]:
+        """
+        Streaming health check - sends updates when status changes.
+
+        For now, just send current status once (Kubernetes doesn't use Watch).
+        A full implementation would monitor status changes and stream updates.
+
+        Args:
+            request: Contains service name
+            context: gRPC context
+
+        Yields:
+            HealthCheckResponse messages when status changes
+        """
+        service_name = request.service
+        logger.debug(f"Health watch request for service: '{service_name}'")
+
+        # Send current status
+        response = await self.Check(request, context)
+        yield response
+
+        # Note: Full Watch implementation would monitor status changes
+        # and stream updates. For K8s probes, Check is sufficient.

sglang/srt/grpc/scheduler_launcher.py

@@ -0,0 +1,181 @@
+"""
+Scheduler process management for gRPC server.
+
+This module handles launching and managing scheduler processes for the gRPC server,
+including tensor parallelism, pipeline parallelism, and data parallelism configurations.
+"""
+
+import logging
+import multiprocessing as mp
+import signal
+from typing import Dict, List, Optional, Tuple
+
+from sglang.srt.managers.data_parallel_controller import (
+    run_data_parallel_controller_process,
+)
+from sglang.srt.managers.scheduler import run_scheduler_process
+from sglang.srt.server_args import PortArgs, ServerArgs
+from sglang.srt.utils import configure_logger, prepare_model_and_tokenizer
+from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
+
+logger = logging.getLogger(__name__)
+
+
+def run_scheduler_with_signal_handling(*args, **kwargs):
+    """
+    Wrapper for run_scheduler_process that ignores SIGINT.
+
+    The scheduler process should not handle Ctrl+C - it should only terminate
+    when the parent gRPC server exits (via kill_itself_when_parent_died).
+
+    Args:
+        *args: Positional arguments for run_scheduler_process
+        **kwargs: Keyword arguments for run_scheduler_process
+    """
+    # Ignore SIGINT in this subprocess - let the parent handle it
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    # Now run the actual scheduler process
+    run_scheduler_process(*args, **kwargs)
+
+
+def launch_scheduler_process_only(
+    server_args: ServerArgs,
+    port_args: Optional[PortArgs] = None,
+) -> Tuple[Dict, PortArgs, List[mp.Process]]:
+    """
+    Launch only the scheduler process(es) without tokenizer/detokenizer.
+
+    This function handles all scheduler startup logic including:
+    - Tensor parallelism (tp_size)
+    - Pipeline parallelism (pp_size)
+    - Data parallelism (dp_size)
+    - Multi-node distributed setup
+
+    Args:
+        server_args: Server configuration
+        port_args: Port configuration (created if None)
+
+    Returns:
+        Tuple of (scheduler_info, port_args, scheduler_processes):
+        - scheduler_info: Dict with model metadata and configuration
+        - port_args: Port configuration used for IPC
+        - scheduler_processes: List of launched scheduler Process objects
+
+    Raises:
+        RuntimeError: If any scheduler process fails to initialize
+    """
+    # Configure global environment
+    configure_logger(server_args)
+    server_args.check_server_args()
+
+    # Fix CUDA multiprocessing issues - must be called before any CUDA operations
+    mp.set_start_method("spawn", force=True)
+
+    # Allocate ports for inter-process communications
+    if port_args is None:
+        port_args = PortArgs.init_new(server_args)
+        logger.info(f"{server_args=}")
+
+    # Prepare model and tokenizer paths
+    server_args.model_path, server_args.tokenizer_path = prepare_model_and_tokenizer(
+        server_args.model_path, server_args.tokenizer_path
+    )
+
+    scheduler_procs = []
+
+    if server_args.dp_size == 1:
+        # Single data parallel group - launch TP/PP schedulers
+        memory_saver_adapter = TorchMemorySaverAdapter.create(
+            enable=server_args.enable_memory_saver
+        )
+        scheduler_pipe_readers = []
+
+        # Calculate TP/PP distribution across nodes
+        nnodes_per_tp_group = max(server_args.nnodes // server_args.pp_size, 1)
+        tp_size_per_node = server_args.tp_size // nnodes_per_tp_group
+        tp_rank_range = range(
+            tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group),
+            tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group + 1),
+        )
+
+        pp_size_per_node = max(server_args.pp_size // server_args.nnodes, 1)
+        pp_rank_range = range(
+            pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group),
+            pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group + 1),
+        )
+
+        # Launch scheduler for each TP/PP rank combination
+        for pp_rank in pp_rank_range:
+            for tp_rank in tp_rank_range:
+                reader, writer = mp.Pipe(duplex=False)
+
+                # Calculate GPU ID for this rank
+                gpu_id = (
+                    server_args.base_gpu_id
+                    + ((pp_rank % pp_size_per_node) * tp_size_per_node)
+                    + (tp_rank % tp_size_per_node) * server_args.gpu_id_step
+                )
+
+                # Calculate MoE expert parallel rank
+                moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size)
+
+                # Create scheduler process
+                proc = mp.Process(
+                    target=run_scheduler_with_signal_handling,
+                    args=(
+                        server_args,
+                        port_args,
+                        gpu_id,
+                        tp_rank,
+                        moe_ep_rank,
+                        pp_rank,
+                        None,  # dp_rank
+                        writer,
+                    ),
+                )
+
+                with memory_saver_adapter.configure_subprocess():
+                    proc.start()
+
+                scheduler_procs.append(proc)
+                scheduler_pipe_readers.append(reader)
+    else:
+        # Data parallelism - launch data parallel controller
+        reader, writer = mp.Pipe(duplex=False)
+        scheduler_pipe_readers = [reader]
+
+        proc = mp.Process(
+            target=run_data_parallel_controller_process,
+            args=(server_args, port_args, writer),
+        )
+        proc.start()
+        scheduler_procs.append(proc)
+
+    # TODO(CatherineSue): handle cases for multi-node
+
+    # Wait for all scheduler processes to be ready
+    scheduler_infos = []
+    for i, reader in enumerate(scheduler_pipe_readers):
+        try:
+            data = reader.recv()
+        except EOFError:
+            logger.error(
+                f"Rank {i} scheduler is dead. Please check if there are relevant logs."
+            )
+            scheduler_procs[i].join()
+            logger.error(f"Exit code: {scheduler_procs[i].exitcode}")
+            raise RuntimeError(f"Failed to initialize scheduler rank {i}")
+
+        if data.get("status") != "ready":
+            raise RuntimeError(
+                f"Scheduler rank {i} initialization failed: {data.get('error', 'Unknown error')}"
+            )
+        scheduler_infos.append(data)
+
+    logger.info(
+        f"All {len(scheduler_procs)} scheduler process(es) initialized successfully"
+    )
+
+    # Return the first scheduler's info (they should all be the same)
+    return scheduler_infos[0], port_args, scheduler_procs

sglang/srt/grpc/sglang_scheduler_pb2.py

@@ -1,3 +1,6 @@
+# This file is auto-generated. Do not edit manually.
+# Regenerate with: python compile_proto.py
+
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
@@ -26,7 +29,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
 from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xc7\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x16\n\x0emax_new_tokens\x18\x08 \x01(\x05\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\x05\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x11\n\tlora_path\x18\x10 \x01(\t\x12\t\n\x01n\x18\x11 \x01(\x05\x12\x15\n\rtoken_healing\x18\x12 \x01(\x08\x12\x16\n\x0emin_new_tokens\x18\x13 \x01(\x05\x12\x12\n\nignore_eos\x18\x14 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x15 \x01(\x08\x12\x17\n\x0fstream_interval\x18\x16 \x01(\x05\x12H\n\nlogit_bias\x18\x17 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12\x16\n\x0estructural_tag\x18\x18 \x01(\t\x12.\n\rcustom_params\x18\x19 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraint\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe9\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\x05\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x15\n\rdp_balance_id\x18\x11 \x01(\x05\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\x05\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\xf5\x01\n\x13GenerateStreamChunk\x12\x10\n\x08token_id\x18\x01 \x01(\x05\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12\x31\n\x08logprobs\x18\x06 \x01(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12\x15\n\rhidden_states\x18\x07 \x03(\x02\x12\x17\n\x0fgeneration_time\x18\x08 \x01(\x02\x12\x12\n\nqueue_time\x18\t \x01(\x05\"\xcd\x02\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\x05\x12\x13\n\x0boutput_text\x18\x02 \x01(\t\x12K\n\rfinish_reason\x18\x03 \x01(\x0e\x32\x34.sglang.grpc.scheduler.GenerateComplete.FinishReason\x12\x35\n\x0c\x61ll_logprobs\x18\x0b \x03(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12>\n\x11\x61ll_hidden_states\x18\x0c \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\"L\n\x0c\x46inishReason\x12\x08\n\x04STOP\x10\x00\x12\n\n\x06LENGTH\x10\x01\x12\r\n\tEOS_TOKEN\x10\x02\x12\x0c\n\x08STOP_STR\x10\x03\x12\t\n\x05\x41\x42ORT\x10\x04\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"\x84\x01\n\x08LogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\x12\x13\n\x0btoken_texts\x18\x04 \x03(\t\"E\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x13\n\x0btoken_texts\x18\x03 \x03(\t\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xbc\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12\x17\n\x0fgeneration_time\x18\x05 \x01(\x02\x12:\n\x10\x62\x61tch_embeddings\x18\x06 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"N\n\x12HealthCheckRequest\x12\x38\n\ttokenized\x18\x01 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\xfe\x02\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponseb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x1b\n\x0emax_new_tokens\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\r\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x18\n\x0estructural_tag\x18\x10 \x01(\tH\x00\x12\t\n\x01n\x18\x11 \x01(\x05\x12\x16\n\x0emin_new_tokens\x18\x12 \x01(\x05\x12\x12\n\nignore_eos\x18\x13 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x14 \x01(\x08\x12\x1c\n\x0fstream_interval\x18\x15 \x01(\x05H\x02\x88\x01\x01\x12H\n\nlogit_bias\x18\x16 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12.\n\rcustom_params\x18\x17 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraintB\x11\n\x0f_max_new_tokensB\x12\n\x10_stream_interval\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe2\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\r\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x0e\n\x06stream\x18\x11 \x01(\x08\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\r\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\x95\x02\n\x13GenerateStreamChunk\x12\x11\n\ttoken_ids\x18\x01 \x03(\r\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x03 \x01(\x05\x12\x15\n\rcached_tokens\x18\x04 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12\x15\n\rhidden_states\x18\x06 \x03(\x02\x12<\n\x0einput_logprobs\x18\x07 \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x08 \x01(\r\"\x9b\x03\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\r\x12\x15\n\rfinish_reason\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12>\n\x0foutput_logprobs\x18\x06 \x01(\x0b\x32%.sglang.grpc.scheduler.OutputLogProbs\x12>\n\x11\x61ll_hidden_states\x18\x07 \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\x12\x1a\n\x10matched_token_id\x18\x08 \x01(\rH\x00\x12\x1a\n\x10matched_stop_str\x18\t \x01(\tH\x00\x12<\n\x0einput_logprobs\x18\n \x01(\x0b\x32$.sglang.grpc.scheduler.InputLogProbs\x12\r\n\x05index\x18\x0b \x01(\rB\x0e\n\x0cmatched_stop\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"u\n\x0eOutputLogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"\x9e\x01\n\rInputLogProbs\x12@\n\x0etoken_logprobs\x18\x01 \x03(\x0b\x32(.sglang.grpc.scheduler.InputTokenLogProb\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\"1\n\x11InputTokenLogProb\x12\x12\n\x05value\x18\x01 \x01(\x02H\x00\x88\x01\x01\x42\x08\n\x06_value\"0\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xa3\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12:\n\x10\x62\x61tch_embeddings\x18\x05 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"\x14\n\x12HealthCheckRequest\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x15\n\x13GetModelInfoRequest\"\xea\x02\n\x14GetModelInfoResponse\x12\x12\n\nmodel_path\x18\x01 \x01(\t\x12\x16\n\x0etokenizer_path\x18\x02 \x01(\t\x12\x15\n\ris_generation\x18\x03 \x01(\x08\x12!\n\x19preferred_sampling_params\x18\x04 \x01(\t\x12\x16\n\x0eweight_version\x18\x05 \x01(\t\x12\x19\n\x11served_model_name\x18\x06 \x01(\t\x12\x1a\n\x12max_context_length\x18\x07 \x01(\x05\x12\x12\n\nvocab_size\x18\x08 \x01(\x05\x12\x17\n\x0fsupports_vision\x18\t \x01(\x08\x12\x12\n\nmodel_type\x18\n \x01(\t\x12\x15\n\reos_token_ids\x18\x0b \x03(\x05\x12\x14\n\x0cpad_token_id\x18\x0c \x01(\x05\x12\x14\n\x0c\x62os_token_id\x18\r \x01(\x05\x12\x19\n\x11max_req_input_len\x18\x0e \x01(\x05\"\x16\n\x14GetServerInfoRequest\"\xb7\x02\n\x15GetServerInfoResponse\x12,\n\x0bserver_args\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\x12/\n\x0escheduler_info\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x17\n\x0f\x61\x63tive_requests\x18\x03 \x01(\x05\x12\x11\n\tis_paused\x18\x04 \x01(\x08\x12\x1e\n\x16last_receive_timestamp\x18\x05 \x01(\x01\x12\x16\n\x0euptime_seconds\x18\x06 \x01(\x01\x12\x16\n\x0esglang_version\x18\x07 \x01(\t\x12\x13\n\x0bserver_type\x18\x08 \x01(\t\x12.\n\nstart_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp2\xd3\x04\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponse\x12g\n\x0cGetModelInfo\x12*.sglang.grpc.scheduler.GetModelInfoRequest\x1a+.sglang.grpc.scheduler.GetModelInfoResponse\x12j\n\rGetServerInfo\x12+.sglang.grpc.scheduler.GetServerInfoRequest\x1a,.sglang.grpc.scheduler.GetServerInfoResponseb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -36,71 +39,81 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._loaded_options = None
   _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_options = b'8\001'
   _globals['_SAMPLINGPARAMS']._serialized_start=113
-  _globals['_SAMPLINGPARAMS']._serialized_end=824
-  _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=762
-  _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=810
-  _globals['_DISAGGREGATEDPARAMS']._serialized_start=826
-  _globals['_DISAGGREGATEDPARAMS']._serialized_end=919
-  _globals['_GENERATEREQUEST']._serialized_start=922
-  _globals['_GENERATEREQUEST']._serialized_end=1539
-  _globals['_TOKENIZEDINPUT']._serialized_start=1541
-  _globals['_TOKENIZEDINPUT']._serialized_end=1599
-  _globals['_MULTIMODALINPUTS']._serialized_start=1602
-  _globals['_MULTIMODALINPUTS']._serialized_end=1813
-  _globals['_GENERATERESPONSE']._serialized_start=1816
-  _globals['_GENERATERESPONSE']._serialized_end=2043
-  _globals['_GENERATESTREAMCHUNK']._serialized_start=2046
-  _globals['_GENERATESTREAMCHUNK']._serialized_end=2291
-  _globals['_GENERATECOMPLETE']._serialized_start=2294
-  _globals['_GENERATECOMPLETE']._serialized_end=2627
-  _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_start=2551
-  _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_end=2627
-  _globals['_GENERATEERROR']._serialized_start=2629
-  _globals['_GENERATEERROR']._serialized_end=2704
-  _globals['_LOGPROBS']._serialized_start=2707
-  _globals['_LOGPROBS']._serialized_end=2839
-  _globals['_TOPLOGPROBS']._serialized_start=2841
-  _globals['_TOPLOGPROBS']._serialized_end=2910
-  _globals['_HIDDENSTATES']._serialized_start=2912
-  _globals['_HIDDENSTATES']._serialized_end=2975
-  _globals['_EMBEDREQUEST']._serialized_start=2978
-  _globals['_EMBEDREQUEST']._serialized_end=3308
-  _globals['_EMBEDRESPONSE']._serialized_start=3311
-  _globals['_EMBEDRESPONSE']._serialized_end=3468
-  _globals['_EMBEDCOMPLETE']._serialized_start=3471
-  _globals['_EMBEDCOMPLETE']._serialized_end=3659
-  _globals['_EMBEDDING']._serialized_start=3661
-  _globals['_EMBEDDING']._serialized_end=3703
-  _globals['_EMBEDERROR']._serialized_start=3705
-  _globals['_EMBEDERROR']._serialized_end=3765
-  _globals['_HEALTHCHECKREQUEST']._serialized_start=3767
-  _globals['_HEALTHCHECKREQUEST']._serialized_end=3845
-  _globals['_HEALTHCHECKRESPONSE']._serialized_start=3847
-  _globals['_HEALTHCHECKRESPONSE']._serialized_end=3902
-  _globals['_ABORTREQUEST']._serialized_start=3904
-  _globals['_ABORTREQUEST']._serialized_end=3954
-  _globals['_ABORTRESPONSE']._serialized_start=3956
-  _globals['_ABORTRESPONSE']._serialized_end=4005
-  _globals['_LOADLORAREQUEST']._serialized_start=4007
-  _globals['_LOADLORAREQUEST']._serialized_end=4080
-  _globals['_LOADLORARESPONSE']._serialized_start=4082
-  _globals['_LOADLORARESPONSE']._serialized_end=4154
-  _globals['_UNLOADLORAREQUEST']._serialized_start=4156
-  _globals['_UNLOADLORAREQUEST']._serialized_end=4195
-  _globals['_UNLOADLORARESPONSE']._serialized_start=4197
-  _globals['_UNLOADLORARESPONSE']._serialized_end=4251
-  _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4253
-  _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4372
-  _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4374
-  _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4431
-  _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4433
-  _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4478
-  _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4480
-  _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4546
-  _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4548
-  _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4613
-  _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4615
-  _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4675
-  _globals['_SGLANGSCHEDULER']._serialized_start=4678
-  _globals['_SGLANGSCHEDULER']._serialized_end=5060
+  _globals['_SAMPLINGPARAMS']._serialized_end=833
+  _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=732
+  _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=780
+  _globals['_DISAGGREGATEDPARAMS']._serialized_start=835
+  _globals['_DISAGGREGATEDPARAMS']._serialized_end=928
+  _globals['_GENERATEREQUEST']._serialized_start=931
+  _globals['_GENERATEREQUEST']._serialized_end=1541
+  _globals['_TOKENIZEDINPUT']._serialized_start=1543
+  _globals['_TOKENIZEDINPUT']._serialized_end=1601
+  _globals['_MULTIMODALINPUTS']._serialized_start=1604
+  _globals['_MULTIMODALINPUTS']._serialized_end=1815
+  _globals['_GENERATERESPONSE']._serialized_start=1818
+  _globals['_GENERATERESPONSE']._serialized_end=2045
+  _globals['_GENERATESTREAMCHUNK']._serialized_start=2048
+  _globals['_GENERATESTREAMCHUNK']._serialized_end=2325
+  _globals['_GENERATECOMPLETE']._serialized_start=2328
+  _globals['_GENERATECOMPLETE']._serialized_end=2739
+  _globals['_GENERATEERROR']._serialized_start=2741
+  _globals['_GENERATEERROR']._serialized_end=2816
+  _globals['_OUTPUTLOGPROBS']._serialized_start=2818
+  _globals['_OUTPUTLOGPROBS']._serialized_end=2935
+  _globals['_INPUTLOGPROBS']._serialized_start=2938
+  _globals['_INPUTLOGPROBS']._serialized_end=3096
+  _globals['_INPUTTOKENLOGPROB']._serialized_start=3098
+  _globals['_INPUTTOKENLOGPROB']._serialized_end=3147
+  _globals['_TOPLOGPROBS']._serialized_start=3149
+  _globals['_TOPLOGPROBS']._serialized_end=3197
+  _globals['_HIDDENSTATES']._serialized_start=3199
+  _globals['_HIDDENSTATES']._serialized_end=3262
+  _globals['_EMBEDREQUEST']._serialized_start=3265
+  _globals['_EMBEDREQUEST']._serialized_end=3595
+  _globals['_EMBEDRESPONSE']._serialized_start=3598
+  _globals['_EMBEDRESPONSE']._serialized_end=3755
+  _globals['_EMBEDCOMPLETE']._serialized_start=3758
+  _globals['_EMBEDCOMPLETE']._serialized_end=3921
+  _globals['_EMBEDDING']._serialized_start=3923
+  _globals['_EMBEDDING']._serialized_end=3965
+  _globals['_EMBEDERROR']._serialized_start=3967
+  _globals['_EMBEDERROR']._serialized_end=4027
+  _globals['_HEALTHCHECKREQUEST']._serialized_start=4029
+  _globals['_HEALTHCHECKREQUEST']._serialized_end=4049
+  _globals['_HEALTHCHECKRESPONSE']._serialized_start=4051
+  _globals['_HEALTHCHECKRESPONSE']._serialized_end=4106
+  _globals['_ABORTREQUEST']._serialized_start=4108
+  _globals['_ABORTREQUEST']._serialized_end=4158
+  _globals['_ABORTRESPONSE']._serialized_start=4160
+  _globals['_ABORTRESPONSE']._serialized_end=4209
+  _globals['_LOADLORAREQUEST']._serialized_start=4211
+  _globals['_LOADLORAREQUEST']._serialized_end=4284
+  _globals['_LOADLORARESPONSE']._serialized_start=4286
+  _globals['_LOADLORARESPONSE']._serialized_end=4358
+  _globals['_UNLOADLORAREQUEST']._serialized_start=4360
+  _globals['_UNLOADLORAREQUEST']._serialized_end=4399
+  _globals['_UNLOADLORARESPONSE']._serialized_start=4401
+  _globals['_UNLOADLORARESPONSE']._serialized_end=4455
+  _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4457
+  _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4576
+  _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4578
+  _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4635
+  _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4637
+  _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4682
+  _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4684
+  _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4750
+  _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4752
+  _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4817
+  _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4819
+  _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4879
+  _globals['_GETMODELINFOREQUEST']._serialized_start=4881
+  _globals['_GETMODELINFOREQUEST']._serialized_end=4902
+  _globals['_GETMODELINFORESPONSE']._serialized_start=4905
+  _globals['_GETMODELINFORESPONSE']._serialized_end=5267
+  _globals['_GETSERVERINFOREQUEST']._serialized_start=5269
+  _globals['_GETSERVERINFOREQUEST']._serialized_end=5291
+  _globals['_GETSERVERINFORESPONSE']._serialized_start=5294
+  _globals['_GETSERVERINFORESPONSE']._serialized_end=5605
+  _globals['_SGLANGSCHEDULER']._serialized_start=5608
+  _globals['_SGLANGSCHEDULER']._serialized_end=6203
 # @@protoc_insertion_point(module_scope)