sglang 0.4.9.post6__py3-none-any.whl → 0.4.10.post1__py3-none-any.whl

sglang/srt/managers/template_manager.py

@@ -53,7 +53,7 @@ class TemplateManager:
     def __init__(self):
         self._chat_template_name: Optional[str] = None
         self._completion_template_name: Optional[str] = None
-        self._jinja_template_content_format: Optional[str] = None
+        self._jinja_template_content_format: Optional[str] = "openai"
 
     @property
     def chat_template_name(self) -> Optional[str]:
@@ -71,31 +71,60 @@ class TemplateManager:
         return self._jinja_template_content_format
 
     def load_chat_template(
-        self, tokenizer_manager, chat_template_arg: str, model_path: str
+        self, tokenizer_manager, chat_template_arg: Optional[str], model_path: str
     ) -> None:
         """
         Load a chat template from various sources.
 
         Args:
             tokenizer_manager: The tokenizer manager instance
-            chat_template_arg: Template name or file path
+            chat_template_arg: Template name, file path, or None to auto-detect
             model_path: Path to the model
         """
-        logger.info(f"Loading chat template: {chat_template_arg}")
+        if chat_template_arg:
+            self._load_explicit_chat_template(tokenizer_manager, chat_template_arg)
+        else:
+            # Try HuggingFace template first
+            hf_template = self._resolve_hf_chat_template(tokenizer_manager)
+            if hf_template:
+                self._jinja_template_content_format = (
+                    detect_jinja_template_content_format(hf_template)
+                )
+                logger.info(
+                    f"Using default HuggingFace chat template with detected content format: {self._jinja_template_content_format}"
+                )
+                return
 
-        if not chat_template_exists(chat_template_arg):
-            if not os.path.exists(chat_template_arg):
-                raise RuntimeError(
-                    f"Chat template {chat_template_arg} is not a built-in template name "
-                    "or a valid chat template file path."
+            # Fallback to SGLang template guessing
+            self.guess_chat_template_from_model_path(model_path)
+
+            # Set default format if no template was found
+            if self._chat_template_name is None:
+                self._jinja_template_content_format = "string"
+                logger.info(
+                    "No chat template found, defaulting to 'string' content format"
                 )
 
-            if chat_template_arg.endswith(".jinja"):
-                self._load_jinja_template(tokenizer_manager, chat_template_arg)
-            else:
-                self._load_json_chat_template(chat_template_arg)
-        else:
+    def _load_explicit_chat_template(
+        self, tokenizer_manager, chat_template_arg: str
+    ) -> None:
+        """Load explicitly specified chat template."""
+        logger.info(f"Loading chat template from argument: {chat_template_arg}")
+
+        if chat_template_exists(chat_template_arg):
             self._chat_template_name = chat_template_arg
+            return
+
+        if not os.path.exists(chat_template_arg):
+            raise RuntimeError(
+                f"Chat template {chat_template_arg} is not a built-in template name "
+                "or a valid chat template file path."
+            )
+
+        if chat_template_arg.endswith(".jinja"):
+            self._load_jinja_template(tokenizer_manager, chat_template_arg)
+        else:
+            self._load_json_chat_template(chat_template_arg)
 
     def guess_chat_template_from_model_path(self, model_path: str) -> None:
         """
@@ -146,10 +175,7 @@ class TemplateManager:
             completion_template: Optional completion template name/path
         """
         # Load chat template
-        if chat_template:
-            self.load_chat_template(tokenizer_manager, chat_template, model_path)
-        else:
-            self.guess_chat_template_from_model_path(model_path)
+        self.load_chat_template(tokenizer_manager, chat_template, model_path)
 
         # Load completion template
         if completion_template:
@@ -166,7 +192,7 @@ class TemplateManager:
             chat_template
         )
         logger.info(
-            f"Detected chat template content format: {self._jinja_template_content_format}"
+            f"Detected user specified Jinja chat template with content format: {self._jinja_template_content_format}"
         )
 
     def _load_json_chat_template(self, template_path: str) -> None:
@@ -224,3 +250,20 @@ class TemplateManager:
                 override=True,
             )
         self._completion_template_name = template["name"]
+
+    def _resolve_hf_chat_template(self, tokenizer_manager) -> Optional[str]:
+        """
+        Resolve HuggingFace chat template.
+
+        Returns the chat template string if found, None otherwise.
+        """
+        tokenizer = tokenizer_manager.tokenizer
+
+        # Try to get AutoTokenizer chat template
+        try:
+            return tokenizer.get_chat_template()
+        except Exception as e:
+            logger.debug(f"Error getting chat template via get_chat_template(): {e}")
+
+        logger.debug("No HuggingFace chat template found")
+        return None

sglang/srt/managers/tokenizer_manager.py

@@ -170,16 +170,6 @@ class ReqState:
     output_token_ids_logprobs_idx: List = dataclasses.field(default_factory=list)
 
 
-def _determine_tensor_transport_mode(server_args: ServerArgs) -> TensorTransportMode:
-    is_cross_node = server_args.dist_init_addr
-
-    if is_cross_node:
-        # Fallback to default CPU transport for multi-node
-        return "default"
-    else:
-        return "cuda_ipc"
-
-
 class TokenizerManager:
     """TokenizerManager is a process that tokenizes the text."""
 
@@ -199,16 +189,6 @@ class TokenizerManager:
             else None
         )
         self.crash_dump_folder = server_args.crash_dump_folder
-        self.crash_dump_performed = False  # Flag to ensure dump is only called once
-
-        # Init inter-process communication
-        context = zmq.asyncio.Context(2)
-        self.recv_from_detokenizer = get_zmq_socket(
-            context, zmq.PULL, port_args.tokenizer_ipc_name, True
-        )
-        self.send_to_scheduler = get_zmq_socket(
-            context, zmq.PUSH, port_args.scheduler_input_ipc_name, True
-        )
 
         # Read model args
         self.model_path = server_args.model_path
@@ -218,8 +198,7 @@ class TokenizerManager:
         self.is_image_gen = self.model_config.is_image_gen
         self.context_len = self.model_config.context_len
         self.image_token_id = self.model_config.image_token_id
-        self._updating = False
-        self._cond = asyncio.Condition()
+        self.max_req_input_len = None  # Will be set later in engine.py
 
         if self.model_config.is_multimodal:
             import_processors()
@@ -258,39 +237,57 @@ class TokenizerManager:
                     revision=server_args.revision,
                 )
 
-        # Initialize the `LoRARegistry` with initial LoRA adapter paths provided in `server_args`.
-        # The registry dynamically updates as adapters are loaded / unloaded during runtime. It
-        # serves as the source of truth for available adapters and maps user-friendly LoRA names
-        # to internally used unique LoRA IDs.
-        self.lora_registry = LoRARegistry(self.server_args.lora_paths or {})
+        # Init inter-process communication
+        context = zmq.asyncio.Context(2)
+        self.recv_from_detokenizer = get_zmq_socket(
+            context, zmq.PULL, port_args.tokenizer_ipc_name, True
+        )
+        self.send_to_scheduler = get_zmq_socket(
+            context, zmq.PUSH, port_args.scheduler_input_ipc_name, True
+        )
 
-        # Store states
+        # Request states
         self.no_create_loop = False
         self.rid_to_state: Dict[str, ReqState] = {}
+        self.asyncio_tasks = set()
+
+        # Health check
         self.health_check_failed = False
         self.gracefully_exit = False
         self.last_receive_tstamp = 0
+
+        # Dumping
         self.dump_requests_folder = ""  # By default do not dump
         self.dump_requests_threshold = 1000
         self.dump_request_list: List[Tuple] = []
-        self.crash_dump_request_list: deque[Tuple] = deque()
         self.log_request_metadata = self.get_log_request_metadata()
+        self.crash_dump_request_list: deque[Tuple] = deque()
+        self.crash_dump_performed = False  # Flag to ensure dump is only called once
+
+        # Session
         self.session_futures = {}  # session_id -> asyncio event
-        self.max_req_input_len = None
-        self.asyncio_tasks = set()
 
+        # Weight updates
         # The event to notify the weight sync is finished.
         self.model_update_lock = RWLock()
         self.model_update_result: Optional[Awaitable[UpdateWeightFromDiskReqOutput]] = (
             None
         )
+        self._is_updating = False
+        self._is_updating_cond = asyncio.Condition()
 
+        # LoRA
+        # Initialize the `LoRARegistry` with initial LoRA adapter paths provided in `server_args`.
+        # The registry dynamically updates as adapters are loaded / unloaded during runtime. It
+        # serves as the source of truth for available adapters and maps user-friendly LoRA names
+        # to internally used unique LoRA IDs.
+        self.lora_registry = LoRARegistry(self.server_args.lora_paths or {})
         # Lock to serialize LoRA update operations.
         # Please note that, unlike `model_update_lock`, this does not block inference, allowing
         # LoRA updates and inference to overlap.
         self.lora_update_lock = asyncio.Lock()
 
-        # For pd disaggregtion
+        # For PD disaggregtion
         self.disaggregation_mode = DisaggregationMode(
             self.server_args.disaggregation_mode
         )
@@ -458,17 +455,11 @@ class TokenizerManager:
         request: Optional[fastapi.Request] = None,
     ):
         created_time = time.time()
-        async with self._cond:
-            await self._cond.wait_for(lambda: not self._updating)
-
         self.auto_create_handle_loop()
         obj.normalize_batch_and_arguments()
 
-        if isinstance(obj, EmbeddingReqInput) and self.is_generation:
-            raise ValueError(
-                "This model does not appear to be an embedding model by default. "
-                "Please add `--is-embedding` when launching the server or try another model."
-            )
+        async with self._is_updating_cond:
+            await self._is_updating_cond.wait_for(lambda: not self._is_updating)
 
         if self.log_requests:
             max_length, skip_names, _ = self.log_request_metadata
@@ -567,6 +558,12 @@ class TokenizerManager:
                 f"model's context length ({self.context_len} tokens)."
             )
 
+        if isinstance(obj, EmbeddingReqInput) and self.is_generation:
+            raise ValueError(
+                "This model does not appear to be an embedding model by default. "
+                "Please add `--is-embedding` when launching the server or try another model."
+            )
+
         # Check total tokens (input + max_new_tokens)
         max_new_tokens = obj.sampling_params.get("max_new_tokens")
         if (
@@ -959,14 +956,14 @@ class TokenizerManager:
         await self.expert_distribution_communicator(ExpertDistributionReq.DUMP_RECORD)
 
     async def pause_generation(self):
-        async with self._cond:
-            self._updating = True
+        async with self._is_updating_cond:
+            self._is_updating = True
             self.abort_request(abort_all=True)
 
     async def continue_generation(self):
-        async with self._cond:
-            self._updating = False
-            self._cond.notify_all()
+        async with self._is_updating_cond:
+            self._is_updating = False
+            self._is_updating_cond.notify_all()
 
     async def update_weights_from_disk(
         self,
@@ -1208,14 +1205,6 @@ class TokenizerManager:
         # Many DP ranks
         return [res.internal_state for res in responses]
 
-    async def get_load(self) -> dict:
-        # TODO(lsyin): fake load report server
-        if not self.current_load_lock.locked():
-            async with self.current_load_lock:
-                internal_state = await self.get_internal_state()
-                self.current_load = internal_state[0]["load"]
-        return {"load": self.current_load}
-
     async def set_internal_state(
         self, obj: SetInternalStateReq
     ) -> SetInternalStateReqOutput:
@@ -1224,6 +1213,14 @@ class TokenizerManager:
         )
         return [res.internal_state for res in responses]
 
+    async def get_load(self) -> dict:
+        # TODO(lsyin): fake load report server
+        if not self.current_load_lock.locked():
+            async with self.current_load_lock:
+                internal_state = await self.get_internal_state()
+                self.current_load = internal_state[0]["load"]
+        return {"load": self.current_load}
+
     def get_log_request_metadata(self):
         max_length = None
         skip_names = None
@@ -1343,11 +1340,24 @@ class TokenizerManager:
                 "SIGTERM/SIGQUIT/Exception triggered, but crash dump already performed, skipping."
             )
             return
-        logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
-        self.crash_dump_performed = True
+
         if not self.crash_dump_folder:
             return
 
+        logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
+        self.crash_dump_performed = True
+
+        # Check if NFS directory is available
+        # expected_nfs_dir = "/" + self.crash_dump_folder.lstrip("/").split("/")[0]
+        # use_nfs_dir = os.path.isdir(expected_nfs_dir) and os.access(
+        #     expected_nfs_dir, os.W_OK
+        # )
+        use_nfs_dir = False
+        if not use_nfs_dir:
+            logger.error(
+                f"Expected NFS directory is not available or writable. Uploading to GCS."
+            )
+
         data_to_dump = []
         if self.crash_dump_request_list:
             data_to_dump.extend(self.crash_dump_request_list)
@@ -1357,7 +1367,12 @@ class TokenizerManager:
         for rid, state in self.rid_to_state.items():
             if not state.finished:
                 unfinished_requests.append(
-                    (state.obj, {}, state.created_time, time.time())
+                    (
+                        state.obj,
+                        state.out_list[-1] if state.out_list else {},
+                        state.created_time,
+                        time.time(),
+                    )
                 )
         if unfinished_requests:
             data_to_dump.extend(unfinished_requests)
@@ -1365,10 +1380,11 @@ class TokenizerManager:
         if not data_to_dump:
             return
 
+        object_name = f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl'
         filename = os.path.join(
             self.crash_dump_folder,
             os.getenv("HOSTNAME", None),
-            f"crash_dump_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.pkl",
+            object_name,
         )
 
         os.makedirs(os.path.dirname(filename), exist_ok=True)
@@ -1383,6 +1399,24 @@ class TokenizerManager:
             f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}"
         )
 
+        def _upload_file_to_gcs(bucket_name, source_file_path, object_name):
+            from google.cloud import storage
+
+            client = storage.Client()
+            bucket = client.bucket(bucket_name)
+            blob = bucket.blob(object_name)
+            blob.upload_from_filename(source_file_path, if_generation_match=0)
+            logger.error(
+                f"Successfully uploaded {source_file_path} to gs://{bucket_name}/{object_name}"
+            )
+
+        if not use_nfs_dir:
+            _upload_file_to_gcs(
+                "sglang_crash_dump",
+                filename,
+                os.getenv("HOSTNAME", None) + "/" + object_name,
+            )
+
     async def sigterm_watchdog(self):
         while not self.gracefully_exit:
             await asyncio.sleep(5)
@@ -1426,7 +1460,7 @@ class TokenizerManager:
         while True:
             recv_obj = await self.recv_from_detokenizer.recv_pyobj()
             self._result_dispatcher(recv_obj)
-            self.last_receive_tstamp = time.perf_counter()
+            self.last_receive_tstamp = time.time()
 
     def _handle_batch_output(
         self,
@@ -1697,24 +1731,13 @@ class TokenizerManager:
                 self.dump_requests_folder,
                 datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".pkl",
             )
-            logger.info(f"Dump {len(self.dump_request_list)} requests to {filename}")
-
-            to_dump = self.dump_request_list
+            self._dump_data_to_file(
+                data_list=self.dump_request_list,
+                filename=filename,
+                log_message=f"Dump {len(self.dump_request_list)} requests to {filename}",
+            )
             self.dump_request_list = []
 
-            to_dump_with_server_args = {
-                "server_args": self.server_args,
-                "requests": to_dump,
-            }
-
-            def background_task():
-                os.makedirs(self.dump_requests_folder, exist_ok=True)
-                with open(filename, "wb") as f:
-                    pickle.dump(to_dump_with_server_args, f)
-
-            # Schedule the task to run in the background without awaiting it
-            asyncio.create_task(asyncio.to_thread(background_task))
-
     def record_request_for_crash_dump(self, state: ReqState, out_dict: dict):
         current_time = time.time()
         self.crash_dump_request_list.append(
@@ -1727,6 +1750,22 @@ class TokenizerManager:
         ):
             self.crash_dump_request_list.popleft()
 
+    def _dump_data_to_file(
+        self, data_list: List[Tuple], filename: str, log_message: str
+    ):
+        logger.info(log_message)
+        to_dump_with_server_args = {
+            "server_args": self.server_args,
+            "requests": data_list.copy(),
+        }
+
+        def background_task():
+            os.makedirs(os.path.dirname(filename), exist_ok=True)
+            with open(filename, "wb") as f:
+                pickle.dump(to_dump_with_server_args, f)
+
+        asyncio.create_task(asyncio.to_thread(background_task))
+
     def _handle_abort_req(self, recv_obj):
         state = self.rid_to_state[recv_obj.rid]
         state.finished = True
@@ -1862,6 +1901,16 @@ class TokenizerManager:
         return scores
 
 
+def _determine_tensor_transport_mode(server_args: ServerArgs) -> TensorTransportMode:
+    is_cross_node = server_args.dist_init_addr
+
+    if is_cross_node:
+        # Fallback to default CPU transport for multi-node
+        return "default"
+    else:
+        return "cuda_ipc"
+
+
 async def print_exception_wrapper(func):
     """
     Sometimes an asyncio function does not print exception.

sglang/srt/managers/tp_worker.py

@@ -56,6 +56,7 @@ class TpModelWorker:
         server_args: ServerArgs,
         gpu_id: int,
         tp_rank: int,
+        moe_ep_rank: int,
         pp_rank: int,
         dp_rank: Optional[int],
         nccl_port: int,
@@ -66,6 +67,7 @@ class TpModelWorker:
         # Parse args
         self.tp_size = server_args.tp_size
         self.tp_rank = tp_rank
+        self.moe_ep_rank = moe_ep_rank
         self.pp_rank = pp_rank
 
         # Init model and tokenizer
@@ -85,6 +87,8 @@ class TpModelWorker:
             gpu_id=gpu_id,
             tp_rank=tp_rank,
             tp_size=server_args.tp_size,
+            moe_ep_rank=moe_ep_rank,
+            moe_ep_size=server_args.ep_size,
             pp_rank=pp_rank,
             pp_size=server_args.pp_size,
             nccl_port=nccl_port,

sglang/srt/managers/tp_worker_overlap_thread.py

@@ -58,13 +58,14 @@ class TpModelWorkerClient:
         server_args: ServerArgs,
         gpu_id: int,
         tp_rank: int,
+        moe_ep_rank: int,
         pp_rank: int,
         dp_rank: Optional[int],
         nccl_port: int,
     ):
         # Load the model
         self.worker = TpModelWorker(
-            server_args, gpu_id, tp_rank, pp_rank, dp_rank, nccl_port
+            server_args, gpu_id, tp_rank, moe_ep_rank, pp_rank, dp_rank, nccl_port
         )
         self.max_running_requests = self.worker.max_running_requests
         self.device = self.worker.device

sglang/srt/mem_cache/hicache_storage.py

@@ -2,7 +2,7 @@ import hashlib
 import logging
 import os
 from abc import ABC, abstractmethod
-from typing import List, Optional
+from typing import Any, List, Optional
 
 import torch
 
@@ -39,7 +39,10 @@ class HiCacheStorage(ABC):
 
     @abstractmethod
     def get(
-        self, key: str, target_location: Optional[torch.Tensor] = None
+        self,
+        key: str,
+        target_location: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
     ) -> torch.Tensor | None:
         """
         Retrieve the value associated with the given key.
@@ -49,7 +52,10 @@ class HiCacheStorage(ABC):
 
     @abstractmethod
     def batch_get(
-        self, keys: List[str], target_locations: Optional[List[torch.Tensor]] = None
+        self,
+        keys: List[str],
+        target_locations: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
     ) -> List[torch.Tensor | None]:
         """
         Retrieve values for multiple keys.
@@ -58,7 +64,13 @@ class HiCacheStorage(ABC):
         pass
 
     @abstractmethod
-    def set(self, key, value) -> bool:
+    def set(
+        self,
+        key: str,
+        value: Optional[Any] = None,
+        target_location: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
+    ) -> bool:
         """
         Store the value associated with the given key.
         Returns True if the operation was successful, False otherwise.
@@ -66,7 +78,13 @@ class HiCacheStorage(ABC):
         pass
 
     @abstractmethod
-    def batch_set(self, keys: List[str], values: List[torch.Tensor]) -> bool:
+    def batch_set(
+        self,
+        keys: List[str],
+        values: Optional[Any] = None,
+        target_locations: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
+    ) -> bool:
         """
         Store multiple key-value pairs.
         Returns True if all operations were successful, False otherwise.
@@ -74,7 +92,7 @@ class HiCacheStorage(ABC):
         pass
 
     @abstractmethod
-    def exists(self, key: str) -> bool:
+    def exists(self, key: str) -> bool | dict:
         """
         Check if the key exists in the storage.
         Returns True if the key exists, False otherwise.
@@ -85,7 +103,7 @@ class HiCacheStorage(ABC):
 class HiCacheFile(HiCacheStorage):
 
     def __init__(self, file_path: str = "/tmp/hicache"):
-        self.file_path = file_path
+        self.file_path = os.getenv("SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR", file_path)
         tp_rank = get_tensor_model_parallel_rank()
         tp_size = get_tensor_model_parallel_world_size()
         self.tp_suffix = f"_{tp_rank}_{tp_size}" if tp_size > 1 else ""
@@ -97,25 +115,38 @@ class HiCacheFile(HiCacheStorage):
         return key + self.tp_suffix
 
     def get(
-        self, key: str, target_location: Optional[torch.Tensor] = None
+        self,
+        key: str,
+        target_location: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
     ) -> torch.Tensor | None:
         key = self._get_suffixed_key(key)
         tensor_path = os.path.join(self.file_path, f"{key}.bin")
         try:
-            # todo: fixing the target_location logic to enable in-place loading
-            loaded_tensor = torch.load(tensor_path)
-            if isinstance(loaded_tensor, torch.Tensor):
-                return loaded_tensor
+            if target_location is not None:
+                # Load directly into target_location's memory buffer
+                with open(tensor_path, "rb") as f:
+                    target_location.set_(
+                        torch.frombuffer(f.read(), dtype=target_location.dtype)
+                        .reshape(target_location.shape)
+                        .storage()
+                    )
+                return target_location
             else:
-                logger.error(f"Loaded data for key {key} is not a tensor.")
-                return None
+                loaded_tensor = torch.load(tensor_path)
+                if isinstance(loaded_tensor, torch.Tensor):
+                    return loaded_tensor
+                else:
+                    logger.error(f"Loaded data for key {key} is not a tensor.")
+                    return None
         except FileNotFoundError:
             return None
 
     def batch_get(
         self,
         keys: List[str],
-        target_locations: Optional[List[torch.Tensor]] = None,
+        target_locations: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
     ) -> List[torch.Tensor | None]:
         return [
             self.get(key, target_location)
@@ -124,7 +155,13 @@ class HiCacheFile(HiCacheStorage):
             )
         ]
 
-    def set(self, key: str, value: torch.Tensor) -> bool:
+    def set(
+        self,
+        key: str,
+        value: Optional[Any] = None,
+        target_location: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
+    ) -> bool:
         key = self._get_suffixed_key(key)
         tensor_path = os.path.join(self.file_path, f"{key}.bin")
         if self.exists(key):
@@ -137,7 +174,13 @@ class HiCacheFile(HiCacheStorage):
             logger.error(f"Failed to save tensor {key}: {e}")
             return False
 
-    def batch_set(self, keys: List[str], values: List[torch.Tensor]) -> bool:
+    def batch_set(
+        self,
+        keys: List[str],
+        values: Optional[Any] = None,
+        target_locations: Optional[Any] = None,
+        target_sizes: Optional[Any] = None,
+    ) -> bool:
         for key, value in zip(keys, values):
             if not self.set(key, value):
                 return False