skypilot-nightly 1.0.0.dev20250903__py3-none-any.whl → 1.0.0.dev20250905__py3-none-any.whl

sky/global_user_state.py

@@ -299,9 +299,7 @@ def create_table(engine: sqlalchemy.engine.Engine):
 # a session has already been created with _SQLALCHEMY_ENGINE = e1,
 # and then another thread overwrites _SQLALCHEMY_ENGINE = e2
 # which could result in e1 being garbage collected unexpectedly.
-def initialize_and_get_db(
-    pg_pool_class: Optional[sqlalchemy.pool.Pool] = None
-) -> sqlalchemy.engine.Engine:
+def initialize_and_get_db() -> sqlalchemy.engine.Engine:
     global _SQLALCHEMY_ENGINE
 
     if _SQLALCHEMY_ENGINE is not None:
@@ -310,8 +308,7 @@ def initialize_and_get_db(
         if _SQLALCHEMY_ENGINE is not None:
             return _SQLALCHEMY_ENGINE
         # get an engine to the db
-        engine = migration_utils.get_engine('state',
-                                            pg_pool_class=pg_pool_class)
+        engine = db_utils.get_engine('state')
 
         # run migrations if needed
         create_table(engine)
@@ -2315,3 +2312,18 @@ def set_system_config(config_key: str, config_value: str) -> None:
             })
         session.execute(upsert_stmnt)
         session.commit()
+
+
+@_init_db
+def get_max_db_connections() -> Optional[int]:
+    """Get the maximum number of connections for the engine."""
+    assert _SQLALCHEMY_ENGINE is not None
+    if (_SQLALCHEMY_ENGINE.dialect.name ==
+            db_utils.SQLAlchemyDialect.SQLITE.value):
+        return None
+    with sqlalchemy.orm.Session(_SQLALCHEMY_ENGINE) as session:
+        max_connections = session.execute(
+            sqlalchemy.text('SHOW max_connections')).scalar()
+        if max_connections is None:
+            return None
+        return int(max_connections)

sky/jobs/state.py

@@ -157,7 +157,7 @@ def initialize_and_get_db() -> sqlalchemy.engine.Engine:
         if _SQLALCHEMY_ENGINE is not None:
             return _SQLALCHEMY_ENGINE
         # get an engine to the db
-        engine = migration_utils.get_engine('spot_jobs')
+        engine = db_utils.get_engine('spot_jobs')
 
         # run migrations if needed
         create_table(engine)

sky/models.py

@@ -4,7 +4,7 @@ import collections
 import dataclasses
 import getpass
 import os
-from typing import Any, Dict, Optional
+from typing import Any, ClassVar, Dict, Optional
 
 import pydantic
 
@@ -100,6 +100,11 @@ class KubernetesNodesInfo:
 
 class VolumeConfig(pydantic.BaseModel):
     """Configuration for creating a volume."""
+    # If any fields changed, increment the version. For backward compatibility,
+    # modify the __setstate__ method to handle the old version.
+    _VERSION: ClassVar[int] = 1
+
+    _version: int
     name: str
     type: str
     cloud: str
@@ -110,3 +115,21 @@ class VolumeConfig(pydantic.BaseModel):
     config: Dict[str, Any] = {}
     labels: Optional[Dict[str, str]] = None
     id_on_cloud: Optional[str] = None
+
+    def __getstate__(self) -> Dict[str, Any]:
+        state = super().__getstate__()
+        state['_version'] = self._VERSION
+        return state
+
+    def __setstate__(self, state: Dict[str, Any]) -> None:
+        """Set state from pickled state, for backward compatibility."""
+        super().__setstate__(state)
+        version = state.pop('_version', None)
+        if version is None:
+            version = -1
+
+        if version < 0:
+            state['id_on_cloud'] = None
+
+        state['_version'] = self._VERSION
+        self.__dict__.update(state)

sky/provision/kubernetes/instance.py

@@ -1047,8 +1047,10 @@ def stop_instances(
     raise NotImplementedError()
 
 
-def _delete_services(name_prefix: str, namespace: str,
-                     context: Optional[str]) -> None:
+def _delete_services(name_prefix: str,
+                     namespace: str,
+                     context: Optional[str],
+                     skip_ssh_service: bool = False) -> None:
     """Delete services with the given name prefix.
 
     Args:
@@ -1057,7 +1059,9 @@ def _delete_services(name_prefix: str, namespace: str,
         context: Kubernetes context
     """
     # TODO(andy): We should use tag for the service filter.
-    for service_name in [name_prefix, f'{name_prefix}-ssh']:
+    services = ([name_prefix, f'{name_prefix}-ssh']
+                if not skip_ssh_service else [name_prefix])
+    for service_name in services:
         # Since we are not saving this lambda, it's a false positive.
         # TODO(andyl): Wait for
         # https://github.com/pylint-dev/pylint/issues/5263.
@@ -1083,6 +1087,9 @@ def _terminate_node(namespace: str,
         # Delete services for the head pod
         # services are specified in sky/templates/kubernetes-ray.yml.j2
         _delete_services(pod_name, namespace, context)
+    else:
+        # No ssh service is created for worker pods
+        _delete_services(pod_name, namespace, context, skip_ssh_service=True)
 
     # Note - delete pod after all other resources are deleted.
     # This is to ensure there are no leftover resources if this down is run

sky/serve/serve_state.py

@@ -130,7 +130,7 @@ def initialize_and_get_db() -> sqlalchemy.engine.Engine:
         if _SQLALCHEMY_ENGINE is not None:
             return _SQLALCHEMY_ENGINE
         # get an engine to the db
-        engine = migration_utils.get_engine('serve/services')
+        engine = db_utils.get_engine('serve/services')
 
         # run migrations if needed
         create_table(engine)

sky/server/config.py

@@ -2,6 +2,7 @@
 
 import dataclasses
 import enum
+from typing import Optional
 
 from sky import sky_logging
 from sky.server import constants as server_constants
@@ -61,6 +62,7 @@ class QueueBackend(enum.Enum):
 class WorkerConfig:
     garanteed_parallelism: int
     burstable_parallelism: int
+    num_db_connections_per_worker: int
 
 
 @dataclasses.dataclass
@@ -68,10 +70,13 @@ class ServerConfig:
     num_server_workers: int
     long_worker_config: WorkerConfig
     short_worker_config: WorkerConfig
+    num_db_connections_per_worker: int
     queue_backend: QueueBackend
 
 
-def compute_server_config(deploy: bool) -> ServerConfig:
+def compute_server_config(deploy: bool,
+                          max_db_connections: Optional[int] = None
+                         ) -> ServerConfig:
     """Compute the server config based on environment.
 
     We have different assumptions for the resources in different deployment
@@ -114,7 +119,17 @@ def compute_server_config(deploy: bool) -> ServerConfig:
     queue_backend = QueueBackend.MULTIPROCESSING
     burstable_parallel_for_long = 0
     burstable_parallel_for_short = 0
+    # if num_db_connections_per_worker is 0, server will use NullPool
+    # to conserve the number of concurrent db connections.
+    # This could lead to performance degradation.
+    num_db_connections_per_worker = 0
     num_server_workers = cpu_count
+
+    # +1 for the event loop running the main process
+    # and gc daemons in the '__main__' body of sky/server/server.py
+    max_parallel_all_workers = (max_parallel_for_long + max_parallel_for_short +
+                                num_server_workers + 1)
+
     if not deploy:
         # For local mode, use local queue backend since we only run 1 uvicorn
         # worker in local mode and no multiprocessing is needed.
@@ -140,6 +155,16 @@ def compute_server_config(deploy: bool) -> ServerConfig:
                 'SkyPilot API server will run in low resource mode because '
                 'the available memory is less than '
                 f'{server_constants.MIN_AVAIL_MEM_GB}GB.')
+    elif max_db_connections is not None:
+        if max_parallel_all_workers > max_db_connections:
+            logger.warning(
+                f'Max parallel all workers ({max_parallel_all_workers}) '
+                f'is greater than max db connections ({max_db_connections}). '
+                'Increase the number of max db connections to '
+                f'at least {max_parallel_all_workers} for optimal performance.')
+        else:
+            num_db_connections_per_worker = 1
+
     logger.info(
         f'SkyPilot API server will start {num_server_workers} server processes '
         f'with {max_parallel_for_long} background workers for long requests '
@@ -150,10 +175,13 @@ def compute_server_config(deploy: bool) -> ServerConfig:
         queue_backend=queue_backend,
         long_worker_config=WorkerConfig(
             garanteed_parallelism=max_parallel_for_long,
-            burstable_parallelism=burstable_parallel_for_long),
+            burstable_parallelism=burstable_parallel_for_long,
+            num_db_connections_per_worker=num_db_connections_per_worker),
         short_worker_config=WorkerConfig(
             garanteed_parallelism=max_parallel_for_short,
-            burstable_parallelism=burstable_parallel_for_short),
+            burstable_parallelism=burstable_parallel_for_short,
+            num_db_connections_per_worker=num_db_connections_per_worker),
+        num_db_connections_per_worker=num_db_connections_per_worker,
     )
 
 

sky/server/requests/executor.py

@@ -57,6 +57,7 @@ from sky.utils import subprocess_utils
 from sky.utils import tempstore
 from sky.utils import timeline
 from sky.utils import yaml_utils
+from sky.utils.db import db_utils
 from sky.workspaces import core as workspaces_core
 
 if typing.TYPE_CHECKING:
@@ -152,6 +153,8 @@ class RequestWorker:
         self.schedule_type = schedule_type
         self.garanteed_parallelism = config.garanteed_parallelism
         self.burstable_parallelism = config.burstable_parallelism
+        self.num_db_connections_per_worker = (
+            config.num_db_connections_per_worker)
         self._thread: Optional[threading.Thread] = None
         self._cancel_event = threading.Event()
 
@@ -190,8 +193,9 @@ class RequestWorker:
             # multiple requests can share the same process pid, which may cause
             # issues with SkyPilot core functions if they rely on the exit of
             # the process, such as subprocess_daemon.py.
-            fut = executor.submit_until_success(_request_execution_wrapper,
-                                                request_id, ignore_return_value)
+            fut = executor.submit_until_success(
+                _request_execution_wrapper, request_id, ignore_return_value,
+                self.num_db_connections_per_worker)
             # Monitor the result of the request execution.
             threading.Thread(target=self.handle_task_result,
                              args=(fut, request_element),
@@ -351,7 +355,8 @@ def _sigterm_handler(signum: int, frame: Optional['types.FrameType']) -> None:
 
 
 def _request_execution_wrapper(request_id: str,
-                               ignore_return_value: bool) -> None:
+                               ignore_return_value: bool,
+                               num_db_connections_per_worker: int = 0) -> None:
     """Wrapper for a request execution.
 
     It wraps the execution of a request to:
@@ -362,6 +367,7 @@ def _request_execution_wrapper(request_id: str,
     4. Handle the SIGTERM signal to abort the request gracefully.
     5. Maintain the lifecycle of the temp dir used by the request.
     """
+    db_utils.set_max_connections(num_db_connections_per_worker)
     # Handle the SIGTERM signal to abort the request processing gracefully.
     signal.signal(signal.SIGTERM, _sigterm_handler)
 

sky/server/requests/requests.py

@@ -1,5 +1,6 @@
 """Utilities for REST API."""
 import asyncio
+import atexit
 import contextlib
 import dataclasses
 import enum
@@ -16,6 +17,7 @@ import traceback
 from typing import (Any, AsyncContextManager, Callable, Dict, Generator, List,
                     NamedTuple, Optional, Tuple)
 
+import anyio
 import colorama
 import filelock
 
@@ -31,7 +33,6 @@ from sky.server.requests import payloads
 from sky.server.requests.serializers import decoders
 from sky.server.requests.serializers import encoders
 from sky.utils import common_utils
-from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 from sky.utils.db import db_utils
 
@@ -783,17 +784,15 @@ def set_request_cancelled(request_id: str) -> None:
 
 @init_db
 @metrics_lib.time_me
-def _delete_requests(requests: List[Request]):
+async def _delete_requests(requests: List[Request]):
     """Clean up requests by their IDs."""
     id_list_str = ','.join(repr(req.request_id) for req in requests)
     assert _DB is not None
-    with _DB.conn:
-        cursor = _DB.conn.cursor()
-        cursor.execute(
-            f'DELETE FROM {REQUEST_TABLE} WHERE request_id IN ({id_list_str})')
+    await _DB.execute_and_commit_async(
+        f'DELETE FROM {REQUEST_TABLE} WHERE request_id IN ({id_list_str})')
 
 
-def clean_finished_requests_with_retention(retention_seconds: int):
+async def clean_finished_requests_with_retention(retention_seconds: int):
     """Clean up finished requests older than the retention period.
 
     This function removes old finished requests (SUCCEEDED, FAILED, CANCELLED)
@@ -803,17 +802,19 @@ def clean_finished_requests_with_retention(retention_seconds: int):
         retention_seconds: Requests older than this many seconds will be
             deleted.
     """
-    reqs = get_request_tasks(
+    reqs = await get_request_tasks_async(
         req_filter=RequestTaskFilter(status=RequestStatus.finished_status(),
                                      finished_before=time.time() -
                                      retention_seconds))
 
-    subprocess_utils.run_in_parallel(
-        func=lambda req: req.log_path.unlink(missing_ok=True),
-        args=reqs,
-        num_threads=len(reqs))
+    futs = []
+    for req in reqs:
+        futs.append(
+            asyncio.create_task(
+                anyio.Path(req.log_path.absolute()).unlink(missing_ok=True)))
+    await asyncio.gather(*futs)
 
-    _delete_requests(reqs)
+    await _delete_requests(reqs)
 
     # To avoid leakage of the log file, logs must be deleted before the
     # request task in the database.
@@ -838,7 +839,16 @@ async def requests_gc_daemon():
             logger.info('Requests GC daemon cancelled')
             break
         except Exception as e:  # pylint: disable=broad-except
-            logger.error(f'Error running requests GC daemon: {e}')
+            logger.error(f'Error running requests GC daemon: {e}'
+                         f'traceback: {traceback.format_exc()}')
         # Run the daemon at most once every hour to avoid too frequent
         # cleanup.
         await asyncio.sleep(max(retention_seconds, 3600))
+
+
+def _cleanup():
+    if _DB is not None:
+        asyncio.run(_DB.close())
+
+
+atexit.register(_cleanup)

sky/server/server.py

@@ -24,7 +24,6 @@ import aiofiles
 import anyio
 import fastapi
 from fastapi.middleware import cors
-from sqlalchemy import pool
 import starlette.middleware.base
 import uvloop
 
@@ -72,6 +71,7 @@ from sky.utils import dag_utils
 from sky.utils import perf_utils
 from sky.utils import status_lib
 from sky.utils import subprocess_utils
+from sky.utils.db import db_utils
 from sky.volumes.server import server as volumes_rest
 from sky.workspaces import server as workspaces_rest
 
@@ -1322,18 +1322,17 @@ async def download(download_body: payloads.DownloadBody,
                                     detail=f'Error creating zip file: {str(e)}')
 
 
+# TODO(aylei): run it asynchronously after global_user_state support async op
 @app.post('/provision_logs')
-async def provision_logs(cluster_body: payloads.ClusterNameBody,
-                         follow: bool = True,
-                         tail: int = 0) -> fastapi.responses.StreamingResponse:
+def provision_logs(cluster_body: payloads.ClusterNameBody,
+                   follow: bool = True,
+                   tail: int = 0) -> fastapi.responses.StreamingResponse:
     """Streams the provision.log for the latest launch request of a cluster."""
     # Prefer clusters table first, then cluster_history as fallback.
-    log_path_str = await context_utils.to_thread(
-        global_user_state.get_cluster_provision_log_path,
+    log_path_str = global_user_state.get_cluster_provision_log_path(
         cluster_body.cluster_name)
     if not log_path_str:
-        log_path_str = await context_utils.to_thread(
-            global_user_state.get_cluster_history_provision_log_path,
+        log_path_str = global_user_state.get_cluster_history_provision_log_path(
             cluster_body.cluster_name)
     if not log_path_str:
         raise fastapi.HTTPException(
@@ -1908,13 +1907,6 @@ if __name__ == '__main__':
 
     skyuvicorn.add_timestamp_prefix_for_server_logs()
 
-    # Initialize global user state db
-    global_user_state.initialize_and_get_db(pool.QueuePool)
-    # Initialize request db
-    requests_lib.reset_db_and_logs()
-    # Restore the server user hash
-    _init_or_restore_server_user_hash()
-
     parser = argparse.ArgumentParser()
     parser.add_argument('--host', default='127.0.0.1')
     parser.add_argument('--port', default=46580, type=int)
@@ -1930,7 +1922,17 @@ if __name__ == '__main__':
     # that it is shown only when the API server is started.
     usage_lib.maybe_show_privacy_policy()
 
-    config = server_config.compute_server_config(cmd_args.deploy)
+    # Initialize global user state db
+    db_utils.set_max_connections(1)
+    global_user_state.initialize_and_get_db()
+    # Initialize request db
+    requests_lib.reset_db_and_logs()
+    # Restore the server user hash
+    _init_or_restore_server_user_hash()
+    max_db_connections = global_user_state.get_max_db_connections()
+    config = server_config.compute_server_config(cmd_args.deploy,
+                                                 max_db_connections)
+
     num_workers = config.num_server_workers
 
     queue_server: Optional[multiprocessing.Process] = None
@@ -1955,11 +1957,12 @@ if __name__ == '__main__':
         logger.info(f'Starting SkyPilot API server, workers={num_workers}')
         # We don't support reload for now, since it may cause leakage of request
         # workers or interrupt running requests.
-        config = uvicorn.Config('sky.server.server:app',
-                                host=cmd_args.host,
-                                port=cmd_args.port,
-                                workers=num_workers)
-        skyuvicorn.run(config)
+        uvicorn_config = uvicorn.Config('sky.server.server:app',
+                                        host=cmd_args.host,
+                                        port=cmd_args.port,
+                                        workers=num_workers)
+        skyuvicorn.run(uvicorn_config,
+                       max_db_connections=config.num_db_connections_per_worker)
     except Exception as exc:  # pylint: disable=broad-except
         logger.error(f'Failed to start SkyPilot API server: '
                      f'{common_utils.format_exception(exc, use_bracket=True)}')

sky/server/uvicorn.py

@@ -26,6 +26,7 @@ from sky.utils import context_utils
 from sky.utils import env_options
 from sky.utils import perf_utils
 from sky.utils import subprocess_utils
+from sky.utils.db import db_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -88,9 +89,12 @@ class Server(uvicorn.Server):
     - Run the server process with contextually aware.
     """
 
-    def __init__(self, config: uvicorn.Config):
+    def __init__(self,
+                 config: uvicorn.Config,
+                 max_db_connections: Optional[int] = None):
         super().__init__(config=config)
         self.exiting: bool = False
+        self.max_db_connections = max_db_connections
 
     def handle_exit(self, sig: int, frame: Union[FrameType, None]) -> None:
         """Handle exit signal.
@@ -196,6 +200,8 @@ class Server(uvicorn.Server):
 
     def run(self, *args, **kwargs):
         """Run the server process."""
+        if self.max_db_connections is not None:
+            db_utils.set_max_connections(self.max_db_connections)
         add_timestamp_prefix_for_server_logs()
         context_utils.hijack_sys_attrs()
         # Use default loop policy of uvicorn (use uvloop if available).
@@ -210,14 +216,14 @@ class Server(uvicorn.Server):
             asyncio.run(self.serve(*args, **kwargs))
 
 
-def run(config: uvicorn.Config):
+def run(config: uvicorn.Config, max_db_connections: Optional[int] = None):
     """Run unvicorn server."""
     if config.reload:
         # Reload and multi-workers are mutually exclusive
         # in uvicorn. Since we do not use reload now, simply
         # guard by an exception.
         raise ValueError('Reload is not supported yet.')
-    server = Server(config=config)
+    server = Server(config=config, max_db_connections=max_db_connections)
     try:
         if config.workers is not None and config.workers > 1:
             sock = config.bind_socket()

sky/skylet/constants.py

@@ -362,7 +362,7 @@ SKY_SSH_USER_PLACEHOLDER = 'skypilot:ssh_user'
 
 RCLONE_CONFIG_DIR = '~/.config/rclone'
 RCLONE_CONFIG_PATH = f'{RCLONE_CONFIG_DIR}/rclone.conf'
-RCLONE_LOG_DIR = '~/.sky/rclone_log'
+RCLONE_MOUNT_CACHED_LOG_DIR = '~/.sky/rclone_log'
 RCLONE_CACHE_DIR = '~/.cache/rclone'
 RCLONE_CACHE_REFRESH_INTERVAL = 10
 

sky/skypilot_config.py

@@ -227,7 +227,7 @@ def _get_config_from_path(path: Optional[str]) -> config_utils.Config:
     return parse_and_validate_config_file(path)
 
 
-def _resolve_user_config_path() -> Optional[str]:
+def resolve_user_config_path() -> Optional[str]:
     # find the user config file path, None if not resolved.
     user_config_path = _get_config_file_path(ENV_VAR_GLOBAL_CONFIG)
     if user_config_path:
@@ -252,7 +252,7 @@ def _resolve_user_config_path() -> Optional[str]:
 
 def get_user_config() -> config_utils.Config:
     """Returns the user config."""
-    return _get_config_from_path(_resolve_user_config_path())
+    return _get_config_from_path(resolve_user_config_path())
 
 
 def _resolve_project_config_path() -> Optional[str]:
@@ -574,8 +574,13 @@ def _reload_config_as_server() -> None:
                 'If db config is specified, no other config is allowed')
         logger.debug('retrieving config from database')
         with _DB_USE_LOCK:
-            sqlalchemy_engine = sqlalchemy.create_engine(db_url,
-                                                         poolclass=NullPool)
+            dispose_engine = False
+            if db_utils.get_max_connections() == 0:
+                dispose_engine = True
+                sqlalchemy_engine = sqlalchemy.create_engine(db_url,
+                                                             poolclass=NullPool)
+            else:
+                sqlalchemy_engine = db_utils.get_engine('config')
             db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata,
                                                      sqlalchemy_engine)
 
@@ -597,7 +602,8 @@ def _reload_config_as_server() -> None:
                 server_config = overlay_skypilot_config(server_config,
                                                         db_config)
             # Close the engine to avoid connection leaks
-            sqlalchemy_engine.dispose()
+            if dispose_engine:
+                sqlalchemy_engine.dispose()
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(f'server config: \n'
                      f'{yaml_utils.dump_yaml_str(dict(server_config))}')
@@ -611,7 +617,7 @@ def _reload_config_as_client() -> None:
     _set_loaded_config_path(None)
 
     overrides: List[config_utils.Config] = []
-    user_config_path = _resolve_user_config_path()
+    user_config_path = resolve_user_config_path()
     user_config = _get_config_from_path(user_config_path)
     if user_config:
         overrides.append(user_config)
@@ -867,8 +873,13 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
             raise ValueError('Cannot change db url while server is running')
         if existing_db_url:
             with _DB_USE_LOCK:
-                sqlalchemy_engine = sqlalchemy.create_engine(existing_db_url,
-                                                             poolclass=NullPool)
+                dispose_engine = False
+                if db_utils.get_max_connections() == 0:
+                    dispose_engine = True
+                    sqlalchemy_engine = sqlalchemy.create_engine(
+                        existing_db_url, poolclass=NullPool)
+                else:
+                    sqlalchemy_engine = db_utils.get_engine('config')
                 db_utils.add_all_tables_to_db_sqlalchemy(
                     Base.metadata, sqlalchemy_engine)
 
@@ -897,7 +908,8 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
                 _set_config_yaml_to_db(API_SERVER_CONFIG_KEY, config)
                 db_updated = True
                 # Close the engine to avoid connection leaks
-                sqlalchemy_engine.dispose()
+                if dispose_engine:
+                    sqlalchemy_engine.dispose()
 
     if not db_updated:
         # save to the local file (PVC in Kubernetes, local file otherwise)

sky/ssh_node_pools/server.py

@@ -15,7 +15,7 @@ router = fastapi.APIRouter()
 
 
 @router.get('')
-async def get_ssh_node_pools() -> Dict[str, Any]:
+def get_ssh_node_pools() -> Dict[str, Any]:
     """Get all SSH Node Pool configurations."""
     try:
         return ssh_node_pools_core.get_all_pools()
@@ -27,7 +27,7 @@ async def get_ssh_node_pools() -> Dict[str, Any]:
 
 
 @router.post('')
-async def update_ssh_node_pools(pools_config: Dict[str, Any]) -> Dict[str, str]:
+def update_ssh_node_pools(pools_config: Dict[str, Any]) -> Dict[str, str]:
     """Update SSH Node Pool configurations."""
     try:
         ssh_node_pools_core.update_pools(pools_config)
@@ -39,7 +39,7 @@ async def update_ssh_node_pools(pools_config: Dict[str, Any]) -> Dict[str, str]:
 
 
 @router.delete('/{pool_name}')
-async def delete_ssh_node_pool(pool_name: str) -> Dict[str, str]:
+def delete_ssh_node_pool(pool_name: str) -> Dict[str, str]:
     """Delete a SSH Node Pool configuration."""
     try:
         if ssh_node_pools_core.delete_pool(pool_name):
@@ -83,7 +83,7 @@ async def upload_ssh_key(request: fastapi.Request) -> Dict[str, str]:
 
 
 @router.get('/keys')
-async def list_ssh_keys() -> List[str]:
+def list_ssh_keys() -> List[str]:
     """List available SSH keys."""
     try:
         return ssh_node_pools_core.list_ssh_keys()
@@ -200,7 +200,7 @@ async def down_ssh_node_pool_general(
 
 
 @router.get('/{pool_name}/status')
-async def get_ssh_node_pool_status(pool_name: str) -> Dict[str, str]:
+def get_ssh_node_pool_status(pool_name: str) -> Dict[str, str]:
     """Get the status of a specific SSH Node Pool."""
     try:
         # Call ssh_status to check the context

sky/users/permission.py

@@ -226,6 +226,12 @@ class PermissionService:
         self._load_policy_no_lock()
         return self.enforcer.get_roles_for_user(user_id)
 
+    def get_users_for_role(self, role: str) -> List[str]:
+        """Get all users for a role."""
+        self._lazy_initialize()
+        self._load_policy_no_lock()
+        return self.enforcer.get_users_for_role(role)
+
     def check_endpoint_permission(self, user_id: str, path: str,
                                   method: str) -> bool:
         """Check permission."""