skypilot-nightly 1.0.0.dev20250311__py3-none-any.whl → 1.0.0.dev20250313__py3-none-any.whl

sky/server/requests/executor.py

@@ -27,8 +27,8 @@ import os
 import queue as queue_lib
 import signal
 import sys
+import threading
 import time
-import traceback
 import typing
 from typing import Any, Callable, Generator, List, Optional, TextIO, Tuple
 
@@ -41,11 +41,13 @@ from sky import skypilot_config
 from sky.server import common as server_common
 from sky.server import constants as server_constants
 from sky.server.requests import payloads
+from sky.server.requests import preconditions
 from sky.server.requests import requests as api_requests
 from sky.server.requests.queues import mp_queue
 from sky.skylet import constants
 from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
 
@@ -262,13 +264,7 @@ def _request_execution_wrapper(request_id: str,
             _restore_output(original_stdout, original_stderr)
             return
         except (Exception, SystemExit) as e:  # pylint: disable=broad-except
-            with ux_utils.enable_traceback():
-                stacktrace = traceback.format_exc()
-            setattr(e, 'stacktrace', stacktrace)
-            with api_requests.update_request(request_id) as request_task:
-                assert request_task is not None, request_id
-                request_task.status = api_requests.RequestStatus.FAILED
-                request_task.set_error(e)
+            api_requests.set_request_failed(request_id, e)
             _restore_output(original_stdout, original_stderr)
             logger.info(f'Request {request_id} failed due to '
                         f'{common_utils.format_exception(e)}')
@@ -283,16 +279,37 @@ def _request_execution_wrapper(request_id: str,
             logger.info(f'Request {request_id} finished')
 
 
-def schedule_request(request_id: str,
-                     request_name: str,
-                     request_body: payloads.RequestBody,
-                     func: Callable[P, Any],
-                     request_cluster_name: Optional[str] = None,
-                     ignore_return_value: bool = False,
-                     schedule_type: api_requests.ScheduleType = api_requests.
-                     ScheduleType.LONG,
-                     is_skypilot_system: bool = False) -> None:
-    """Enqueue a request to the request queue."""
+def schedule_request(
+        request_id: str,
+        request_name: str,
+        request_body: payloads.RequestBody,
+        func: Callable[P, Any],
+        request_cluster_name: Optional[str] = None,
+        ignore_return_value: bool = False,
+        schedule_type: api_requests.ScheduleType = (
+            api_requests.ScheduleType.LONG),
+        is_skypilot_system: bool = False,
+        precondition: Optional[preconditions.Precondition] = None) -> None:
+    """Enqueue a request to the request queue.
+
+    Args:
+        request_id: ID of the request.
+        request_name: Name of the request type, e.g. "sky.launch".
+        request_body: The request body containing parameters and environment
+            variables.
+        func: The function to execute when the request is processed.
+        request_cluster_name: The name of the cluster associated with this
+            request, if any.
+        ignore_return_value: If True, the return value of the function will be
+            ignored.
+        schedule_type: The type of scheduling to use for this request, refer to
+            `api_requests.ScheduleType` for more details.
+        is_skypilot_system: Denote whether the request is from SkyPilot system.
+        precondition: If a precondition is provided, the request will only be
+            scheduled for execution when the precondition is met (returns True).
+            The precondition is waited asynchronously and does not block the
+            caller.
+    """
     user_id = request_body.env_vars[constants.USER_ID_ENV_VAR]
     if is_skypilot_system:
         user_id = server_constants.SKYPILOT_SYSTEM_USER_ID
@@ -314,10 +331,17 @@ def schedule_request(request_id: str,
         return
 
     request.log_path.touch()
-    input_tuple = (request_id, ignore_return_value)
 
-    logger.info(f'Queuing request: {request_id}')
-    _get_queue(schedule_type).put(input_tuple)
+    def enqueue():
+        input_tuple = (request_id, ignore_return_value)
+        logger.info(f'Queuing request: {request_id}')
+        _get_queue(schedule_type).put(input_tuple)
+
+    if precondition is not None:
+        # Wait async to avoid blocking caller.
+        precondition.wait_async(on_condition_met=enqueue)
+    else:
+        enqueue()
 
 
 def executor_initializer(proc_group: str):
@@ -431,13 +455,17 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
 
     logger.info('Request queues created')
 
+    long_workers = []
     for worker_id in range(max_parallel_for_long):
         worker = RequestWorker(id=worker_id,
                                schedule_type=api_requests.ScheduleType.LONG)
         worker_proc = multiprocessing.Process(target=request_worker,
                                               args=(worker, 1))
-        worker_proc.start()
+        long_workers.append(worker_proc)
         sub_procs.append(worker_proc)
+    threading.Thread(target=subprocess_utils.slow_start_processes,
+                     args=(long_workers,),
+                     daemon=True).start()
 
     # Start a worker for short requests.
     worker = RequestWorker(id=1, schedule_type=api_requests.ScheduleType.SHORT)

sky/server/requests/preconditions.py

@@ -0,0 +1,174 @@
+"""Precondition for a request to be executed.
+
+Preconditions are introduced so that:
+- Wait for precondition does not block executor process, which is expensive;
+- Cross requests knowledge (e.g. waiting for other requests to be completed)
+  can be handled at precondition level, instead of invading the execution
+  logic of specific requests.
+"""
+import abc
+import asyncio
+import time
+from typing import Callable, Optional, Tuple
+
+from sky import exceptions
+from sky import global_user_state
+from sky import sky_logging
+from sky.server.requests import event_loop
+from sky.server.requests import requests as api_requests
+from sky.utils import common_utils
+from sky.utils import status_lib
+
+# The default interval seconds to check the precondition.
+_PRECONDITION_CHECK_INTERVAL = 1
+# The default timeout seconds to wait for the precondition to be met.
+_PRECONDITION_TIMEOUT = 60 * 60
+
+logger = sky_logging.init_logger(__name__)
+
+
+class Precondition(abc.ABC):
+    """Abstract base class for a precondition for a request to be executed.
+
+    A Precondition can be waited in either of the following ways:
+    - await Precondition: wait for the precondition to be met.
+    - Precondition.wait_async: wait for the precondition to be met in background
+      and execute the given callback on met.
+    """
+
+    def __init__(self,
+                 request_id: str,
+                 check_interval: float = _PRECONDITION_CHECK_INTERVAL,
+                 timeout: float = _PRECONDITION_TIMEOUT):
+        self.request_id = request_id
+        self.check_interval = check_interval
+        self.timeout = timeout
+
+    def __await__(self):
+        """Make Precondition awaitable."""
+        return self._wait().__await__()
+
+    def wait_async(
+            self,
+            on_condition_met: Optional[Callable[[], None]] = None) -> None:
+        """Wait precondition asynchronously and execute the callback on met."""
+
+        async def wait_with_callback():
+            met = await self
+            if met and on_condition_met is not None:
+                on_condition_met()
+
+        event_loop.run(wait_with_callback())
+
+    @abc.abstractmethod
+    async def check(self) -> Tuple[bool, Optional[str]]:
+        """Check if the precondition is met.
+
+        Note that compared to _request_execution_wrapper, the env vars and
+        skypilot config here are not overridden since the lack of process
+        isolation, which may cause issues if the check accidentally depends on
+        these. Make sure the check function is independent of the request
+        environment.
+        TODO(aylei): a new request context isolation mechanism is needed to
+        enable more tasks/sub-tasks to be processed in coroutines or threads.
+
+        Returns:
+            A tuple of (bool, Optional[str]).
+            The bool indicates if the precondition is met.
+            The str is the current status of the precondition if any.
+        """
+        raise NotImplementedError
+
+    async def _wait(self) -> bool:
+        """Wait for the precondition to be met.
+
+        Args:
+            on_condition_met: Callback to execute when the precondition is met.
+        """
+        start_time = time.time()
+        last_status_msg = ''
+        while True:
+            if self.timeout > 0 and time.time() - start_time > self.timeout:
+                # Cancel the request on timeout.
+                api_requests.set_request_failed(
+                    self.request_id,
+                    exceptions.RequestCancelled(
+                        f'Request {self.request_id} precondition wait timed '
+                        f'out after {self.timeout}s'))
+                return False
+
+            # Check if the request has been cancelled
+            request = api_requests.get_request(self.request_id)
+            if request is None:
+                logger.error(f'Request {self.request_id} not found')
+                return False
+            if request.status == api_requests.RequestStatus.CANCELLED:
+                logger.debug(f'Request {self.request_id} cancelled')
+                return False
+
+            try:
+                met, status_msg = await self.check()
+                if met:
+                    return True
+                if status_msg is not None and status_msg != last_status_msg:
+                    # Update the status message if it has changed.
+                    with api_requests.update_request(self.request_id) as req:
+                        assert req is not None, self.request_id
+                        req.status_msg = status_msg
+                    last_status_msg = status_msg
+            except (Exception, SystemExit, KeyboardInterrupt) as e:  # pylint: disable=broad-except
+                api_requests.set_request_failed(self.request_id, e)
+                logger.info(f'Request {self.request_id} failed due to '
+                            f'{common_utils.format_exception(e)}')
+                return False
+
+            await asyncio.sleep(self.check_interval)
+
+
+class ClusterStartCompletePrecondition(Precondition):
+    """Whether the start process of a cluster is complete.
+
+    This condition only waits the start process of a cluster to complete, e.g.
+    `sky launch` or `sky start`.
+    For cluster that has been started but not in UP status, bypass the waiting
+    in favor of:
+    - allowing the task to refresh cluster status from cloud vendor;
+    - unified error message in task handlers.
+
+    Args:
+        request_id: The request ID of the task.
+        cluster_name: The name of the cluster to wait for.
+    """
+
+    def __init__(self, request_id: str, cluster_name: str, **kwargs):
+        super().__init__(request_id=request_id, **kwargs)
+        self.cluster_name = cluster_name
+
+    async def check(self) -> Tuple[bool, Optional[str]]:
+        cluster_record = global_user_state.get_cluster_from_name(
+            self.cluster_name)
+        if (cluster_record and
+                cluster_record['status'] is status_lib.ClusterStatus.UP):
+            # Shortcut for started clusters, ignore cluster not found
+            # since the cluster record might not yet be created by the
+            # launch task.
+            return True, None
+        # Check if there is a task starting the cluster, we do not check
+        # SUCCEEDED requests since successfully launched cluster can be
+        # restarted later on.
+        # Note that since the requests are not persistent yet between restarts,
+        # a cluster might be started in halfway and requests are lost.
+        # We unify these situations into a single state: the process of starting
+        # the cluster is done (either normally or abnormally) but cluster is not
+        # in UP status.
+        requests = api_requests.get_request_tasks(
+            status=[
+                api_requests.RequestStatus.RUNNING,
+                api_requests.RequestStatus.PENDING
+            ],
+            include_request_names=['sky.launch', 'sky.start'],
+            cluster_names=[self.cluster_name])
+        if len(requests) == 0:
+            # No runnning or pending tasks, the start process is done.
+            return True, None
+        return False, f'Waiting for cluster {self.cluster_name} to be UP.'

sky/server/requests/requests.py

@@ -10,6 +10,7 @@ import shutil
 import signal
 import sqlite3
 import time
+import traceback
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import colorama
@@ -27,6 +28,7 @@ from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import db_utils
 from sky.utils import env_options
+from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -34,6 +36,7 @@ logger = sky_logging.init_logger(__name__)
 REQUEST_TABLE = 'requests'
 COL_CLUSTER_NAME = 'cluster_name'
 COL_USER_ID = 'user_id'
+COL_STATUS_MSG = 'status_msg'
 REQUEST_LOG_PATH_PREFIX = '~/sky_logs/api_server/requests'
 
 # TODO(zhwu): For scalability, there are several TODOs:
@@ -81,6 +84,7 @@ REQUEST_COLUMNS = [
     COL_CLUSTER_NAME,
     'schedule_type',
     COL_USER_ID,
+    COL_STATUS_MSG,
 ]
 
 
@@ -109,6 +113,7 @@ class RequestPayload:
     user_name: Optional[str] = None
     # Resources the request operates on.
     cluster_name: Optional[str] = None
+    status_msg: Optional[str] = None
 
 
 @dataclasses.dataclass
@@ -129,6 +134,8 @@ class Request:
     schedule_type: ScheduleType = ScheduleType.LONG
     # Resources the request operates on.
     cluster_name: Optional[str] = None
+    # Status message of the request, indicates the reason of current status.
+    status_msg: Optional[str] = None
 
     @property
     def log_path(self) -> pathlib.Path:
@@ -138,7 +145,7 @@ class Request:
         log_path = (log_path_prefix / self.request_id).with_suffix('.log')
         return log_path
 
-    def set_error(self, error: Exception) -> None:
+    def set_error(self, error: BaseException) -> None:
         """Set the error."""
         # TODO(zhwu): pickle.dump does not work well with custom exceptions if
         # it has more than 1 arguments.
@@ -212,6 +219,7 @@ class Request:
             user_id=self.user_id,
             user_name=user_name,
             cluster_name=self.cluster_name,
+            status_msg=self.status_msg,
         )
 
     def encode(self) -> RequestPayload:
@@ -232,6 +240,7 @@ class Request:
                 schedule_type=self.schedule_type.value,
                 user_id=self.user_id,
                 cluster_name=self.cluster_name,
+                status_msg=self.status_msg,
             )
         except (TypeError, ValueError) as e:
             # The error is unexpected, so we don't suppress the stack trace.
@@ -262,6 +271,7 @@ class Request:
                 schedule_type=ScheduleType(payload.schedule_type),
                 user_id=payload.user_id,
                 cluster_name=payload.cluster_name,
+                status_msg=payload.status_msg,
             )
         except (TypeError, ValueError) as e:
             logger.error(
@@ -415,7 +425,8 @@ def create_table(cursor, conn):
         pid INTEGER,
         {COL_CLUSTER_NAME} TEXT,
         schedule_type TEXT,
-        {COL_USER_ID} TEXT)""")
+        {COL_USER_ID} TEXT,
+        {COL_STATUS_MSG} TEXT)""")
 
 
 _DB = None
@@ -436,7 +447,7 @@ def init_db(func):
 
 def reset_db_and_logs():
     """Create the database."""
-    common_utils.remove_file_if_exists(_DB_PATH)
+    server_common.clear_local_api_server_database()
     shutil.rmtree(pathlib.Path(REQUEST_LOG_PATH_PREFIX).expanduser(),
                   ignore_errors=True)
     shutil.rmtree(server_common.API_SERVER_CLIENT_DIR.expanduser(),
@@ -507,8 +518,9 @@ def create_if_not_exists(request: Request) -> bool:
 def get_request_tasks(
     status: Optional[List[RequestStatus]] = None,
     cluster_names: Optional[List[str]] = None,
-    exclude_request_names: Optional[List[str]] = None,
     user_id: Optional[str] = None,
+    exclude_request_names: Optional[List[str]] = None,
+    include_request_names: Optional[List[str]] = None,
 ) -> List[Request]:
     """Get a list of requests that match the given filters.
 
@@ -516,9 +528,21 @@ def get_request_tasks(
         status: a list of statuses of the requests to filter on.
         cluster_names: a list of cluster names to filter requests on.
         exclude_request_names: a list of request names to exclude from results.
+            Mutually exclusive with include_request_names.
         user_id: the user ID to filter requests on.
             If None, all users are included.
+        include_request_names: a list of request names to filter on.
+            Mutually exclusive with exclude_request_names.
+
+    Raises:
+        ValueError: If both exclude_request_names and include_request_names are
+            provided.
     """
+    if exclude_request_names is not None and include_request_names is not None:
+        raise ValueError(
+            'Only one of exclude_request_names or include_request_names can be '
+            'provided, not both.')
+
     filters = []
     filter_params = []
     if status is not None:
@@ -534,6 +558,10 @@ def get_request_tasks(
     if user_id is not None:
         filters.append(f'{COL_USER_ID} = ?')
         filter_params.append(user_id)
+    if include_request_names is not None:
+        request_names_str = ','.join(
+            repr(name) for name in include_request_names)
+        filters.append(f'name IN ({request_names_str})')
     assert _DB is not None
     with _DB.conn:
         cursor = _DB.conn.cursor()
@@ -565,3 +593,14 @@ def _add_or_update_request_no_lock(request: Request):
         cursor.execute(
             f'INSERT OR REPLACE INTO {REQUEST_TABLE} ({key_str}) '
             f'VALUES ({fill_str})', row)
+
+
+def set_request_failed(request_id: str, e: BaseException) -> None:
+    """Set a request to failed and populate the error message."""
+    with ux_utils.enable_traceback():
+        stacktrace = traceback.format_exc()
+    setattr(e, 'stacktrace', stacktrace)
+    with update_request(request_id) as request_task:
+        assert request_task is not None, request_id
+        request_task.status = RequestStatus.FAILED
+        request_task.set_error(e)

sky/server/server.py

@@ -6,6 +6,7 @@ import contextlib
 import dataclasses
 import datetime
 import logging
+import multiprocessing
 import os
 import pathlib
 import re
@@ -38,6 +39,7 @@ from sky.server import constants as server_constants
 from sky.server import stream_utils
 from sky.server.requests import executor
 from sky.server.requests import payloads
+from sky.server.requests import preconditions
 from sky.server.requests import requests as requests_lib
 from sky.skylet import constants
 from sky.usage import usage_lib
@@ -47,6 +49,7 @@ from sky.utils import common_utils
 from sky.utils import dag_utils
 from sky.utils import env_options
 from sky.utils import status_lib
+from sky.utils import subprocess_utils
 
 # pylint: disable=ungrouped-imports
 if sys.version_info >= (3, 10):
@@ -496,13 +499,18 @@ async def launch(launch_body: payloads.LaunchBody,
 # pylint: disable=redefined-builtin
 async def exec(request: fastapi.Request, exec_body: payloads.ExecBody) -> None:
     """Executes a task on an existing cluster."""
+    cluster_name = exec_body.cluster_name
     executor.schedule_request(
         request_id=request.state.request_id,
         request_name='exec',
         request_body=exec_body,
         func=execution.exec,
+        precondition=preconditions.ClusterStartCompletePrecondition(
+            request_id=request.state.request_id,
+            cluster_name=cluster_name,
+        ),
         schedule_type=requests_lib.ScheduleType.LONG,
-        request_cluster_name=exec_body.cluster_name,
+        request_cluster_name=cluster_name,
     )
 
 
@@ -1088,6 +1096,9 @@ async def complete_storage_name(incomplete: str,) -> List[str]:
 
 if __name__ == '__main__':
     import uvicorn
+
+    from sky.server import uvicorn as skyuvicorn
+
     requests_lib.reset_db_and_logs()
 
     parser = argparse.ArgumentParser()
@@ -1109,16 +1120,26 @@ if __name__ == '__main__':
         logger.info(f'Starting SkyPilot API server, workers={num_workers}')
         # We don't support reload for now, since it may cause leakage of request
         # workers or interrupt running requests.
-        uvicorn.run('sky.server.server:app',
-                    host=cmd_args.host,
-                    port=cmd_args.port,
-                    workers=num_workers)
+        config = uvicorn.Config('sky.server.server:app',
+                                host=cmd_args.host,
+                                port=cmd_args.port,
+                                workers=num_workers)
+        skyuvicorn.run(config)
     except Exception as exc:  # pylint: disable=broad-except
         logger.error(f'Failed to start SkyPilot API server: '
                      f'{common_utils.format_exception(exc, use_bracket=True)}')
         raise
     finally:
         logger.info('Shutting down SkyPilot API server...')
-        for sub_proc in sub_procs:
-            sub_proc.terminate()
-            sub_proc.join()
+
+        def cleanup(proc: multiprocessing.Process) -> None:
+            try:
+                proc.terminate()
+                proc.join()
+            finally:
+                # The process may not be started yet, close it anyway.
+                proc.close()
+
+        subprocess_utils.run_in_parallel(cleanup,
+                                         sub_procs,
+                                         num_threads=len(sub_procs))

sky/server/stream_utils.py

@@ -55,19 +55,22 @@ async def log_streamer(request_id: Optional[str],
         if show_request_waiting_spinner:
             yield status_msg.init()
             yield status_msg.start()
-        is_waiting_msg_logged = False
+        last_waiting_msg = ''
         waiting_msg = (f'Waiting for {request_task.name!r} request to be '
                        f'scheduled: {request_id}')
         while request_task.status < requests_lib.RequestStatus.RUNNING:
+            if request_task.status_msg is not None:
+                waiting_msg = request_task.status_msg
             if show_request_waiting_spinner:
                 yield status_msg.update(f'[dim]{waiting_msg}[/dim]')
-            elif plain_logs and not is_waiting_msg_logged:
-                is_waiting_msg_logged = True
+            elif plain_logs and waiting_msg != last_waiting_msg:
+                # Only log when waiting message changes.
+                last_waiting_msg = waiting_msg
                 # Use smaller padding (1024 bytes) to force browser rendering
                 yield f'{waiting_msg}' + ' ' * 4096 + '\n'
-            # Sleep 0 to yield, so other coroutines can run. This busy waiting
-            # loop is performance critical for short-running requests, so we do
-            # not want to yield too long.
+            # Sleep shortly to avoid storming the DB and CPU and allow other
+            # coroutines to run. This busy waiting loop is performance critical
+            # for short-running requests, so we do not want to yield too long.
             await asyncio.sleep(0.1)
             request_task = requests_lib.get_request(request_id)
             if not follow:

sky/server/uvicorn.py

@@ -0,0 +1,81 @@
+"""Uvicorn wrapper for SkyPilot API server.
+
+This module is a wrapper around uvicorn to customize the behavior of the
+server.
+"""
+import os
+import threading
+from typing import Optional
+
+import uvicorn
+from uvicorn.supervisors import multiprocess
+
+from sky.utils import subprocess_utils
+
+
+def run(config: uvicorn.Config):
+    """Run unvicorn server."""
+    if config.reload:
+        # Reload and multi-workers are mutually exclusive
+        # in uvicorn. Since we do not use reload now, simply
+        # guard by an exception.
+        raise ValueError('Reload is not supported yet.')
+    server = uvicorn.Server(config=config)
+    try:
+        if config.workers is not None and config.workers > 1:
+            sock = config.bind_socket()
+            SlowStartMultiprocess(config, target=server.run,
+                                  sockets=[sock]).run()
+        else:
+            server.run()
+    finally:
+        # Copied from unvicorn.run()
+        if config.uds and os.path.exists(config.uds):
+            os.remove(config.uds)
+
+
+class SlowStartMultiprocess(multiprocess.Multiprocess):
+    """Uvicorn Multiprocess wrapper with slow start.
+
+    Slow start offers faster and more stable  start time.
+    Profile shows the start time is more stable and accelerated from
+    ~7s to ~3.3s on a 12-core machine after switching LONG workers and
+    Uvicorn workers to slow start.
+    Refer to subprocess_utils.slow_start_processes() for more details.
+    """
+
+    def __init__(self, config: uvicorn.Config, **kwargs):
+        """Initialize the multiprocess wrapper.
+
+        Args:
+            config: The uvicorn config.
+        """
+        super().__init__(config, **kwargs)
+        self._init_thread: Optional[threading.Thread] = None
+
+    def init_processes(self) -> None:
+        # Slow start worker processes asynchronously to avoid blocking signal
+        # handling of uvicorn.
+        self._init_thread = threading.Thread(target=self.slow_start_processes,
+                                             daemon=True)
+        self._init_thread.start()
+
+    def slow_start_processes(self) -> None:
+        """Initialize processes with slow start."""
+        to_start = []
+        # Init N worker processes
+        for _ in range(self.processes_num):
+            to_start.append(
+                multiprocess.Process(self.config, self.target, self.sockets))
+        # Start the processes with slow start, we only append start to
+        # self.processes because Uvicorn periodically restarts unstarted
+        # workers.
+        subprocess_utils.slow_start_processes(to_start,
+                                              on_start=self.processes.append,
+                                              should_exit=self.should_exit)
+
+    def terminate_all(self) -> None:
+        """Wait init thread to finish before terminating all processes."""
+        if self._init_thread is not None:
+            self._init_thread.join()
+        super().terminate_all()

sky/setup_files/dependencies.py

@@ -120,7 +120,10 @@ extras_require: Dict[str, List[str]] = {
     # https://github.com/googleapis/google-api-python-client/commit/f6e9d3869ed605b06f7cbf2e8cf2db25108506e6
     'gcp': ['google-api-python-client>=2.69.0', 'google-cloud-storage'],
     'ibm': [
-        'ibm-cloud-sdk-core', 'ibm-vpc', 'ibm-platform-services', 'ibm-cos-sdk'
+        'ibm-cloud-sdk-core',
+        'ibm-vpc',
+        'ibm-platform-services>=0.48.0',
+        'ibm-cos-sdk',
     ] + local_ray,
     'docker': ['docker'] + local_ray,
     'lambda': [],  # No dependencies needed for lambda

sky/utils/accelerator_registry.py

@@ -77,7 +77,7 @@ def canonicalize_accelerator_name(accelerator: str,
     # Look for Kubernetes accelerators online if the accelerator is not found
     # in the public cloud catalog. This is to make sure custom accelerators
     # on Kubernetes can be correctly canonicalized.
-    if not names and cloud_str in ['kubernetes', None]:
+    if not names and cloud_str in ['Kubernetes', None]:
         with rich_utils.safe_status(
                 ux_utils.spinner_message('Listing accelerators on Kubernetes')):
             searched = service_catalog.list_accelerators(