skypilot-nightly 1.0.0.dev20250311__py3-none-any.whl → 1.0.0.dev20250313__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '439de1a24a6f0a9601051ecdc3e565308bac442a'
+_SKYPILOT_COMMIT_SHA = '6044bbfe8712221e8d0da08ce8ce7ec36ab66caf'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250311'
+__version__ = '1.0.0.dev20250313'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/gcp.py

@@ -68,6 +68,13 @@ def credential_error_exception():
     return exceptions.DefaultCredentialsError
 
 
+@common.load_lazy_modules(_LAZY_MODULES)
+def gcp_auth_refresh_error_exception():
+    """GCP auth refresh error exception."""
+    from google.auth import exceptions
+    return exceptions.RefreshError
+
+
 @common.load_lazy_modules(_LAZY_MODULES)
 def get_credentials(cred_type: str, credentials_field: str):
     """Get GCP credentials."""

sky/adaptors/nebius.py

@@ -6,9 +6,11 @@ from sky.adaptors import common
 NEBIUS_TENANT_ID_FILENAME = 'NEBIUS_TENANT_ID.txt'
 NEBIUS_IAM_TOKEN_FILENAME = 'NEBIUS_IAM_TOKEN.txt'
 NEBIUS_PROJECT_ID_FILENAME = 'NEBIUS_PROJECT_ID.txt'
+NEBIUS_CREDENTIALS_FILENAME = 'credentials.json'
 NEBIUS_TENANT_ID_PATH = '~/.nebius/' + NEBIUS_TENANT_ID_FILENAME
 NEBIUS_IAM_TOKEN_PATH = '~/.nebius/' + NEBIUS_IAM_TOKEN_FILENAME
 NEBIUS_PROJECT_ID_PATH = '~/.nebius/' + NEBIUS_PROJECT_ID_FILENAME
+NEBIUS_CREDENTIALS_PATH = '~/.nebius/' + NEBIUS_CREDENTIALS_FILENAME
 
 MAX_RETRIES_TO_DISK_CREATE = 120
 MAX_RETRIES_TO_INSTANCE_STOP = 120
@@ -72,6 +74,11 @@ def get_iam_token():
     return _iam_token
 
 
+def is_token_or_cred_file_exist():
+    return (os.path.exists(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH)) or
+            os.path.exists(os.path.expanduser(NEBIUS_CREDENTIALS_PATH)))
+
+
 def get_project_id():
     global _project_id
     if _project_id is None:
@@ -97,4 +104,7 @@ def get_tenant_id():
 
 
 def sdk():
-    return nebius.sdk.SDK(credentials=get_iam_token())
+    if get_iam_token() is not None:
+        return nebius.sdk.SDK(credentials=get_iam_token())
+    return nebius.sdk.SDK(
+        credentials_file_name=os.path.expanduser(NEBIUS_CREDENTIALS_PATH))

sky/backends/backend_utils.py

@@ -1802,6 +1802,21 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
         status == status_lib.ClusterStatus.UP for status in node_statuses) and
                     len(node_statuses) == handle.launched_nodes)
 
+    def get_node_counts_from_ray_status(
+            runner: command_runner.CommandRunner) -> Tuple[int, int, str, str]:
+        rc, output, stderr = runner.run(
+            instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
+            stream_logs=False,
+            require_outputs=True,
+            separate_stderr=True)
+        if rc:
+            raise RuntimeError(
+                f'Refreshing status ({cluster_name!r}): Failed to check '
+                f'ray cluster\'s healthiness with '
+                f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
+                f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
+        return (*_count_healthy_nodes_from_ray(output), output, stderr)
+
     def run_ray_status_to_check_ray_cluster_healthy() -> bool:
         try:
             # NOTE: fetching the IPs is very slow as it calls into
@@ -1822,26 +1837,34 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 raise exceptions.FetchClusterInfoError(
                     reason=exceptions.FetchClusterInfoError.Reason.HEAD)
             head_runner = runners[0]
-            rc, output, stderr = head_runner.run(
-                instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
-                stream_logs=False,
-                require_outputs=True,
-                separate_stderr=True)
-            if rc:
-                raise RuntimeError(
-                    f'Refreshing status ({cluster_name!r}): Failed to check '
-                    f'ray cluster\'s healthiness with '
-                    f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
-                    f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
 
-            ready_head, ready_workers = _count_healthy_nodes_from_ray(output)
             total_nodes = handle.launched_nodes * handle.num_ips_per_node
-            if ready_head + ready_workers == total_nodes:
-                return True
+
+            for i in range(5):
+                ready_head, ready_workers, output, stderr = (
+                    get_node_counts_from_ray_status(head_runner))
+                if ready_head + ready_workers == total_nodes:
+                    return True
+                logger.debug(f'Refreshing status ({cluster_name!r}) attempt '
+                             f'{i}: ray status not showing all nodes '
+                             f'({ready_head + ready_workers}/{total_nodes});\n'
+                             f'output:\n{output}\nstderr:\n{stderr}')
+
+                # If cluster JUST started, maybe not all the nodes have shown
+                # up. Try again for a few seconds.
+                # Note: We are okay with this performance hit because it's very
+                # rare to normally hit this case. It requires:
+                # - All the instances in the cluster are up on the cloud side
+                #   (not preempted), but
+                # - The ray cluster is somehow degraded so not all instances are
+                #   showing up
+                time.sleep(1)
+
             raise RuntimeError(
                 f'Refreshing status ({cluster_name!r}): ray status not showing '
                 f'all nodes ({ready_head + ready_workers}/'
-                f'{total_nodes}); output: {output}; stderr: {stderr}')
+                f'{total_nodes});\noutput:\n{output}\nstderr:\n{stderr}')
+
         except exceptions.FetchClusterInfoError:
             logger.debug(
                 f'Refreshing status ({cluster_name!r}) failed to get IPs.')

sky/backends/cloud_vm_ray_backend.py

@@ -772,32 +772,6 @@ class FailoverCloudErrorHandlerV1:
             setattr(e, 'detailed_reason', detailed_reason)
             raise e
 
-    @staticmethod
-    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
-                        launchable_resources: 'resources_lib.Resources',
-                        region: 'clouds.Region',
-                        zones: Optional[List['clouds.Zone']], stdout: str,
-                        stderr: str):
-        del region, zones  # Unused.
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout,
-            stderr,
-            is_error_str_known=lambda x: 'LambdaCloudError:' in x.strip())
-        messages = '\n  '.join(errors)
-        style = colorama.Style
-        logger.warning(f'  {style.DIM}{messages}{style.RESET_ALL}')
-        _add_to_blocked_resources(blocked_resources,
-                                  launchable_resources.copy(zone=None))
-
-        # Sometimes, LambdaCloudError will list available regions.
-        for e in errors:
-            if e.find('Regions with capacity available:') != -1:
-                for r in service_catalog.regions('lambda'):
-                    if e.find(r.name) == -1:
-                        _add_to_blocked_resources(
-                            blocked_resources,
-                            launchable_resources.copy(region=r.name, zone=None))
-
     @staticmethod
     def _scp_handler(blocked_resources: Set['resources_lib.Resources'],
                      launchable_resources: 'resources_lib.Resources',
@@ -846,32 +820,6 @@ class FailoverCloudErrorHandlerV1:
             _add_to_blocked_resources(blocked_resources,
                                       launchable_resources.copy(zone=zone.name))
 
-    # Apr, 2023 by Hysun(hysun.he@oracle.com): Added support for OCI
-    @staticmethod
-    def _oci_handler(blocked_resources: Set['resources_lib.Resources'],
-                     launchable_resources: 'resources_lib.Resources',
-                     region: 'clouds.Region',
-                     zones: Optional[List['clouds.Zone']], stdout: str,
-                     stderr: str):
-        known_service_errors = [
-            'NotAuthorizedOrNotFound', 'CannotParseRequest', 'InternalError',
-            'LimitExceeded', 'NotAuthenticated'
-        ]
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout, stderr, lambda x: 'VcnSubnetNotFound' in x.strip() or
-            ('oci.exceptions.ServiceError' in x.strip() and any(
-                known_err in x.strip() for known_err in known_service_errors)))
-        logger.warning(f'Got error(s) in {region.name}:')
-        messages = '\n\t'.join(errors)
-        style = colorama.Style
-        logger.warning(f'{style.DIM}\t{messages}{style.RESET_ALL}')
-
-        if zones is not None:
-            for zone in zones:
-                _add_to_blocked_resources(
-                    blocked_resources,
-                    launchable_resources.copy(zone=zone.name))
-
     @staticmethod
     def update_blocklist_on_error(
             blocked_resources: Set['resources_lib.Resources'],
@@ -1123,6 +1071,23 @@ class FailoverCloudErrorHandlerV2:
                     blocked_resources,
                     launchable_resources.copy(zone=zone.name))
 
+    @staticmethod
+    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
+                        launchable_resources: 'resources_lib.Resources',
+                        region: 'clouds.Region',
+                        zones: Optional[List['clouds.Zone']], error: Exception):
+        output = str(error)
+        # Sometimes, lambda cloud error will list available regions.
+        if output.find('Regions with capacity available:') != -1:
+            for r in service_catalog.regions('lambda'):
+                if output.find(r.name) == -1:
+                    _add_to_blocked_resources(
+                        blocked_resources,
+                        launchable_resources.copy(region=r.name, zone=None))
+        else:
+            FailoverCloudErrorHandlerV2._default_handler(
+                blocked_resources, launchable_resources, region, zones, error)
+
     @staticmethod
     def _default_handler(blocked_resources: Set['resources_lib.Resources'],
                          launchable_resources: 'resources_lib.Resources',

sky/cli.py

@@ -100,6 +100,7 @@ an autogenerated name."""
 # The maximum number of in-progress managed jobs to show in the status
 # command.
 _NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS = 5
+_NUM_MANAGED_JOBS_TO_SHOW = 50
 
 _STATUS_PROPERTY_CLUSTER_NUM_ERROR_MESSAGE = (
     '{cluster_num} cluster{plural} {verb}. Please specify {cause} '
@@ -1389,16 +1390,16 @@ def _handle_jobs_queue_request(
         request_id: str,
         show_all: bool,
         show_user: bool,
-        limit_num_jobs_to_show: bool = False,
+        max_num_jobs_to_show: Optional[int],
         is_called_by_user: bool = False) -> Tuple[Optional[int], str]:
     """Get the in-progress managed jobs.
 
     Args:
         show_all: Show all information of each job (e.g., region, price).
         show_user: Show the user who submitted the job.
-        limit_num_jobs_to_show: If True, limit the number of jobs to show to
-            _NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS, which is mainly used by
-            `sky status`.
+        max_num_jobs_to_show: If not None, limit the number of jobs to show to
+            this number, which is mainly used by `sky status`
+            and `sky jobs queue`.
         is_called_by_user: If this function is called by user directly, or an
             internal call.
 
@@ -1459,12 +1460,10 @@ def _handle_jobs_queue_request(
         msg += ('Failed to query managed jobs: '
                 f'{common_utils.format_exception(e, use_bracket=True)}')
     else:
-        max_jobs_to_show = (_NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS
-                            if limit_num_jobs_to_show else None)
         msg = managed_jobs.format_job_table(managed_jobs_,
                                             show_all=show_all,
                                             show_user=show_user,
-                                            max_jobs=max_jobs_to_show)
+                                            max_jobs=max_num_jobs_to_show)
     return num_in_progress_jobs, msg
 
 
@@ -1875,7 +1874,7 @@ def status(verbose: bool, refresh: bool, ip: bool, endpoints: bool,
                     managed_jobs_queue_request_id,
                     show_all=False,
                     show_user=all_users,
-                    limit_num_jobs_to_show=not all,
+                    max_num_jobs_to_show=_NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS,
                     is_called_by_user=False)
             except KeyboardInterrupt:
                 sdk.api_cancel(managed_jobs_queue_request_id, silent=True)
@@ -3943,10 +3942,15 @@ def jobs_launch(
               is_flag=True,
               required=False,
               help='Show jobs from all users.')
+@click.option('--all',
+              default=False,
+              is_flag=True,
+              required=False,
+              help='Show all jobs.')
 @usage_lib.entrypoint
 # pylint: disable=redefined-builtin
 def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
-               all_users: bool):
+               all_users: bool, all: bool):
     """Show statuses of managed jobs.
 
     Each managed jobs can have one of the following statuses:
@@ -4004,10 +4008,13 @@ def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
     with rich_utils.client_status('[cyan]Checking managed jobs[/]'):
         managed_jobs_request_id = managed_jobs.queue(
             refresh=refresh, skip_finished=skip_finished, all_users=all_users)
-        _, msg = _handle_jobs_queue_request(managed_jobs_request_id,
-                                            show_all=verbose,
-                                            show_user=all_users,
-                                            is_called_by_user=True)
+        max_num_jobs_to_show = (_NUM_MANAGED_JOBS_TO_SHOW if not all else None)
+        num_jobs, msg = _handle_jobs_queue_request(
+            managed_jobs_request_id,
+            show_all=verbose,
+            show_user=all_users,
+            max_num_jobs_to_show=max_num_jobs_to_show,
+            is_called_by_user=True)
     if not skip_finished:
         in_progress_only_hint = ''
     else:
@@ -4015,6 +4022,12 @@ def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
     click.echo(f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
                f'Managed jobs{colorama.Style.RESET_ALL}'
                f'{in_progress_only_hint}\n{msg}')
+    if max_num_jobs_to_show and num_jobs and max_num_jobs_to_show < num_jobs:
+        click.echo(
+            f'{colorama.Fore.CYAN}'
+            f'Only showing the latest {max_num_jobs_to_show} '
+            f'managed jobs'
+            f'(use --all to show all managed jobs) {colorama.Style.RESET_ALL} ')
 
 
 @jobs.command('cancel', cls=_DocumentedCodeCommand)

sky/client/cli.py

@@ -100,6 +100,7 @@ an autogenerated name."""
 # The maximum number of in-progress managed jobs to show in the status
 # command.
 _NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS = 5
+_NUM_MANAGED_JOBS_TO_SHOW = 50
 
 _STATUS_PROPERTY_CLUSTER_NUM_ERROR_MESSAGE = (
     '{cluster_num} cluster{plural} {verb}. Please specify {cause} '
@@ -1389,16 +1390,16 @@ def _handle_jobs_queue_request(
         request_id: str,
         show_all: bool,
         show_user: bool,
-        limit_num_jobs_to_show: bool = False,
+        max_num_jobs_to_show: Optional[int],
         is_called_by_user: bool = False) -> Tuple[Optional[int], str]:
     """Get the in-progress managed jobs.
 
     Args:
         show_all: Show all information of each job (e.g., region, price).
         show_user: Show the user who submitted the job.
-        limit_num_jobs_to_show: If True, limit the number of jobs to show to
-            _NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS, which is mainly used by
-            `sky status`.
+        max_num_jobs_to_show: If not None, limit the number of jobs to show to
+            this number, which is mainly used by `sky status`
+            and `sky jobs queue`.
         is_called_by_user: If this function is called by user directly, or an
             internal call.
 
@@ -1459,12 +1460,10 @@ def _handle_jobs_queue_request(
         msg += ('Failed to query managed jobs: '
                 f'{common_utils.format_exception(e, use_bracket=True)}')
     else:
-        max_jobs_to_show = (_NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS
-                            if limit_num_jobs_to_show else None)
         msg = managed_jobs.format_job_table(managed_jobs_,
                                             show_all=show_all,
                                             show_user=show_user,
-                                            max_jobs=max_jobs_to_show)
+                                            max_jobs=max_num_jobs_to_show)
     return num_in_progress_jobs, msg
 
 
@@ -1875,7 +1874,7 @@ def status(verbose: bool, refresh: bool, ip: bool, endpoints: bool,
                     managed_jobs_queue_request_id,
                     show_all=False,
                     show_user=all_users,
-                    limit_num_jobs_to_show=not all,
+                    max_num_jobs_to_show=_NUM_MANAGED_JOBS_TO_SHOW_IN_STATUS,
                     is_called_by_user=False)
             except KeyboardInterrupt:
                 sdk.api_cancel(managed_jobs_queue_request_id, silent=True)
@@ -3943,10 +3942,15 @@ def jobs_launch(
               is_flag=True,
               required=False,
               help='Show jobs from all users.')
+@click.option('--all',
+              default=False,
+              is_flag=True,
+              required=False,
+              help='Show all jobs.')
 @usage_lib.entrypoint
 # pylint: disable=redefined-builtin
 def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
-               all_users: bool):
+               all_users: bool, all: bool):
     """Show statuses of managed jobs.
 
     Each managed jobs can have one of the following statuses:
@@ -4004,10 +4008,13 @@ def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
     with rich_utils.client_status('[cyan]Checking managed jobs[/]'):
         managed_jobs_request_id = managed_jobs.queue(
             refresh=refresh, skip_finished=skip_finished, all_users=all_users)
-        _, msg = _handle_jobs_queue_request(managed_jobs_request_id,
-                                            show_all=verbose,
-                                            show_user=all_users,
-                                            is_called_by_user=True)
+        max_num_jobs_to_show = (_NUM_MANAGED_JOBS_TO_SHOW if not all else None)
+        num_jobs, msg = _handle_jobs_queue_request(
+            managed_jobs_request_id,
+            show_all=verbose,
+            show_user=all_users,
+            max_num_jobs_to_show=max_num_jobs_to_show,
+            is_called_by_user=True)
     if not skip_finished:
         in_progress_only_hint = ''
     else:
@@ -4015,6 +4022,12 @@ def jobs_queue(verbose: bool, refresh: bool, skip_finished: bool,
     click.echo(f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
                f'Managed jobs{colorama.Style.RESET_ALL}'
                f'{in_progress_only_hint}\n{msg}')
+    if max_num_jobs_to_show and num_jobs and max_num_jobs_to_show < num_jobs:
+        click.echo(
+            f'{colorama.Fore.CYAN}'
+            f'Only showing the latest {max_num_jobs_to_show} '
+            f'managed jobs'
+            f'(use --all to show all managed jobs) {colorama.Style.RESET_ALL} ')
 
 
 @jobs.command('cancel', cls=_DocumentedCodeCommand)

sky/client/sdk.py

@@ -32,7 +32,6 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.client import common as client_common
 from sky.server import common as server_common
-from sky.server import constants as server_constants
 from sky.server.requests import payloads
 from sky.server.requests import requests as requests_lib
 from sky.skylet import constants
@@ -1707,14 +1706,8 @@ def api_stop() -> None:
                                                      force=True)
             found = True
 
-    # Remove the database for requests including any files starting with
-    # api.constants.API_SERVER_REQUEST_DB_PATH
-    db_path = os.path.expanduser(server_constants.API_SERVER_REQUEST_DB_PATH)
-    for extension in ['', '-shm', '-wal']:
-        try:
-            os.remove(f'{db_path}{extension}')
-        except FileNotFoundError:
-            logger.debug(f'Database file {db_path}{extension} not found.')
+    # Remove the database for requests.
+    server_common.clear_local_api_server_database()
 
     if found:
         logger.info(f'{colorama.Fore.GREEN}SkyPilot API server stopped.'

sky/clouds/gcp.py

@@ -843,7 +843,10 @@ class GCP(clouds.Cloud):
         permissions = {'permissions': gcp_minimal_permissions}
         request = crm.projects().testIamPermissions(resource=project,
                                                     body=permissions)
-        ret_permissions = request.execute().get('permissions', [])
+        try:
+            ret_permissions = request.execute().get('permissions', [])
+        except gcp.gcp_auth_refresh_error_exception() as e:
+            return False, common_utils.format_exception(e, use_bracket=True)
 
         diffs = set(gcp_minimal_permissions).difference(set(ret_permissions))
         if diffs:

sky/clouds/nebius.py

@@ -17,6 +17,7 @@ _CREDENTIAL_FILES = [
     nebius.NEBIUS_TENANT_ID_FILENAME,
     nebius.NEBIUS_IAM_TOKEN_FILENAME,
     nebius.NEBIUS_PROJECT_ID_FILENAME,
+    nebius.NEBIUS_CREDENTIALS_FILENAME
 ]
 
 
@@ -252,15 +253,16 @@ class Nebius(clouds.Cloud):
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """ Verify that the user has valid credentials for Nebius. """
         logging.debug('Nebius cloud check credentials')
-        token = nebius.get_iam_token()
-        token_msg = ('    Credentials can be set up by running: \n'\
-                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n')  # pylint: disable=line-too-long
+        token_cred_msg = ('    Credentials can be set up by running: \n'\
+                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n'\
+                          '    or generate  ~/.nebius/credentials.json')  # pylint: disable=line-too-long
+
         tenant_msg = ('   Copy your tenat ID from the web console and save it to file \n'  # pylint: disable=line-too-long
                       f'        $ echo $NEBIUS_TENANT_ID_PATH > {nebius.NEBIUS_TENANT_ID_PATH} \n'  # pylint: disable=line-too-long
                       '   Or if you have 1 tenant you can run:\n'  # pylint: disable=line-too-long
                       f'        $ nebius --format json iam whoami|jq -r \'.user_profile.tenants[0].tenant_id\' > {nebius.NEBIUS_TENANT_ID_PATH} \n')  # pylint: disable=line-too-long
-        if token is None:
-            return False, f'{token_msg}'
+        if not nebius.is_token_or_cred_file_exist():
+            return False, f'{token_cred_msg}'
         sdk = nebius.sdk()
         tenant_id = nebius.get_tenant_id()
         if tenant_id is None:
@@ -272,7 +274,7 @@ class Nebius(clouds.Cloud):
         except nebius.request_error() as e:
             return False, (
                 f'{e.status} \n'  # First line is indented by 4 spaces
-                f'{token_msg}'
+                f'{token_cred_msg}'
                 f'{tenant_msg}')
         return True, None
 

sky/data/storage.py

@@ -143,6 +143,22 @@ class StoreType(enum.Enum):
 
         raise ValueError(f'Unsupported cloud for StoreType: {cloud}')
 
+    def to_cloud(self) -> str:
+        if self == StoreType.S3:
+            return str(clouds.AWS())
+        elif self == StoreType.GCS:
+            return str(clouds.GCP())
+        elif self == StoreType.AZURE:
+            return str(clouds.Azure())
+        elif self == StoreType.R2:
+            return cloudflare.NAME
+        elif self == StoreType.IBM:
+            return str(clouds.IBM())
+        elif self == StoreType.OCI:
+            return str(clouds.OCI())
+        else:
+            raise ValueError(f'Unknown store type: {self}')
+
     @classmethod
     def from_store(cls, store: 'AbstractStore') -> 'StoreType':
         if isinstance(store, S3Store):

sky/exceptions.py

@@ -28,12 +28,19 @@ GIT_FATAL_EXIT_CODE = 128
 ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 
 
-def is_safe_exception(exc: Exception) -> bool:
+def is_safe_exception(exc: BaseException) -> bool:
     """Returns True if the exception is safe to send to clients.
 
     Safe exceptions are:
     1. Built-in exceptions
     2. SkyPilot's own exceptions
+
+    Args:
+        exc: The exception to check, accept BaseException to handle SystemExit
+            and KeyboardInterrupt.
+
+    Returns:
+        True if the exception is safe to send to clients, False otherwise.
     """
     module = type(exc).__module__
 
@@ -48,7 +55,7 @@ def is_safe_exception(exc: Exception) -> bool:
     return False
 
 
-def wrap_exception(exc: Exception) -> Exception:
+def wrap_exception(exc: BaseException) -> BaseException:
     """Wraps non-safe exceptions into SkyPilot exceptions
 
     This is used to wrap exceptions that are not safe to deserialize at clients.
@@ -64,7 +71,8 @@ def wrap_exception(exc: Exception) -> Exception:
                       error_type=type(exc).__name__)
 
 
-def serialize_exception(e: Exception) -> Dict[str, Any]:
+# Accept BaseException to handle SystemExit and KeyboardInterrupt
+def serialize_exception(e: BaseException) -> Dict[str, Any]:
     """Serialize the exception.
 
     This function also wraps any unsafe exceptions (e.g., cloud exceptions)

sky/provision/kubernetes/utils.py

@@ -717,6 +717,15 @@ def check_instance_fits(context: Optional[str],
             fits, reason = check_tpu_fits(k8s_instance_type, gpu_nodes)
             if reason is not None:
                 return fits, reason
+        else:
+            # Check if any of the GPU nodes have sufficient number of GPUs.
+            gpu_nodes = [
+                node for node in gpu_nodes if
+                get_node_accelerator_count(node.status.allocatable) >= acc_count
+            ]
+            if not gpu_nodes:
+                return False, (
+                    f'No GPU nodes found with {acc_count} or more GPUs.')
 
         candidate_nodes = gpu_nodes
         not_fit_reason_prefix = (
@@ -853,7 +862,7 @@ def get_accelerator_label_key_value(
                 for label, value in label_list:
                     if (label_formatter.match_label_key(label) and
                             label_formatter.get_accelerator_from_label_value(
-                                value) == acc_type):
+                                value).lower() == acc_type.lower()):
                         if is_tpu_on_gke(acc_type):
                             assert isinstance(label_formatter,
                                               GKELabelFormatter)

sky/server/common.py

@@ -428,3 +428,19 @@ def reload_for_new_request(client_entrypoint: Optional[str],
     # necessary because the logger is initialized before the environment
     # variables are set, such as SKYPILOT_DEBUG.
     sky_logging.reload_logger()
+
+
+def clear_local_api_server_database() -> None:
+    """Removes the local API server database.
+
+    The CLI can call this during cleanup of a local API server, or the API
+    server can call it during startup.
+    """
+    # Remove the database for requests including any files starting with
+    # api.constants.API_SERVER_REQUEST_DB_PATH
+    db_path = os.path.expanduser(server_constants.API_SERVER_REQUEST_DB_PATH)
+    for extension in ['', '-shm', '-wal']:
+        try:
+            os.remove(f'{db_path}{extension}')
+        except FileNotFoundError:
+            logger.debug(f'Database file {db_path}{extension} not found.')

sky/server/requests/event_loop.py

@@ -0,0 +1,31 @@
+"""Executor event loop to process tasks in coroutines."""
+import asyncio
+import concurrent.futures
+import threading
+from typing import Coroutine, Optional
+
+# Dedicated event loop for requests, isolated with the event loop managed
+# by uvicorn. This is responsible for light-weight async tasks or sub-tasks,
+# refer to `executor.py` for more details about cooperation between the event
+# loop and executor process pool.
+_EVENT_LOOP: Optional[asyncio.AbstractEventLoop] = None
+_LOCK = threading.Lock()
+
+
+def run(coro: Coroutine) -> concurrent.futures.Future:
+    """Run a coroutine asynchronously in the request event loop."""
+    return asyncio.run_coroutine_threadsafe(coro, get_event_loop())
+
+
+def get_event_loop() -> asyncio.AbstractEventLoop:
+    """Open and get the event loop."""
+    global _EVENT_LOOP
+    if _EVENT_LOOP is not None and not _EVENT_LOOP.is_closed():
+        return _EVENT_LOOP
+    with _LOCK:
+        if _EVENT_LOOP is None or _EVENT_LOOP.is_closed():
+            _EVENT_LOOP = asyncio.new_event_loop()
+            loop_thread = threading.Thread(target=_EVENT_LOOP.run_forever,
+                                           daemon=True)
+            loop_thread.start()
+    return _EVENT_LOOP