skypilot-nightly 1.0.0.dev20250310__tar.gz → 1.0.0.dev20250312__tar.gz

{skypilot_nightly-1.0.0.dev20250310/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250312}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250310
+Version: 1.0.0.dev20250312
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'f5170ee3a1a6711f9ed54941aae2dee4e6525acc'
+_SKYPILOT_COMMIT_SHA = '78a42b6e733bbc29b68efe0e9c79191eaaca9fcd'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250310'
+__version__ = '1.0.0.dev20250312'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/adaptors/nebius.py

@@ -6,9 +6,11 @@ from sky.adaptors import common
 NEBIUS_TENANT_ID_FILENAME = 'NEBIUS_TENANT_ID.txt'
 NEBIUS_IAM_TOKEN_FILENAME = 'NEBIUS_IAM_TOKEN.txt'
 NEBIUS_PROJECT_ID_FILENAME = 'NEBIUS_PROJECT_ID.txt'
+NEBIUS_CREDENTIALS_FILENAME = 'credentials.json'
 NEBIUS_TENANT_ID_PATH = '~/.nebius/' + NEBIUS_TENANT_ID_FILENAME
 NEBIUS_IAM_TOKEN_PATH = '~/.nebius/' + NEBIUS_IAM_TOKEN_FILENAME
 NEBIUS_PROJECT_ID_PATH = '~/.nebius/' + NEBIUS_PROJECT_ID_FILENAME
+NEBIUS_CREDENTIALS_PATH = '~/.nebius/' + NEBIUS_CREDENTIALS_FILENAME
 
 MAX_RETRIES_TO_DISK_CREATE = 120
 MAX_RETRIES_TO_INSTANCE_STOP = 120
@@ -72,6 +74,11 @@ def get_iam_token():
     return _iam_token
 
 
+def is_token_or_cred_file_exist():
+    return (os.path.exists(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH)) or
+            os.path.exists(os.path.expanduser(NEBIUS_CREDENTIALS_PATH)))
+
+
 def get_project_id():
     global _project_id
     if _project_id is None:
@@ -97,4 +104,7 @@ def get_tenant_id():
 
 
 def sdk():
-    return nebius.sdk.SDK(credentials=get_iam_token())
+    if get_iam_token() is not None:
+        return nebius.sdk.SDK(credentials=get_iam_token())
+    return nebius.sdk.SDK(
+        credentials_file_name=os.path.expanduser(NEBIUS_CREDENTIALS_PATH))

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/backends/backend_utils.py

@@ -1802,6 +1802,21 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
         status == status_lib.ClusterStatus.UP for status in node_statuses) and
                     len(node_statuses) == handle.launched_nodes)
 
+    def get_node_counts_from_ray_status(
+            runner: command_runner.CommandRunner) -> Tuple[int, int, str, str]:
+        rc, output, stderr = runner.run(
+            instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
+            stream_logs=False,
+            require_outputs=True,
+            separate_stderr=True)
+        if rc:
+            raise RuntimeError(
+                f'Refreshing status ({cluster_name!r}): Failed to check '
+                f'ray cluster\'s healthiness with '
+                f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
+                f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
+        return (*_count_healthy_nodes_from_ray(output), output, stderr)
+
     def run_ray_status_to_check_ray_cluster_healthy() -> bool:
         try:
             # NOTE: fetching the IPs is very slow as it calls into
@@ -1822,26 +1837,34 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 raise exceptions.FetchClusterInfoError(
                     reason=exceptions.FetchClusterInfoError.Reason.HEAD)
             head_runner = runners[0]
-            rc, output, stderr = head_runner.run(
-                instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
-                stream_logs=False,
-                require_outputs=True,
-                separate_stderr=True)
-            if rc:
-                raise RuntimeError(
-                    f'Refreshing status ({cluster_name!r}): Failed to check '
-                    f'ray cluster\'s healthiness with '
-                    f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
-                    f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
 
-            ready_head, ready_workers = _count_healthy_nodes_from_ray(output)
             total_nodes = handle.launched_nodes * handle.num_ips_per_node
-            if ready_head + ready_workers == total_nodes:
-                return True
+
+            for i in range(5):
+                ready_head, ready_workers, output, stderr = (
+                    get_node_counts_from_ray_status(head_runner))
+                if ready_head + ready_workers == total_nodes:
+                    return True
+                logger.debug(f'Refreshing status ({cluster_name!r}) attempt '
+                             f'{i}: ray status not showing all nodes '
+                             f'({ready_head + ready_workers}/{total_nodes});\n'
+                             f'output:\n{output}\nstderr:\n{stderr}')
+
+                # If cluster JUST started, maybe not all the nodes have shown
+                # up. Try again for a few seconds.
+                # Note: We are okay with this performance hit because it's very
+                # rare to normally hit this case. It requires:
+                # - All the instances in the cluster are up on the cloud side
+                #   (not preempted), but
+                # - The ray cluster is somehow degraded so not all instances are
+                #   showing up
+                time.sleep(1)
+
             raise RuntimeError(
                 f'Refreshing status ({cluster_name!r}): ray status not showing '
                 f'all nodes ({ready_head + ready_workers}/'
-                f'{total_nodes}); output: {output}; stderr: {stderr}')
+                f'{total_nodes});\noutput:\n{output}\nstderr:\n{stderr}')
+
         except exceptions.FetchClusterInfoError:
             logger.debug(
                 f'Refreshing status ({cluster_name!r}) failed to get IPs.')

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/backends/cloud_vm_ray_backend.py

@@ -772,32 +772,6 @@ class FailoverCloudErrorHandlerV1:
             setattr(e, 'detailed_reason', detailed_reason)
             raise e
 
-    @staticmethod
-    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
-                        launchable_resources: 'resources_lib.Resources',
-                        region: 'clouds.Region',
-                        zones: Optional[List['clouds.Zone']], stdout: str,
-                        stderr: str):
-        del region, zones  # Unused.
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout,
-            stderr,
-            is_error_str_known=lambda x: 'LambdaCloudError:' in x.strip())
-        messages = '\n  '.join(errors)
-        style = colorama.Style
-        logger.warning(f'  {style.DIM}{messages}{style.RESET_ALL}')
-        _add_to_blocked_resources(blocked_resources,
-                                  launchable_resources.copy(zone=None))
-
-        # Sometimes, LambdaCloudError will list available regions.
-        for e in errors:
-            if e.find('Regions with capacity available:') != -1:
-                for r in service_catalog.regions('lambda'):
-                    if e.find(r.name) == -1:
-                        _add_to_blocked_resources(
-                            blocked_resources,
-                            launchable_resources.copy(region=r.name, zone=None))
-
     @staticmethod
     def _scp_handler(blocked_resources: Set['resources_lib.Resources'],
                      launchable_resources: 'resources_lib.Resources',
@@ -846,32 +820,6 @@ class FailoverCloudErrorHandlerV1:
             _add_to_blocked_resources(blocked_resources,
                                       launchable_resources.copy(zone=zone.name))
 
-    # Apr, 2023 by Hysun(hysun.he@oracle.com): Added support for OCI
-    @staticmethod
-    def _oci_handler(blocked_resources: Set['resources_lib.Resources'],
-                     launchable_resources: 'resources_lib.Resources',
-                     region: 'clouds.Region',
-                     zones: Optional[List['clouds.Zone']], stdout: str,
-                     stderr: str):
-        known_service_errors = [
-            'NotAuthorizedOrNotFound', 'CannotParseRequest', 'InternalError',
-            'LimitExceeded', 'NotAuthenticated'
-        ]
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout, stderr, lambda x: 'VcnSubnetNotFound' in x.strip() or
-            ('oci.exceptions.ServiceError' in x.strip() and any(
-                known_err in x.strip() for known_err in known_service_errors)))
-        logger.warning(f'Got error(s) in {region.name}:')
-        messages = '\n\t'.join(errors)
-        style = colorama.Style
-        logger.warning(f'{style.DIM}\t{messages}{style.RESET_ALL}')
-
-        if zones is not None:
-            for zone in zones:
-                _add_to_blocked_resources(
-                    blocked_resources,
-                    launchable_resources.copy(zone=zone.name))
-
     @staticmethod
     def update_blocklist_on_error(
             blocked_resources: Set['resources_lib.Resources'],
@@ -1123,6 +1071,23 @@ class FailoverCloudErrorHandlerV2:
                     blocked_resources,
                     launchable_resources.copy(zone=zone.name))
 
+    @staticmethod
+    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
+                        launchable_resources: 'resources_lib.Resources',
+                        region: 'clouds.Region',
+                        zones: Optional[List['clouds.Zone']], error: Exception):
+        output = str(error)
+        # Sometimes, lambda cloud error will list available regions.
+        if output.find('Regions with capacity available:') != -1:
+            for r in service_catalog.regions('lambda'):
+                if output.find(r.name) == -1:
+                    _add_to_blocked_resources(
+                        blocked_resources,
+                        launchable_resources.copy(region=r.name, zone=None))
+        else:
+            FailoverCloudErrorHandlerV2._default_handler(
+                blocked_resources, launchable_resources, region, zones, error)
+
     @staticmethod
     def _default_handler(blocked_resources: Set['resources_lib.Resources'],
                          launchable_resources: 'resources_lib.Resources',

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/clouds/kubernetes.py

@@ -2,9 +2,10 @@
 import os
 import re
 import typing
-from typing import Dict, Iterator, List, Optional, Tuple, Union
+from typing import Dict, Iterator, List, Optional, Set, Tuple, Union
 
 from sky import clouds
+from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import kubernetes
@@ -78,6 +79,11 @@ class Kubernetes(clouds.Cloud):
     PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
     STATUS_VERSION = clouds.StatusVersion.SKYPILOT
 
+    _INDENT_PREFIX = ' ' * 4
+
+    # Set of contexts that has logged as temporarily unreachable
+    logged_unreachable_contexts: Set[str] = set()
+
     @property
     def ssh_key_secret_field_name(self):
         # Use a fresh user hash to avoid conflicts in the secret object naming.
@@ -90,6 +96,8 @@ class Kubernetes(clouds.Cloud):
     def _unsupported_features_for_resources(
         cls, resources: 'resources_lib.Resources'
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
+        # TODO(aylei): features need to be regional (per context) to make
+        # multi-kubernetes selection/failover work.
         unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
         context = resources.region
         if context is None:
@@ -106,10 +114,13 @@ class Kubernetes(clouds.Cloud):
             unsupported_features[
                 clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
         # Allow spot instances if supported by the cluster
-        spot_label_key, _ = kubernetes_utils.get_spot_label(context)
-        if spot_label_key is not None:
-            unsupported_features.pop(
-                clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
+        try:
+            spot_label_key, _ = kubernetes_utils.get_spot_label(context)
+            if spot_label_key is not None:
+                unsupported_features.pop(
+                    clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
+        except exceptions.KubeAPIUnreachableError as e:
+            cls._log_unreachable_context(context, str(e))
         return unsupported_features
 
     @classmethod
@@ -170,6 +181,36 @@ class Kubernetes(clouds.Cloud):
         cls._log_skipped_contexts_once(tuple(skipped_contexts))
         return existing_contexts
 
+    @classmethod
+    def _log_unreachable_context(cls,
+                                 context: str,
+                                 reason: Optional[str] = None) -> None:
+        """Logs a Kubernetes context as unreachable.
+
+        Args:
+            context: The Kubernetes context to mark as unreachable.
+            reason: Optional reason for marking the context as unreachable.
+            silent: Whether to suppress the log message.
+        """
+        # Skip if this context has already been logged as unreachable
+        if context in cls.logged_unreachable_contexts:
+            return
+
+        cls.logged_unreachable_contexts.add(context)
+        msg = f'Excluding Kubernetes context {context}'
+        if reason is not None:
+            msg += f': {reason}'
+        logger.info(msg)
+
+        # Check if all existing allowed contexts are now unreachable
+        existing_contexts = cls.existing_allowed_contexts()
+        if existing_contexts and all(ctx in cls.logged_unreachable_contexts
+                                     for ctx in existing_contexts):
+            logger.warning(
+                'All Kubernetes contexts are unreachable. '
+                'Retry if it is a transient error, or run sky check to '
+                'refresh Kubernetes availability if permanent.')
+
     @classmethod
     def regions_with_offering(cls, instance_type: Optional[str],
                               accelerators: Optional[Dict[str, int]],
@@ -198,8 +239,12 @@ class Kubernetes(clouds.Cloud):
             # provision_timeout, after which failover will be triggered.
             for r in regions:
                 context = r.name
-                fits, reason = kubernetes_utils.check_instance_fits(
-                    context, instance_type)
+                try:
+                    fits, reason = kubernetes_utils.check_instance_fits(
+                        context, instance_type)
+                except exceptions.KubeAPIUnreachableError as e:
+                    cls._log_unreachable_context(context, str(e))
+                    continue
                 if fits:
                     regions_to_return.append(r)
                 else:
@@ -609,18 +654,53 @@ class Kubernetes(clouds.Cloud):
                     'Check if you have a valid kubeconfig file' +
                     check_skypilot_config_msg)
         reasons = []
+        hints = []
+        success = False
         for context in existing_allowed_contexts:
             try:
                 check_result = kubernetes_utils.check_credentials(context)
                 if check_result[0]:
-                    return check_result
-                reasons.append(f'{context}: {check_result[1]}')
+                    success = True
+                    if check_result[1] is not None:
+                        hints.append(f'Context {context}: {check_result[1]}')
+                else:
+                    reasons.append(f'Context {context}: {check_result[1]}')
             except Exception as e:  # pylint: disable=broad-except
                 return (False, f'Credential check failed for {context}: '
                         f'{common_utils.format_exception(e)}')
+        if success:
+            return (True, cls._format_credential_check_results(hints, reasons))
         return (False, 'Failed to find available context with working '
                 'credentials. Details:\n' + '\n'.join(reasons))
 
+    @classmethod
+    def _format_credential_check_results(cls, hints: List[str],
+                                         reasons: List[str]) -> str:
+        """Format credential check results with hints and reasons.
+
+        Args:
+            hints: List of successful context check messages.
+            reasons: List of failed context check reasons.
+
+        Returns:
+            A formatted string containing hints and by failure reasons.
+        """
+        message_parts = []
+        if len(hints) == 1 and not reasons:
+            return hints[0]
+        if hints:
+            message_parts.append(f'\n{cls._INDENT_PREFIX}  ' +
+                                 f'\n{cls._INDENT_PREFIX}  '.join(hints))
+        if reasons:
+            if hints:
+                message_parts.append('\n')
+            message_parts.append(
+                f'\n{cls._INDENT_PREFIX}Unavailable contexts (remove from '
+                '"allowed_contexts" config if permanently unavailable): '
+                f'\n{cls._INDENT_PREFIX}  ' +
+                f'\n{cls._INDENT_PREFIX}  '.join(reasons))
+        return ''.join(message_parts)
+
     def get_credential_file_mounts(self) -> Dict[str, str]:
         if os.path.exists(os.path.expanduser(CREDENTIAL_PATH)):
             # Upload kubeconfig to the default path to avoid having to set

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/clouds/nebius.py

@@ -17,6 +17,7 @@ _CREDENTIAL_FILES = [
     nebius.NEBIUS_TENANT_ID_FILENAME,
     nebius.NEBIUS_IAM_TOKEN_FILENAME,
     nebius.NEBIUS_PROJECT_ID_FILENAME,
+    nebius.NEBIUS_CREDENTIALS_FILENAME
 ]
 
 
@@ -252,15 +253,16 @@ class Nebius(clouds.Cloud):
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """ Verify that the user has valid credentials for Nebius. """
         logging.debug('Nebius cloud check credentials')
-        token = nebius.get_iam_token()
-        token_msg = ('    Credentials can be set up by running: \n'\
-                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n')  # pylint: disable=line-too-long
+        token_cred_msg = ('    Credentials can be set up by running: \n'\
+                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n'\
+                          '    or generate  ~/.nebius/credentials.json')  # pylint: disable=line-too-long
+
         tenant_msg = ('   Copy your tenat ID from the web console and save it to file \n'  # pylint: disable=line-too-long
                       f'        $ echo $NEBIUS_TENANT_ID_PATH > {nebius.NEBIUS_TENANT_ID_PATH} \n'  # pylint: disable=line-too-long
                       '   Or if you have 1 tenant you can run:\n'  # pylint: disable=line-too-long
                       f'        $ nebius --format json iam whoami|jq -r \'.user_profile.tenants[0].tenant_id\' > {nebius.NEBIUS_TENANT_ID_PATH} \n')  # pylint: disable=line-too-long
-        if token is None:
-            return False, f'{token_msg}'
+        if not nebius.is_token_or_cred_file_exist():
+            return False, f'{token_cred_msg}'
         sdk = nebius.sdk()
         tenant_id = nebius.get_tenant_id()
         if tenant_id is None:
@@ -272,7 +274,7 @@ class Nebius(clouds.Cloud):
         except nebius.request_error() as e:
             return False, (
                 f'{e.status} \n'  # First line is indented by 4 spaces
-                f'{token_msg}'
+                f'{token_cred_msg}'
                 f'{tenant_msg}')
         return True, None
 

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/clouds/service_catalog/kubernetes_catalog.py

@@ -164,12 +164,13 @@ def _list_accelerators(
 
     accelerators_qtys: Set[Tuple[str, int]] = set()
     keys = lf.get_label_keys()
-    nodes = kubernetes_utils.get_kubernetes_nodes(context)
+    nodes = kubernetes_utils.get_kubernetes_nodes(context=context)
     pods = None
     if realtime:
         # Get the pods to get the real-time GPU usage
         try:
-            pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(context)
+            pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(
+                context=context)
         except kubernetes.api_exception() as e:
             if e.status == 403:
                 logger.warning(

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/exceptions.py

@@ -28,12 +28,19 @@ GIT_FATAL_EXIT_CODE = 128
 ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 
 
-def is_safe_exception(exc: Exception) -> bool:
+def is_safe_exception(exc: BaseException) -> bool:
     """Returns True if the exception is safe to send to clients.
 
     Safe exceptions are:
     1. Built-in exceptions
     2. SkyPilot's own exceptions
+
+    Args:
+        exc: The exception to check, accept BaseException to handle SystemExit
+            and KeyboardInterrupt.
+
+    Returns:
+        True if the exception is safe to send to clients, False otherwise.
     """
     module = type(exc).__module__
 
@@ -48,7 +55,7 @@ def is_safe_exception(exc: Exception) -> bool:
     return False
 
 
-def wrap_exception(exc: Exception) -> Exception:
+def wrap_exception(exc: BaseException) -> BaseException:
     """Wraps non-safe exceptions into SkyPilot exceptions
 
     This is used to wrap exceptions that are not safe to deserialize at clients.
@@ -64,7 +71,8 @@ def wrap_exception(exc: Exception) -> Exception:
                       error_type=type(exc).__name__)
 
 
-def serialize_exception(e: Exception) -> Dict[str, Any]:
+# Accept BaseException to handle SystemExit and KeyboardInterrupt
+def serialize_exception(e: BaseException) -> Dict[str, Any]:
     """Serialize the exception.
 
     This function also wraps any unsafe exceptions (e.g., cloud exceptions)
@@ -156,6 +164,15 @@ class ResourcesUnavailableError(Exception):
         return self
 
 
+class KubeAPIUnreachableError(ResourcesUnavailableError):
+    """Raised when the Kubernetes API is currently unreachable.
+
+    This is a subclass of ResourcesUnavailableError to trigger same failover
+    behavior as other ResourcesUnavailableError.
+    """
+    pass
+
+
 class InvalidCloudConfigs(Exception):
     """Raised when invalid configurations are provided for a given cloud."""
     pass

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/provision/kubernetes/network.py

@@ -157,7 +157,11 @@ def _cleanup_ports_for_loadbalancer(
 ) -> None:
     service_name = _LOADBALANCER_SERVICE_NAME.format(
         cluster_name_on_cloud=cluster_name_on_cloud)
+    # TODO(aylei): test coverage
+    context = provider_config.get(
+        'context', kubernetes_utils.get_current_kube_config_context_name())
     network_utils.delete_namespaced_service(
+        context=context,
         namespace=provider_config.get('namespace', 'default'),
         service_name=service_name,
     )
@@ -169,9 +173,12 @@ def _cleanup_ports_for_ingress(
     provider_config: Dict[str, Any],
 ) -> None:
     # Delete services for each port
+    context = provider_config.get(
+        'context', kubernetes_utils.get_current_kube_config_context_name())
     for port in ports:
         service_name = f'{cluster_name_on_cloud}--skypilot-svc--{port}'
         network_utils.delete_namespaced_service(
+            context=context,
             namespace=provider_config.get('namespace',
                                           kubernetes_utils.DEFAULT_NAMESPACE),
             service_name=service_name,

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/provision/kubernetes/network_utils.py

@@ -194,9 +194,10 @@ def create_or_replace_namespaced_service(
                                         _request_timeout=kubernetes.API_TIMEOUT)
 
 
-def delete_namespaced_service(namespace: str, service_name: str) -> None:
+def delete_namespaced_service(context: Optional[str], namespace: str,
+                              service_name: str) -> None:
     """Deletes a service resource."""
-    core_api = kubernetes.core_api()
+    core_api = kubernetes.core_api(context)
 
     try:
         core_api.delete_namespaced_service(

{skypilot_nightly-1.0.0.dev20250310 → skypilot_nightly-1.0.0.dev20250312}/sky/provision/kubernetes/utils.py

@@ -125,6 +125,10 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
         retry_interval: Initial seconds to wait between retries
         resource_type: Type of resource being accessed (e.g. 'node', 'pod').
             Used to provide more specific error messages.
+
+    Raises:
+        KubeAPIUnreachableError: If the API server of the given context is
+            unreachable.
     """
 
     def decorator(func):
@@ -135,6 +139,9 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
             backoff = common_utils.Backoff(initial_backoff=retry_interval,
                                            max_backoff_factor=3)
 
+            assert 'context' in kwargs, 'context is required'
+            context = kwargs.get('context')
+
             for attempt in range(max_retries):
                 try:
                     return func(*args, **kwargs)
@@ -160,6 +167,8 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
                 if resource_type else ''
             debug_cmd = f' To debug, run: kubectl get {resource_type}s' \
                 if resource_type else ''
+            if context:
+                debug_cmd += f' --context {context}'
 
             if isinstance(last_exception, kubernetes.max_retry_error()):
                 error_msg = f'Timed out{resource_msg} from Kubernetes cluster.'
@@ -170,7 +179,7 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
                 error_msg = (f'Kubernetes configuration error{resource_msg}: '
                              f'{str(last_exception)}')
 
-            raise exceptions.ResourcesUnavailableError(
+            raise exceptions.KubeAPIUnreachableError(
                 f'{error_msg}'
                 f' Please check if the cluster is healthy and retry.'
                 f'{debug_cmd}') from last_exception
@@ -529,7 +538,7 @@ def detect_gpu_label_formatter(
     """
     # Get all labels across all nodes
     node_labels: Dict[str, List[Tuple[str, str]]] = {}
-    nodes = get_kubernetes_nodes(context)
+    nodes = get_kubernetes_nodes(context=context)
     for node in nodes:
         node_labels[node.metadata.name] = []
         for label, value in node.metadata.labels.items():
@@ -564,7 +573,7 @@ def detect_accelerator_resource(
     """
     # Get the set of resources across all nodes
     cluster_resources: Set[str] = set()
-    nodes = get_kubernetes_nodes(context)
+    nodes = get_kubernetes_nodes(context=context)
     for node in nodes:
         cluster_resources.update(node.status.allocatable.keys())
     has_accelerator = (get_gpu_resource_key() in cluster_resources or
@@ -575,7 +584,7 @@ def detect_accelerator_resource(
 
 @annotations.lru_cache(scope='request', maxsize=10)
 @_retry_on_error(resource_type='node')
-def get_kubernetes_nodes(context: Optional[str] = None) -> List[Any]:
+def get_kubernetes_nodes(*, context: Optional[str] = None) -> List[Any]:
     """Gets the kubernetes nodes in the context.
 
     If context is None, gets the nodes in the current context.
@@ -589,8 +598,9 @@ def get_kubernetes_nodes(context: Optional[str] = None) -> List[Any]:
 
 
 @_retry_on_error(resource_type='pod')
-def get_all_pods_in_kubernetes_cluster(
-        context: Optional[str] = None) -> List[Any]:
+def get_all_pods_in_kubernetes_cluster(*,
+                                       context: Optional[str] = None
+                                      ) -> List[Any]:
     """Gets pods in all namespaces in kubernetes cluster indicated by context.
 
     Used for computing cluster resource usage.
@@ -619,9 +629,6 @@ def check_instance_fits(context: Optional[str],
         Optional[str]: Error message if the instance does not fit.
     """
 
-    # TODO(zhwu): this should check the node for specific context, instead
-    # of the default context to make failover fully functional.
-
     def check_cpu_mem_fits(candidate_instance_type: 'KubernetesInstanceType',
                            node_list: List[Any]) -> Tuple[bool, Optional[str]]:
         """Checks if the instance fits on the cluster based on CPU and memory.
@@ -682,7 +689,7 @@ def check_instance_fits(context: Optional[str],
                        f'{tpu_list_in_cluster_str}. Note that multi-host TPU '
                        'podslices are currently not unsupported.')
 
-    nodes = get_kubernetes_nodes(context)
+    nodes = get_kubernetes_nodes(context=context)
     k8s_instance_type = KubernetesInstanceType.\
         from_instance_type(instance)
     acc_type = k8s_instance_type.accelerator_type
@@ -846,7 +853,7 @@ def get_accelerator_label_key_value(
                 for label, value in label_list:
                     if (label_formatter.match_label_key(label) and
                             label_formatter.get_accelerator_from_label_value(
-                                value) == acc_type):
+                                value).lower() == acc_type.lower()):
                         if is_tpu_on_gke(acc_type):
                             assert isinstance(label_formatter,
                                               GKELabelFormatter)
@@ -2083,7 +2090,7 @@ def get_spot_label(
     """
     # Check if the cluster supports spot instances by checking nodes for known
     # spot label keys and values
-    for node in get_kubernetes_nodes(context):
+    for node in get_kubernetes_nodes(context=context):
         for _, (key, value) in SPOT_LABEL_MAP.items():
             if key in node.metadata.labels and node.metadata.labels[
                     key] == value:
@@ -2133,10 +2140,10 @@ def get_kubernetes_node_info(
         Dict[str, KubernetesNodeInfo]: Dictionary containing the node name as
             key and the KubernetesNodeInfo object as value
     """
-    nodes = get_kubernetes_nodes(context)
+    nodes = get_kubernetes_nodes(context=context)
     # Get the pods to get the real-time resource usage
     try:
-        pods = get_all_pods_in_kubernetes_cluster(context)
+        pods = get_all_pods_in_kubernetes_cluster(context=context)
     except kubernetes.api_exception() as e:
         if e.status == 403:
             pods = None
@@ -2443,7 +2450,7 @@ def is_multi_host_tpu(node_metadata_labels: dict) -> bool:
 
 def multi_host_tpu_exists_in_cluster(context: Optional[str] = None) -> bool:
     """Checks if there exists a multi-host TPU within the cluster."""
-    nodes = get_kubernetes_nodes(context)
+    nodes = get_kubernetes_nodes(context=context)
     for node in nodes:
         if is_multi_host_tpu(node.metadata.labels):
             return True