skypilot-nightly 1.0.0.dev20240926__py3-none-any.whl → 1.0.0.dev20240928__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'e95332b9eb8de4cdcac464ff704bf64f3285e776'
+_SKYPILOT_COMMIT_SHA = 'dacf27348ae1446c3c93d0ee2fc57702c5366eac'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240926'
+__version__ = '1.0.0.dev20240928'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/kubernetes.py

@@ -75,15 +75,17 @@ def _load_config(context: Optional[str] = None):
                 suffix += f' Error: {str(e)}'
             # Check if exception was due to no current-context
             if 'Expected key current-context' in str(e):
-                err_str = ('Failed to load Kubernetes configuration. '
-                           'Kubeconfig does not contain any valid context(s).'
-                           f'{suffix}\n'
-                           '    If you were running a local Kubernetes '
-                           'cluster, run `sky local up` to start the cluster.')
+                err_str = (
+                    f'Failed to load Kubernetes configuration for {context!r}. '
+                    'Kubeconfig does not contain any valid context(s).'
+                    f'{suffix}\n'
+                    '    If you were running a local Kubernetes '
+                    'cluster, run `sky local up` to start the cluster.')
             else:
-                err_str = ('Failed to load Kubernetes configuration. '
-                           'Please check if your kubeconfig file exists at '
-                           f'~/.kube/config and is valid.{suffix}')
+                err_str = (
+                    f'Failed to load Kubernetes configuration for {context!r}. '
+                    'Please check if your kubeconfig file exists at '
+                    f'~/.kube/config and is valid.{suffix}')
             err_str += '\nTo disable Kubernetes for SkyPilot: run `sky check`.'
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(err_str) from None

sky/authentication.py

@@ -378,11 +378,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     public_key_path = os.path.expanduser(PUBLIC_SSH_KEY_PATH)
     secret_name = clouds.Kubernetes.SKY_SSH_KEY_SECRET_NAME
     secret_field_name = clouds.Kubernetes().ssh_key_secret_field_name
-    namespace = config['provider'].get(
-        'namespace',
-        kubernetes_utils.get_current_kube_config_context_namespace())
     context = config['provider'].get(
         'context', kubernetes_utils.get_current_kube_config_context_name())
+    namespace = config['provider'].get(
+        'namespace',
+        kubernetes_utils.get_kube_config_context_namespace(context))
     k8s = kubernetes.kubernetes
     with open(public_key_path, 'r', encoding='utf-8') as f:
         public_key = f.read()
@@ -425,8 +425,8 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
             ssh_jump_name,
             nodeport_mode,
             private_key_path=private_key_path,
-            namespace=namespace,
-            context=context)
+            context=context,
+            namespace=namespace)
     elif network_mode == port_forward_mode:
         # Using `kubectl port-forward` creates a direct tunnel to the pod and
         # does not require a ssh jump pod.
@@ -441,7 +441,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
         #   on GKE.
         ssh_target = config['cluster_name'] + '-head'
         ssh_proxy_cmd = kubernetes_utils.get_ssh_proxy_command(
-            ssh_target, port_forward_mode, private_key_path=private_key_path)
+            ssh_target,
+            port_forward_mode,
+            private_key_path=private_key_path,
+            context=context,
+            namespace=namespace)
     else:
         # This should never happen because we check for this in from_str above.
         raise ValueError(f'Unsupported networking mode: {network_mode_str}')

sky/backends/backend_utils.py

@@ -428,6 +428,7 @@ class SSHConfigHelper(object):
               HostName {ip}
               User {username}
               IdentityFile {ssh_key_path}
+              AddKeysToAgent yes
               IdentitiesOnly yes
               ForwardAgent yes
               StrictHostKeyChecking no

sky/backends/cloud_vm_ray_backend.py

@@ -2082,7 +2082,7 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
     """
     # Bump if any fields get added/removed/changed, and add backward
     # compaitibility logic in __setstate__.
-    _VERSION = 8
+    _VERSION = 9
 
     def __init__(
             self,
@@ -2516,6 +2516,19 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
         if version < 8:
             self.cached_cluster_info = None
 
+        if version < 9:
+            # For backward compatibility, we should update the region of a
+            # SkyPilot cluster on Kubernetes to the actual context it is using.
+            # pylint: disable=import-outside-toplevel
+            launched_resources = state['launched_resources']
+            if isinstance(launched_resources.cloud, clouds.Kubernetes):
+                yaml_config = common_utils.read_yaml(
+                    os.path.expanduser(state['_cluster_yaml']))
+                context = kubernetes_utils.get_context_from_config(
+                    yaml_config['provider'])
+                state['launched_resources'] = launched_resources.copy(
+                    region=context)
+
         self.__dict__.update(state)
 
         # Because the update_cluster_ips and update_ssh_ports

sky/cli.py

@@ -3026,14 +3026,11 @@ def show_gpus(
     kubernetes_is_enabled = sky_clouds.cloud_in_iterable(
         sky_clouds.Kubernetes(), global_user_state.get_cached_enabled_clouds())
 
-    if cloud_is_kubernetes and region is not None:
-        raise click.UsageError(
-            'The --region flag cannot be set with --cloud kubernetes.')
-
     def _list_to_str(lst):
         return ', '.join([str(e) for e in lst])
 
     def _get_kubernetes_realtime_gpu_table(
+            context: Optional[str] = None,
             name_filter: Optional[str] = None,
             quantity_filter: Optional[int] = None):
         if quantity_filter:
@@ -3048,7 +3045,7 @@ def show_gpus(
             gpus_only=True,
             clouds='kubernetes',
             name_filter=name_filter,
-            region_filter=region,
+            region_filter=context,
             quantity_filter=quantity_filter,
             case_sensitive=False)
         assert (set(counts.keys()) == set(capacity.keys()) == set(
@@ -3078,11 +3075,11 @@ def show_gpus(
             ])
         return realtime_gpu_table
 
-    def _get_kubernetes_node_info_table():
+    def _get_kubernetes_node_info_table(context: Optional[str]):
         node_table = log_utils.create_table(
             ['NODE_NAME', 'GPU_NAME', 'TOTAL_GPUS', 'FREE_GPUS'])
 
-        node_info_dict = kubernetes_utils.get_kubernetes_node_info()
+        node_info_dict = kubernetes_utils.get_kubernetes_node_info(context)
         for node_name, node_info in node_info_dict.items():
             node_table.add_row([
                 node_name, node_info.gpu_type,
@@ -3116,11 +3113,13 @@ def show_gpus(
             print_section_titles = False
             # If cloud is kubernetes, we want to show real-time capacity
             if kubernetes_is_enabled and (cloud is None or cloud_is_kubernetes):
+                context = region
                 try:
                     # If --cloud kubernetes is not specified, we want to catch
                     # the case where no GPUs are available on the cluster and
                     # print the warning at the end.
-                    k8s_realtime_table = _get_kubernetes_realtime_gpu_table()
+                    k8s_realtime_table = _get_kubernetes_realtime_gpu_table(
+                        context)
                 except ValueError as e:
                     if not cloud_is_kubernetes:
                         # Make it a note if cloud is not kubernetes
@@ -3129,9 +3128,10 @@ def show_gpus(
                 else:
                     print_section_titles = True
                     yield (f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
-                           f'Kubernetes GPUs{colorama.Style.RESET_ALL}\n')
+                           f'Kubernetes GPUs (Context: {context})'
+                           f'{colorama.Style.RESET_ALL}\n')
                     yield from k8s_realtime_table.get_string()
-                    k8s_node_table = _get_kubernetes_node_info_table()
+                    k8s_node_table = _get_kubernetes_node_info_table(context)
                     yield '\n\n'
                     yield (f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
                            f'Kubernetes per node GPU availability'
@@ -5072,15 +5072,7 @@ def local():
     pass
 
 
-@click.option('--gpus/--no-gpus',
-              default=True,
-              is_flag=True,
-              help='Launch cluster without GPU support even '
-              'if GPUs are detected on the host.')
-@local.command('up', cls=_DocumentedCodeCommand)
-@usage_lib.entrypoint
-def local_up(gpus: bool):
-    """Creates a local cluster."""
+def _deploy_local_cluster(gpus: bool):
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -5206,6 +5198,124 @@ def local_up(gpus: bool):
             f'{gpu_hint}')
 
 
+def _deploy_remote_cluster(ip_file: str, ssh_user: str, ssh_key_path: str,
+                           cleanup: bool):
+    success = False
+    path_to_package = os.path.dirname(os.path.dirname(__file__))
+    up_script_path = os.path.join(path_to_package, 'sky/utils/kubernetes',
+                                  'deploy_remote_cluster.sh')
+    # Get directory of script and run it from there
+    cwd = os.path.dirname(os.path.abspath(up_script_path))
+
+    deploy_command = f'{up_script_path} {ip_file} {ssh_user} {ssh_key_path}'
+    if cleanup:
+        deploy_command += ' --cleanup'
+
+    # Convert the command to a format suitable for subprocess
+    deploy_command = shlex.split(deploy_command)
+
+    # Setup logging paths
+    run_timestamp = backend_utils.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_up.log')
+    tail_cmd = 'tail -n100 -f ' + log_path
+
+    # Check if ~/.kube/config exists:
+    if os.path.exists(os.path.expanduser('~/.kube/config')):
+        click.echo('Found existing kube config. '
+                   'It will be backed up to ~/.kube/config.bak.')
+    style = colorama.Style
+    click.echo('To view detailed progress: '
+               f'{style.BRIGHT}{tail_cmd}{style.RESET_ALL}')
+    if cleanup:
+        msg_str = 'Cleaning up remote cluster...'
+    else:
+        msg_str = 'Deploying remote cluster...'
+    with rich_utils.safe_status(f'[bold cyan]{msg_str}'):
+        returncode, _, stderr = log_lib.run_with_log(
+            cmd=deploy_command,
+            log_path=log_path,
+            require_outputs=True,
+            stream_logs=False,
+            line_processor=log_utils.SkyRemoteUpLineProcessor(),
+            cwd=cwd)
+    if returncode == 0:
+        success = True
+    else:
+        with ux_utils.print_exception_no_traceback():
+            raise RuntimeError(
+                'Failed to deploy remote cluster. '
+                f'Full log: {log_path}'
+                f'\nError: {style.BRIGHT}{stderr}{style.RESET_ALL}')
+
+    if success:
+        if cleanup:
+            click.echo(f'{colorama.Fore.GREEN}'
+                       '🎉 Remote cluster cleaned up successfully.'
+                       f'{style.RESET_ALL}')
+        else:
+            click.echo('Cluster deployment done. You can now run tasks on '
+                       'this cluster.\nE.g., run a task with: '
+                       'sky launch --cloud kubernetes -- echo hello world.'
+                       f'\n{colorama.Fore.GREEN}🎉 Remote cluster deployed '
+                       f'successfully. {style.RESET_ALL}')
+
+
+@click.option('--gpus/--no-gpus',
+              default=True,
+              is_flag=True,
+              help='Launch cluster without GPU support even '
+              'if GPUs are detected on the host.')
+@click.option(
+    '--ips',
+    type=str,
+    required=False,
+    help='Path to the file containing IP addresses of remote machines.')
+@click.option('--ssh-user',
+              type=str,
+              required=False,
+              help='SSH username for accessing remote machines.')
+@click.option('--ssh-key-path',
+              type=str,
+              required=False,
+              help='Path to the SSH private key.')
+@click.option('--cleanup',
+              is_flag=True,
+              help='Clean up the remote cluster instead of deploying it.')
+@local.command('up', cls=_DocumentedCodeCommand)
+@usage_lib.entrypoint
+def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
+             cleanup: bool):
+    """Creates a local or remote cluster."""
+
+    def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
+        # If any of --ips, --ssh-user, or --ssh-key-path is specified,
+        # all must be specified
+        if bool(ips) or bool(ssh_user) or bool(ssh_key_path):
+            if not (ips and ssh_user and ssh_key_path):
+                raise click.BadParameter(
+                    'All --ips, --ssh-user, and --ssh-key-path '
+                    'must be specified together.')
+
+        # --cleanup can only be used if --ips, --ssh-user and --ssh-key-path
+        # are all provided
+        if cleanup and not (ips and ssh_user and ssh_key_path):
+            raise click.BadParameter('--cleanup can only be used with '
+                                     '--ips, --ssh-user and --ssh-key-path.')
+
+    _validate_args(ips, ssh_user, ssh_key_path, cleanup)
+
+    # If remote deployment arguments are specified, run remote up script
+    if ips and ssh_user and ssh_key_path:
+        # Convert ips and ssh_key_path to absolute paths
+        ips = os.path.abspath(ips)
+        ssh_key_path = os.path.abspath(ssh_key_path)
+        _deploy_remote_cluster(ips, ssh_user, ssh_key_path, cleanup)
+    else:
+        # Run local deployment (kind) if no remote args are specified
+        _deploy_local_cluster(gpus)
+
+
 @local.command('down', cls=_DocumentedCodeCommand)
 @usage_lib.entrypoint
 def local_down():

sky/clouds/kubernetes.py

@@ -1,4 +1,5 @@
 """Kubernetes."""
+import functools
 import json
 import os
 import re
@@ -52,8 +53,7 @@ class Kubernetes(clouds.Cloud):
     _DEFAULT_MEMORY_CPU_RATIO = 1
     _DEFAULT_MEMORY_CPU_RATIO_WITH_GPU = 4  # Allocate more memory for GPU tasks
     _REPR = 'Kubernetes'
-    _SINGLETON_REGION = 'kubernetes'
-    _regions: List[clouds.Region] = [clouds.Region(_SINGLETON_REGION)]
+    _LEGACY_SINGLETON_REGION = 'kubernetes'
     _CLOUD_UNSUPPORTED_FEATURES = {
         # TODO(romilb): Stopping might be possible to implement with
         #  container checkpointing introduced in Kubernetes v1.25. See:
@@ -88,8 +88,12 @@ class Kubernetes(clouds.Cloud):
         cls, resources: 'resources_lib.Resources'
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
+        context = resources.region
+        if context is None:
+            context = kubernetes_utils.get_current_kube_config_context_name()
         # Features to be disabled for exec auth
-        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth()
+        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth(
+            context)
         if is_exec_auth:
             assert isinstance(message, str), message
             # Controllers cannot spin up new pods with exec auth.
@@ -99,7 +103,7 @@ class Kubernetes(clouds.Cloud):
             unsupported_features[
                 clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
         # Allow spot instances if supported by the cluster
-        spot_label_key, _ = kubernetes_utils.get_spot_label()
+        spot_label_key, _ = kubernetes_utils.get_spot_label(context)
         if spot_label_key is not None:
             unsupported_features.pop(
                 clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
@@ -110,16 +114,87 @@ class Kubernetes(clouds.Cloud):
         return cls._MAX_CLUSTER_NAME_LEN_LIMIT
 
     @classmethod
-    def regions(cls) -> List[clouds.Region]:
-        return cls._regions
+    @functools.lru_cache(maxsize=1)
+    def _log_skipped_contexts_once(cls, skipped_contexts: Tuple[str,
+                                                                ...]) -> None:
+        """Log skipped contexts for only once.
+
+        We don't directly cache the result of _filter_existing_allowed_contexts
+        as the admin policy may update the allowed contexts.
+        """
+        if skipped_contexts:
+            logger.warning(
+                f'Kubernetes contexts {set(skipped_contexts)!r} specified in '
+                '"allowed_contexts" not found in kubeconfig. '
+                'Ignoring these contexts.')
+
+    @classmethod
+    def _existing_allowed_contexts(cls) -> List[str]:
+        """Get existing allowed contexts."""
+        all_contexts = kubernetes_utils.get_all_kube_config_context_names()
+        if all_contexts is None:
+            return []
+        all_contexts = set(all_contexts)
+
+        allowed_contexts = skypilot_config.get_nested(
+            ('kubernetes', 'allowed_contexts'), None)
+
+        if allowed_contexts is None:
+            current_context = (
+                kubernetes_utils.get_current_kube_config_context_name())
+            allowed_contexts = []
+            if current_context is not None:
+                allowed_contexts = [current_context]
+
+        existing_contexts = []
+        skipped_contexts = []
+        for context in allowed_contexts:
+            if context in all_contexts:
+                existing_contexts.append(context)
+            else:
+                skipped_contexts.append(context)
+        cls._log_skipped_contexts_once(tuple(skipped_contexts))
+        return existing_contexts
 
     @classmethod
     def regions_with_offering(cls, instance_type: Optional[str],
                               accelerators: Optional[Dict[str, int]],
                               use_spot: bool, region: Optional[str],
                               zone: Optional[str]) -> List[clouds.Region]:
-        # No notion of regions in Kubernetes - return a single region.
-        return cls.regions()
+        del accelerators, zone, use_spot  # unused
+        existing_contexts = cls._existing_allowed_contexts()
+
+        regions = [clouds.Region(context) for context in existing_contexts]
+
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+
+        # Check if requested instance type will fit in the cluster.
+        # TODO(zhwu,romilb): autoscaler type needs to be regional (per
+        # kubernetes cluster/context).
+        regions_to_return = []
+        autoscaler_type = kubernetes_utils.get_autoscaler_type()
+        if autoscaler_type is None and instance_type is not None:
+            # If autoscaler is not set, check if the instance type fits in the
+            # cluster. Else, rely on the autoscaler to provision the right
+            # instance type without running checks. Worst case, if autoscaling
+            # fails, the pod will be stuck in pending state until
+            # provision_timeout, after which failover will be triggered.
+            for r in regions:
+                context = r.name
+                fits, reason = kubernetes_utils.check_instance_fits(
+                    context, instance_type)
+                if fits:
+                    regions_to_return.append(r)
+                else:
+                    logger.debug(
+                        f'Instance type {instance_type} does '
+                        'not fit in the Kubernetes cluster with context: '
+                        f'{context}. Reason: {reason}')
+        else:
+            regions_to_return = regions
+
+        return regions_to_return
 
     def instance_type_to_hourly_cost(self,
                                      instance_type: str,
@@ -201,9 +276,9 @@ class Kubernetes(clouds.Cloud):
         accelerators: Optional[Dict[str, int]] = None,
         use_spot: bool = False,
     ) -> Iterator[Optional[List[clouds.Zone]]]:
-        del num_nodes, region, instance_type, accelerators, use_spot  # Unused.
-        for r in cls.regions():
-            yield r.zones
+        # Always yield None for zones, since Kubernetes does not have zones, and
+        # we should allow any region get to this point.
+        yield None
 
     @classmethod
     def get_zone_shell_cmd(cls) -> Optional[str]:
@@ -225,7 +300,10 @@ class Kubernetes(clouds.Cloud):
             dryrun: bool = False) -> Dict[str, Optional[str]]:
         del cluster_name, zones, dryrun  # Unused.
         if region is None:
-            region = self._regions[0]
+            context = kubernetes_utils.get_current_kube_config_context_name()
+        else:
+            context = region.name
+        assert context is not None, 'No context found in kubeconfig'
 
         r = resources
         acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
@@ -244,9 +322,14 @@ class Kubernetes(clouds.Cloud):
         acc_count = k.accelerator_count if k.accelerator_count else 0
         acc_type = k.accelerator_type if k.accelerator_type else None
 
-        if resources.image_id is not None:
+        image_id_dict = resources.image_id
+        if image_id_dict is not None:
             # Use custom image specified in resources
-            image_id = resources.image_id['kubernetes']
+            if None in image_id_dict:
+                image_id = image_id_dict[None]
+            else:
+                assert resources.region in image_id_dict, image_id_dict
+                image_id = image_id_dict[resources.region]
             if image_id.startswith('docker:'):
                 image_id = image_id[len('docker:'):]
         else:
@@ -265,7 +348,7 @@ class Kubernetes(clouds.Cloud):
         # If GPUs are requested, set node label to match the GPU type.
         if acc_count > 0 and acc_type is not None:
             k8s_acc_label_key, k8s_acc_label_value = \
-                kubernetes_utils.get_gpu_label_key_value(acc_type)
+                kubernetes_utils.get_gpu_label_key_value(context, acc_type)
 
         port_mode = network_utils.get_port_mode(None)
 
@@ -309,13 +392,10 @@ class Kubernetes(clouds.Cloud):
         deploy_vars = {
             'instance_type': resources.instance_type,
             'custom_resources': custom_resources,
-            'region': region.name,
             'cpus': str(cpus),
             'memory': str(mem),
             'accelerator_count': str(acc_count),
             'timeout': str(timeout),
-            'k8s_namespace':
-                kubernetes_utils.get_current_kube_config_context_namespace(),
             'k8s_port_mode': port_mode.value,
             'k8s_networking_mode': network_utils.get_networking_mode().value,
             'k8s_ssh_key_secret_name': self.SKY_SSH_KEY_SECRET_NAME,
@@ -335,18 +415,30 @@ class Kubernetes(clouds.Cloud):
 
         # Add kubecontext if it is set. It may be None if SkyPilot is running
         # inside a pod with in-cluster auth.
-        curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        if curr_context is not None:
-            deploy_vars['k8s_context'] = curr_context
+        if context is not None:
+            deploy_vars['k8s_context'] = context
+
+        namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+        deploy_vars['k8s_namespace'] = namespace
 
         return deploy_vars
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
     ) -> 'resources_utils.FeasibleResources':
+        # TODO(zhwu): This needs to be updated to return the correct region
+        # (context) that has enough resources.
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
+            regions = self.regions_with_offering(
+                resources.instance_type,
+                accelerators=resources.accelerators,
+                use_spot=resources.use_spot,
+                region=resources.region,
+                zone=resources.zone)
+            if not regions:
+                return resources_utils.FeasibleResources([], [], None)
             resources = resources.copy(accelerators=None)
             return resources_utils.FeasibleResources([resources],
                                                      fuzzy_candidate_list, None)
@@ -391,34 +483,48 @@ class Kubernetes(clouds.Cloud):
                 kubernetes_utils.KubernetesInstanceType.from_resources(
                     gpu_task_cpus, gpu_task_memory, acc_count, acc_type).name)
 
-        # Check if requested instance type will fit in the cluster.
-        autoscaler_type = kubernetes_utils.get_autoscaler_type()
-        if autoscaler_type is None:
-            # If autoscaler is not set, check if the instance type fits in the
-            # cluster. Else, rely on the autoscaler to provision the right
-            # instance type without running checks. Worst case, if autoscaling
-            # fails, the pod will be stuck in pending state until
-            # provision_timeout, after which failover will be triggered.
-            fits, reason = kubernetes_utils.check_instance_fits(
-                chosen_instance_type)
-            if not fits:
-                logger.debug(f'Instance type {chosen_instance_type} does '
-                             'not fit in the Kubernetes cluster. '
-                             f'Reason: {reason}')
-                return resources_utils.FeasibleResources([], [], reason)
-
+        # Check the availability of the specified instance type in all contexts.
+        available_regions = self.regions_with_offering(
+            chosen_instance_type,
+            accelerators=None,
+            use_spot=resources.use_spot,
+            region=resources.region,
+            zone=resources.zone)
+        if not available_regions:
+            return resources_utils.FeasibleResources([], [], None)
         # No fuzzy lists for Kubernetes
+        # We don't set the resources returned with regions, because the
+        # optimizer will further find the valid region (context) for the
+        # resources.
         return resources_utils.FeasibleResources(_make([chosen_instance_type]),
                                                  [], None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         # Test using python API
-        try:
-            return kubernetes_utils.check_credentials()
-        except Exception as e:  # pylint: disable=broad-except
-            return (False, 'Credential check failed: '
-                    f'{common_utils.format_exception(e)}')
+        existing_allowed_contexts = cls._existing_allowed_contexts()
+        if not existing_allowed_contexts:
+            if skypilot_config.loaded_config_path() is None:
+                check_skypilot_config_msg = ''
+            else:
+                check_skypilot_config_msg = (
+                    ' and check "allowed_contexts" in your '
+                    f'{skypilot_config.loaded_config_path()} file.')
+            return (False, 'No available context found in kubeconfig. '
+                    'Check if you have a valid kubeconfig file' +
+                    check_skypilot_config_msg)
+        reasons = []
+        for context in existing_allowed_contexts:
+            try:
+                check_result = kubernetes_utils.check_credentials(context)
+                if check_result[0]:
+                    return check_result
+                reasons.append(f'{context}: {check_result[1]}')
+            except Exception as e:  # pylint: disable=broad-except
+                return (False, f'Credential check failed for {context}: '
+                        f'{common_utils.format_exception(e)}')
+        return (False, 'Failed to find available context with working '
+                'credentials. Details:\n' + '\n'.join(reasons))
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
         if os.path.exists(os.path.expanduser(CREDENTIAL_PATH)):
@@ -433,10 +539,20 @@ class Kubernetes(clouds.Cloud):
             instance_type)
 
     def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
-        if region != self._SINGLETON_REGION:
+        if region == self._LEGACY_SINGLETON_REGION:
+            # For backward compatibility, we allow the region to be set to the
+            # legacy singletonton region.
+            # TODO: Remove this after 0.9.0.
+            return region, zone
+
+        all_contexts = kubernetes_utils.get_all_kube_config_context_names()
+        if all_contexts is None:
+            all_contexts = []
+        if region not in all_contexts:
             raise ValueError(
-                'Kubernetes support does not support setting region.'
-                ' Cluster used is determined by the kubeconfig.')
+                f'Context {region} not found in kubeconfig. Kubernetes only '
+                'supports context names as regions. Available '
+                f'contexts: {all_contexts}')
         if zone is not None:
             raise ValueError('Kubernetes support does not support setting zone.'
                              ' Cluster used is determined by the kubeconfig.')