skypilot-nightly 1.0.0.dev2024053101__py3-none-any.whl → 1.0.0.dev2025022801__py3-none-any.whl

sky/clouds/kubernetes.py

@@ -1,18 +1,21 @@
 """Kubernetes."""
-import json
 import os
 import re
 import typing
-from typing import Dict, Iterator, List, Optional, Tuple
+from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 from sky import clouds
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import kubernetes
 from sky.clouds import service_catalog
+from sky.provision import instance_setup
 from sky.provision.kubernetes import network_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
+from sky.skylet import constants
+from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import registry
 from sky.utils import resources_utils
 from sky.utils import schemas
 
@@ -32,24 +35,21 @@ CREDENTIAL_PATH = os.environ.get('KUBECONFIG', DEFAULT_KUBECONFIG_PATH)
 _SKYPILOT_SYSTEM_NAMESPACE = 'skypilot-system'
 
 
-@clouds.CLOUD_REGISTRY.register
+@registry.CLOUD_REGISTRY.register(aliases=['k8s'])
 class Kubernetes(clouds.Cloud):
     """Kubernetes."""
 
     SKY_SSH_KEY_SECRET_NAME = 'sky-ssh-keys'
     SKY_SSH_JUMP_NAME = 'sky-ssh-jump-pod'
-    PORT_FORWARD_PROXY_CMD_TEMPLATE = \
-        'kubernetes-port-forward-proxy-command.sh.j2'
-    PORT_FORWARD_PROXY_CMD_PATH = '~/.sky/port-forward-proxy-cmd.sh'
-    # Timeout for resource provisioning. This timeout determines how long to
-    # wait for pod to be in pending status before giving up.
-    # Larger timeout may be required for autoscaling clusters, since autoscaler
-    # may take some time to provision new nodes.
-    # Note that this timeout includes time taken by the Kubernetes scheduler
-    # itself, which can be upto 2-3 seconds.
-    # For non-autoscaling clusters, we conservatively set this to 10s.
-    timeout = skypilot_config.get_nested(['kubernetes', 'provision_timeout'],
-                                         10)
+
+    LEGACY_SINGLETON_REGION = 'kubernetes'
+
+    # Limit the length of the cluster name to avoid exceeding the limit of 63
+    # characters for Kubernetes resources. We limit to 42 characters (63-21) to
+    # allow additional characters for creating ingress services to expose ports.
+    # These services are named as {cluster_name_on_cloud}--skypilot-svc--{port},
+    # where the suffix is 21 characters long.
+    _MAX_CLUSTER_NAME_LEN_LIMIT = 42
 
     _SUPPORTS_SERVICE_ACCOUNT_ON_REMOTE = True
 
@@ -57,8 +57,6 @@ class Kubernetes(clouds.Cloud):
     _DEFAULT_MEMORY_CPU_RATIO = 1
     _DEFAULT_MEMORY_CPU_RATIO_WITH_GPU = 4  # Allocate more memory for GPU tasks
     _REPR = 'Kubernetes'
-    _SINGLETON_REGION = 'kubernetes'
-    _regions: List[clouds.Region] = [clouds.Region(_SINGLETON_REGION)]
     _CLOUD_UNSUPPORTED_FEATURES = {
         # TODO(romilb): Stopping might be possible to implement with
         #  container checkpointing introduced in Kubernetes v1.25. See:
@@ -74,8 +72,8 @@ class Kubernetes(clouds.Cloud):
                                                              'Kubernetes.',
     }
 
-    IMAGE_CPU = 'skypilot:cpu-ubuntu-2004'
-    IMAGE_GPU = 'skypilot:gpu-ubuntu-2004'
+    IMAGE_CPU = 'skypilot:custom-cpu-ubuntu-2004'
+    IMAGE_GPU = 'skypilot:custom-gpu-ubuntu-2004'
 
     PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
     STATUS_VERSION = clouds.StatusVersion.SKYPILOT
@@ -85,15 +83,20 @@ class Kubernetes(clouds.Cloud):
         # Use a fresh user hash to avoid conflicts in the secret object naming.
         # This can happen when the controller is reusing the same user hash
         # through USER_ID_ENV_VAR but has a different SSH key.
-        fresh_user_hash = common_utils.get_user_hash(force_fresh_hash=True)
+        fresh_user_hash = common_utils.generate_user_hash()
         return f'ssh-publickey-{fresh_user_hash}'
 
     @classmethod
     def _unsupported_features_for_resources(
         cls, resources: 'resources_lib.Resources'
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
-        unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES
-        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth()
+        unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
+        context = resources.region
+        if context is None:
+            context = kubernetes_utils.get_current_kube_config_context_name()
+        # Features to be disabled for exec auth
+        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth(
+            context)
         if is_exec_auth:
             assert isinstance(message, str), message
             # Controllers cannot spin up new pods with exec auth.
@@ -102,19 +105,112 @@ class Kubernetes(clouds.Cloud):
             # Pod does not have permissions to terminate itself with exec auth.
             unsupported_features[
                 clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
+        # Allow spot instances if supported by the cluster
+        spot_label_key, _ = kubernetes_utils.get_spot_label(context)
+        if spot_label_key is not None:
+            unsupported_features.pop(
+                clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
         return unsupported_features
 
     @classmethod
-    def regions(cls) -> List[clouds.Region]:
-        return cls._regions
+    def max_cluster_name_length(cls) -> Optional[int]:
+        return cls._MAX_CLUSTER_NAME_LEN_LIMIT
+
+    @classmethod
+    @annotations.lru_cache(scope='global', maxsize=1)
+    def _log_skipped_contexts_once(cls, skipped_contexts: Tuple[str,
+                                                                ...]) -> None:
+        """Log skipped contexts for only once.
+
+        We don't directly cache the result of _filter_existing_allowed_contexts
+        as the admin policy may update the allowed contexts.
+        """
+        if skipped_contexts:
+            logger.warning(
+                f'Kubernetes contexts {set(skipped_contexts)!r} specified in '
+                '"allowed_contexts" not found in kubeconfig. '
+                'Ignoring these contexts.')
+
+    @classmethod
+    def existing_allowed_contexts(cls) -> List[str]:
+        """Get existing allowed contexts.
+
+        If None is returned in the list, it means that we are running in a pod
+        with in-cluster auth. In this case, we specify None context, which will
+        use the service account mounted in the pod.
+        """
+        all_contexts = kubernetes_utils.get_all_kube_context_names()
+        if not all_contexts:
+            return []
+
+        all_contexts = set(all_contexts)
+
+        allowed_contexts = skypilot_config.get_nested(
+            ('kubernetes', 'allowed_contexts'), None)
+
+        if allowed_contexts is None:
+            # Try kubeconfig if present
+            current_context = (
+                kubernetes_utils.get_current_kube_config_context_name())
+            if (current_context is None and
+                    kubernetes_utils.is_incluster_config_available()):
+                # If no kubeconfig contexts found, use in-cluster if available
+                current_context = kubernetes.in_cluster_context_name()
+            allowed_contexts = []
+            if current_context is not None:
+                allowed_contexts = [current_context]
+
+        existing_contexts = []
+        skipped_contexts = []
+        for context in allowed_contexts:
+            if context in all_contexts:
+                existing_contexts.append(context)
+            else:
+                skipped_contexts.append(context)
+        cls._log_skipped_contexts_once(tuple(skipped_contexts))
+        return existing_contexts
 
     @classmethod
     def regions_with_offering(cls, instance_type: Optional[str],
                               accelerators: Optional[Dict[str, int]],
                               use_spot: bool, region: Optional[str],
                               zone: Optional[str]) -> List[clouds.Region]:
-        # No notion of regions in Kubernetes - return a single region.
-        return cls.regions()
+        del accelerators, zone, use_spot  # unused
+        existing_contexts = cls.existing_allowed_contexts()
+
+        regions = []
+        for context in existing_contexts:
+            regions.append(clouds.Region(context))
+
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+
+        # Check if requested instance type will fit in the cluster.
+        # TODO(zhwu,romilb): autoscaler type needs to be regional (per
+        # kubernetes cluster/context).
+        regions_to_return = []
+        autoscaler_type = kubernetes_utils.get_autoscaler_type()
+        if autoscaler_type is None and instance_type is not None:
+            # If autoscaler is not set, check if the instance type fits in the
+            # cluster. Else, rely on the autoscaler to provision the right
+            # instance type without running checks. Worst case, if autoscaling
+            # fails, the pod will be stuck in pending state until
+            # provision_timeout, after which failover will be triggered.
+            for r in regions:
+                context = r.name
+                fits, reason = kubernetes_utils.check_instance_fits(
+                    context, instance_type)
+                if fits:
+                    regions_to_return.append(r)
+                else:
+                    logger.debug(
+                        f'Instance type {instance_type} does '
+                        'not fit in the Kubernetes cluster with context: '
+                        f'{context}. Reason: {reason}')
+        else:
+            regions_to_return = regions
+
+        return regions_to_return
 
     def instance_type_to_hourly_cost(self,
                                      instance_type: str,
@@ -140,17 +236,12 @@ class Kubernetes(clouds.Cloud):
     def __repr__(self):
         return self._REPR
 
-    @classmethod
-    def get_port(cls, svc_name) -> int:
-        ns = kubernetes_utils.get_current_kube_config_context_namespace()
-        return kubernetes_utils.get_port(svc_name, ns)
-
     @classmethod
     def get_default_instance_type(
             cls,
             cpus: Optional[str] = None,
             memory: Optional[str] = None,
-            disk_tier: Optional[resources_utils.DiskTier] = None) -> str:
+            disk_tier: Optional['resources_utils.DiskTier'] = None) -> str:
         # TODO(romilb): In the future, we may want to move the instance type
         #  selection + availability checking to a kubernetes_catalog module.
         del disk_tier  # Unused.
@@ -175,7 +266,7 @@ class Kubernetes(clouds.Cloud):
     def get_accelerators_from_instance_type(
         cls,
         instance_type: str,
-    ) -> Optional[Dict[str, int]]:
+    ) -> Optional[Dict[str, Union[int, float]]]:
         inst = kubernetes_utils.KubernetesInstanceType.from_instance_type(
             instance_type)
         return {
@@ -201,9 +292,9 @@ class Kubernetes(clouds.Cloud):
         accelerators: Optional[Dict[str, int]] = None,
         use_spot: bool = False,
     ) -> Iterator[Optional[List[clouds.Zone]]]:
-        del num_nodes, region, instance_type, accelerators, use_spot  # Unused.
-        for r in cls.regions():
-            yield r.zones
+        # Always yield None for zones, since Kubernetes does not have zones, and
+        # we should allow any region get to this point.
+        yield None
 
     @classmethod
     def get_zone_shell_cmd(cls) -> Optional[str]:
@@ -216,23 +307,46 @@ class Kubernetes(clouds.Cloud):
         # we don't have a notion of disk size in Kubernetes.
         return 0
 
+    @staticmethod
+    def _calculate_provision_timeout(num_nodes: int) -> int:
+        """Calculate provision timeout based on number of nodes.
+
+        The timeout scales linearly with the number of nodes to account for
+        scheduling overhead, but is capped to avoid excessive waiting.
+
+        Args:
+            num_nodes: Number of nodes being provisioned
+
+        Returns:
+            Timeout in seconds
+        """
+        base_timeout = 10  # Base timeout for single node
+        per_node_timeout = 0.2  # Additional seconds per node
+        max_timeout = 60  # Cap at 1 minute
+
+        return int(
+            min(base_timeout + (per_node_timeout * (num_nodes - 1)),
+                max_timeout))
+
     def make_deploy_resources_variables(
             self,
             resources: 'resources_lib.Resources',
-            cluster_name_on_cloud: str,
+            cluster_name: 'resources_utils.ClusterName',
             region: Optional['clouds.Region'],
             zones: Optional[List['clouds.Zone']],
+            num_nodes: int,
             dryrun: bool = False) -> Dict[str, Optional[str]]:
-        del cluster_name_on_cloud, zones, dryrun  # Unused.
+        del cluster_name, zones, dryrun  # Unused.
         if region is None:
-            region = self._regions[0]
+            context = kubernetes_utils.get_current_kube_config_context_name()
+        else:
+            context = region.name
+        assert context is not None, 'No context found in kubeconfig'
 
         r = resources
         acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
-        if acc_dict is not None:
-            custom_resources = json.dumps(acc_dict, separators=(',', ':'))
-        else:
-            custom_resources = None
+        custom_resources = resources_utils.make_ray_custom_resources_str(
+            acc_dict)
 
         # resources.memory and cpus are None if they are not explicitly set.
         # We fetch the default values for the instance type in that case.
@@ -244,9 +358,14 @@ class Kubernetes(clouds.Cloud):
         acc_count = k.accelerator_count if k.accelerator_count else 0
         acc_type = k.accelerator_type if k.accelerator_type else None
 
-        if resources.image_id is not None:
+        image_id_dict = resources.image_id
+        if image_id_dict is not None:
             # Use custom image specified in resources
-            image_id = resources.image_id['kubernetes']
+            if None in image_id_dict:
+                image_id = image_id_dict[None]
+            else:
+                assert resources.region in image_id_dict, image_id_dict
+                image_id = image_id_dict[resources.region]
             if image_id.startswith('docker:'):
                 image_id = image_id[len('docker:'):]
         else:
@@ -261,23 +380,48 @@ class Kubernetes(clouds.Cloud):
 
         k8s_acc_label_key = None
         k8s_acc_label_value = None
+        k8s_topology_label_key = None
+        k8s_topology_label_value = None
+        k8s_resource_key = None
+        tpu_requested = False
 
-        # If GPUs are requested, set node label to match the GPU type.
+        # If GPU/TPUs are requested, set node label to match the GPU/TPU type.
         if acc_count > 0 and acc_type is not None:
-            k8s_acc_label_key, k8s_acc_label_value = \
-                kubernetes_utils.get_gpu_label_key_value(acc_type)
+            (k8s_acc_label_key, k8s_acc_label_value, k8s_topology_label_key,
+             k8s_topology_label_value) = (
+                 kubernetes_utils.get_accelerator_label_key_value(
+                     context, acc_type, acc_count))
+            if (k8s_acc_label_key ==
+                    kubernetes_utils.GKELabelFormatter.TPU_LABEL_KEY):
+                tpu_requested = True
+                k8s_resource_key = kubernetes_utils.TPU_RESOURCE_KEY
+            else:
+                k8s_resource_key = kubernetes_utils.get_gpu_resource_key()
 
         port_mode = network_utils.get_port_mode(None)
 
         remote_identity = skypilot_config.get_nested(
             ('kubernetes', 'remote_identity'),
             schemas.get_default_remote_identity('kubernetes'))
-        if (remote_identity ==
+
+        if isinstance(remote_identity, dict):
+            # If remote_identity is a dict, use the service account for the
+            # current context
+            k8s_service_account_name = remote_identity.get(context, None)
+            if k8s_service_account_name is None:
+                err_msg = (f'Context {context!r} not found in '
+                           'remote identities from config.yaml')
+                raise ValueError(err_msg)
+        else:
+            # If remote_identity is not a dict, use
+            k8s_service_account_name = remote_identity
+
+        if (k8s_service_account_name ==
                 schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value):
             # SA name doesn't matter since automounting credentials is disabled
             k8s_service_account_name = 'default'
             k8s_automount_sa_token = 'false'
-        elif (remote_identity ==
+        elif (k8s_service_account_name ==
               schemas.RemoteIdentityOptions.SERVICE_ACCOUNT.value):
             # Use the default service account
             k8s_service_account_name = (
@@ -285,22 +429,55 @@ class Kubernetes(clouds.Cloud):
             k8s_automount_sa_token = 'true'
         else:
             # User specified a custom service account
-            k8s_service_account_name = remote_identity
             k8s_automount_sa_token = 'true'
 
         fuse_device_required = bool(resources.requires_fuse)
 
+        # Configure spot labels, if requested and supported
+        spot_label_key, spot_label_value = None, None
+        if resources.use_spot:
+            spot_label_key, spot_label_value = kubernetes_utils.get_spot_label()
+
+        # Timeout for resource provisioning. This timeout determines how long to
+        # wait for pod to be in pending status before giving up.
+        # Larger timeout may be required for autoscaling clusters, since
+        # autoscaler may take some time to provision new nodes.
+        # Note that this timeout includes time taken by the Kubernetes scheduler
+        # itself, which can be upto 2-3 seconds, and up to 10-15 seconds when
+        # scheduling 100s of pods.
+        # We use a linear scaling formula to determine the timeout based on the
+        # number of nodes.
+
+        timeout = self._calculate_provision_timeout(num_nodes)
+        timeout = skypilot_config.get_nested(
+            ('kubernetes', 'provision_timeout'),
+            timeout,
+            override_configs=resources.cluster_config_overrides)
+
+        # Set environment variables for the pod. Note that SkyPilot env vars
+        # are set separately when the task is run. These env vars are
+        # independent of the SkyPilot task to be run.
+        k8s_env_vars = {kubernetes.IN_CLUSTER_CONTEXT_NAME_ENV_VAR: context}
+
+        # We specify object-store-memory to be 500MB to avoid taking up too
+        # much memory on the head node. 'num-cpus' should be set to limit
+        # the CPU usage on the head pod, otherwise the ray cluster will use the
+        # CPU resources on the node instead within the pod.
+        custom_ray_options = {
+            'object-store-memory': 500000000,
+            # 'num-cpus' must be an integer, but we should not set it to 0 if
+            # cpus is <1.
+            'num-cpus': str(max(int(cpus), 1)),
+        }
         deploy_vars = {
             'instance_type': resources.instance_type,
             'custom_resources': custom_resources,
-            'region': region.name,
             'cpus': str(cpus),
             'memory': str(mem),
             'accelerator_count': str(acc_count),
-            'timeout': str(self.timeout),
-            'k8s_namespace':
-                kubernetes_utils.get_current_kube_config_context_namespace(),
+            'timeout': str(timeout),
             'k8s_port_mode': port_mode.value,
+            'k8s_networking_mode': network_utils.get_networking_mode().value,
             'k8s_ssh_key_secret_name': self.SKY_SSH_KEY_SECRET_NAME,
             'k8s_acc_label_key': k8s_acc_label_key,
             'k8s_acc_label_value': k8s_acc_label_value,
@@ -311,19 +488,51 @@ class Kubernetes(clouds.Cloud):
             'k8s_fuse_device_required': fuse_device_required,
             # Namespace to run the FUSE device manager in
             'k8s_skypilot_system_namespace': _SKYPILOT_SYSTEM_NAMESPACE,
+            'k8s_spot_label_key': spot_label_key,
+            'k8s_spot_label_value': spot_label_value,
+            'tpu_requested': tpu_requested,
+            'k8s_topology_label_key': k8s_topology_label_key,
+            'k8s_topology_label_value': k8s_topology_label_value,
+            'k8s_resource_key': k8s_resource_key,
+            'k8s_env_vars': k8s_env_vars,
             'image_id': image_id,
+            'ray_installation_commands': constants.RAY_INSTALLATION_COMMANDS,
+            'ray_head_start_command': instance_setup.ray_head_start_command(
+                custom_resources, custom_ray_options),
+            'skypilot_ray_port': constants.SKY_REMOTE_RAY_PORT,
+            'ray_worker_start_command': instance_setup.ray_worker_start_command(
+                custom_resources, custom_ray_options, no_restart=False),
         }
 
+        # Add kubecontext if it is set. It may be None if SkyPilot is running
+        # inside a pod with in-cluster auth.
+        if context is not None:
+            deploy_vars['k8s_context'] = context
+
+        namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+        deploy_vars['k8s_namespace'] = namespace
+
         return deploy_vars
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
+        # TODO(zhwu): This needs to be updated to return the correct region
+        # (context) that has enough resources.
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
+            regions = self.regions_with_offering(
+                resources.instance_type,
+                accelerators=resources.accelerators,
+                use_spot=resources.use_spot,
+                region=resources.region,
+                zone=resources.zone)
+            if not regions:
+                return resources_utils.FeasibleResources([], [], None)
             resources = resources.copy(accelerators=None)
-            return ([resources], fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([resources],
+                                                     fuzzy_candidate_list, None)
 
         def _make(instance_list):
             resource_list = []
@@ -365,33 +574,52 @@ class Kubernetes(clouds.Cloud):
                 kubernetes_utils.KubernetesInstanceType.from_resources(
                     gpu_task_cpus, gpu_task_memory, acc_count, acc_type).name)
 
-        # Check if requested instance type will fit in the cluster.
-        autoscaler_type = kubernetes_utils.get_autoscaler_type()
-        if autoscaler_type is None:
-            # If autoscaler is not set, check if the instance type fits in the
-            # cluster. Else, rely on the autoscaler to provision the right
-            # instance type without running checks. Worst case, if autoscaling
-            # fails, the pod will be stuck in pending state until
-            # provision_timeout, after which failover will be triggered.
-            fits, reason = kubernetes_utils.check_instance_fits(
-                chosen_instance_type)
-            if not fits:
-                logger.debug(f'Instance type {chosen_instance_type} does '
-                             'not fit in the Kubernetes cluster. '
-                             f'Reason: {reason}')
-                return [], []
-
+        # Check the availability of the specified instance type in all contexts.
+        available_regions = self.regions_with_offering(
+            chosen_instance_type,
+            accelerators=None,
+            use_spot=resources.use_spot,
+            region=resources.region,
+            zone=resources.zone)
+        if not available_regions:
+            return resources_utils.FeasibleResources([], [], None)
         # No fuzzy lists for Kubernetes
-        return _make([chosen_instance_type]), []
+        # We don't set the resources returned with regions, because the
+        # optimizer will further find the valid region (context) for the
+        # resources.
+        return resources_utils.FeasibleResources(_make([chosen_instance_type]),
+                                                 [], None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         # Test using python API
         try:
-            return kubernetes_utils.check_credentials()
-        except Exception as e:  # pylint: disable=broad-except
-            return (False, 'Credential check failed: '
-                    f'{common_utils.format_exception(e)}')
+            existing_allowed_contexts = cls.existing_allowed_contexts()
+        except ImportError as e:
+            return (False,
+                    f'{common_utils.format_exception(e, use_bracket=True)}')
+        if not existing_allowed_contexts:
+            if skypilot_config.loaded_config_path() is None:
+                check_skypilot_config_msg = ''
+            else:
+                check_skypilot_config_msg = (
+                    ' and check "allowed_contexts" in your '
+                    f'{skypilot_config.loaded_config_path()} file.')
+            return (False, 'No available context found in kubeconfig. '
+                    'Check if you have a valid kubeconfig file' +
+                    check_skypilot_config_msg)
+        reasons = []
+        for context in existing_allowed_contexts:
+            try:
+                check_result = kubernetes_utils.check_credentials(context)
+                if check_result[0]:
+                    return check_result
+                reasons.append(f'{context}: {check_result[1]}')
+            except Exception as e:  # pylint: disable=broad-except
+                return (False, f'Credential check failed for {context}: '
+                        f'{common_utils.format_exception(e)}')
+        return (False, 'Failed to find available context with working '
+                'credentials. Details:\n' + '\n'.join(reasons))
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
         if os.path.exists(os.path.expanduser(CREDENTIAL_PATH)):
@@ -406,30 +634,56 @@ class Kubernetes(clouds.Cloud):
             instance_type)
 
     def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
-        if region != self._SINGLETON_REGION:
+        if region == self.LEGACY_SINGLETON_REGION:
+            # For backward compatibility, we allow the region to be set to the
+            # legacy singleton region.
+            # TODO: Remove this after 0.9.0.
+            return region, zone
+
+        if region == kubernetes.in_cluster_context_name():
+            # If running incluster, we set region to IN_CLUSTER_REGION
+            # since there is no context name available.
+            return region, zone
+
+        all_contexts = kubernetes_utils.get_all_kube_context_names()
+
+        if region not in all_contexts:
             raise ValueError(
-                'Kubernetes support does not support setting region.'
-                ' Cluster used is determined by the kubeconfig.')
+                f'Context {region} not found in kubeconfig. Kubernetes only '
+                'supports context names as regions. Available '
+                f'contexts: {all_contexts}')
         if zone is not None:
             raise ValueError('Kubernetes support does not support setting zone.'
                              ' Cluster used is determined by the kubeconfig.')
         return region, zone
 
+    @staticmethod
+    def get_identity_from_context(context):
+        if 'namespace' in context['context']:
+            namespace = context['context']['namespace']
+        else:
+            namespace = kubernetes_utils.DEFAULT_NAMESPACE
+        user = context['context']['user']
+        cluster = context['context']['cluster']
+        identity_str = f'{cluster}_{user}_{namespace}'
+        return identity_str
+
     @classmethod
-    def get_current_user_identity(cls) -> Optional[List[str]]:
+    def get_user_identities(cls) -> Optional[List[List[str]]]:
         k8s = kubernetes.kubernetes
+        identities = []
         try:
-            _, current_context = k8s.config.list_kube_config_contexts()
-            if 'namespace' in current_context['context']:
-                namespace = current_context['context']['namespace']
-            else:
-                namespace = kubernetes_utils.DEFAULT_NAMESPACE
-
-            user = current_context['context']['user']
-            cluster = current_context['context']['cluster']
-            return [f'{cluster}_{user}_{namespace}']
+            all_contexts, current_context = (
+                k8s.config.list_kube_config_contexts())
         except k8s.config.config_exception.ConfigException:
             return None
+        # Add current context at the head of the list
+        current_identity = [cls.get_identity_from_context(current_context)]
+        identities.append(current_identity)
+        for context in all_contexts:
+            identity = [cls.get_identity_from_context(context)]
+            identities.append(identity)
+        return identities
 
     @classmethod
     def is_label_valid(cls, label_key: str,