skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/clouds/kubernetes.py

@@ -1,35 +1,40 @@
 """Kubernetes."""
+import concurrent.futures
 import os
 import re
-import typing
+import subprocess
+import tempfile
 from typing import Dict, Iterator, List, Optional, Set, Tuple, Union
 
+import colorama
+
+from sky import catalog
 from sky import clouds
 from sky import exceptions
+from sky import resources as resources_lib
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import kubernetes
-from sky.clouds import service_catalog
+from sky.clouds.utils import gcp_utils
 from sky.provision import instance_setup
+from sky.provision.gcp import constants as gcp_constants
 from sky.provision.kubernetes import network_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
+from sky.provision.kubernetes.utils import is_tpu_on_gke
+from sky.provision.kubernetes.utils import KubernetesHighPerformanceNetworkType
+from sky.provision.kubernetes.utils import normalize_tpu_accelerator_name
 from sky.skylet import constants
 from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import env_options
+from sky.utils import kubernetes_enums
 from sky.utils import registry
 from sky.utils import resources_utils
 from sky.utils import schemas
-
-if typing.TYPE_CHECKING:
-    # Renaming to avoid shadowing variables.
-    from sky import resources as resources_lib
+from sky.utils import volume as volume_lib
 
 logger = sky_logging.init_logger(__name__)
 
-# Check if KUBECONFIG is set, and use it if it is.
-DEFAULT_KUBECONFIG_PATH = '~/.kube/config'
-CREDENTIAL_PATH = os.environ.get('KUBECONFIG', DEFAULT_KUBECONFIG_PATH)
-
 # Namespace for SkyPilot resources shared across multiple tenants on the
 # same cluster (even if they might be running in different namespaces).
 # E.g., FUSE device manager daemonset is run in this namespace.
@@ -44,9 +49,6 @@ _FUSERMOUNT_SHARED_DIR = '/var/run/fusermount'
 class Kubernetes(clouds.Cloud):
     """Kubernetes."""
 
-    SKY_SSH_KEY_SECRET_NAME = 'sky-ssh-keys'
-    SKY_SSH_JUMP_NAME = 'sky-ssh-jump-pod'
-
     # Limit the length of the cluster name to avoid exceeding the limit of 63
     # characters for Kubernetes resources. We limit to 42 characters (63-21) to
     # allow additional characters for creating ingress services to expose ports.
@@ -54,9 +56,12 @@ class Kubernetes(clouds.Cloud):
     # where the suffix is 21 characters long.
     _MAX_CLUSTER_NAME_LEN_LIMIT = 42
 
+    _MAX_VOLUME_NAME_LEN_LIMIT = 253
+
     _SUPPORTS_SERVICE_ACCOUNT_ON_REMOTE = True
 
     _DEFAULT_NUM_VCPUS = 2
+    _DEFAULT_NUM_VCPUS_WITH_GPU = 4
     _DEFAULT_MEMORY_CPU_RATIO = 1
     _DEFAULT_MEMORY_CPU_RATIO_WITH_GPU = 4  # Allocate more memory for GPU tasks
     _REPR = 'Kubernetes'
@@ -73,6 +78,12 @@ class Kubernetes(clouds.Cloud):
                                                              'tiers are not '
                                                              'supported in '
                                                              'Kubernetes.',
+        clouds.CloudImplementationFeatures.CUSTOM_MULTI_NETWORK:
+            ('Customized multiple network interfaces are not supported in '
+             'Kubernetes.'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is not supported in this Kubernetes '
+             'cluster.'),
     }
 
     IMAGE_CPU = 'skypilot:custom-cpu-ubuntu-2004'
@@ -86,47 +97,52 @@ class Kubernetes(clouds.Cloud):
     # Set of contexts that has logged as temporarily unreachable
     logged_unreachable_contexts: Set[str] = set()
 
-    @property
-    def ssh_key_secret_field_name(self):
-        # Use a fresh user hash to avoid conflicts in the secret object naming.
-        # This can happen when the controller is reusing the same user hash
-        # through USER_ID_ENV_VAR but has a different SSH key.
-        fresh_user_hash = common_utils.generate_user_hash()
-        return f'ssh-publickey-{fresh_user_hash}'
-
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         # TODO(aylei): features need to be regional (per context) to make
         # multi-kubernetes selection/failover work.
         unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
-        context = resources.region
+        context = region if region is not None else resources.region
         if context is None:
-            context = kubernetes_utils.get_current_kube_config_context_name()
-        # Features to be disabled for exec auth
-        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth(
-            context)
-        if is_exec_auth:
-            assert isinstance(message, str), message
-            # Controllers cannot spin up new pods with exec auth.
-            unsupported_features[
-                clouds.CloudImplementationFeatures.HOST_CONTROLLERS] = message
-            # Pod does not have permissions to down itself with exec auth.
-            unsupported_features[
-                clouds.CloudImplementationFeatures.AUTODOWN] = message
+            contexts = cls.existing_allowed_contexts()
+        else:
+            contexts = [context]
         unsupported_features[clouds.CloudImplementationFeatures.STOP] = (
             'Stopping clusters is not supported on Kubernetes.')
         unsupported_features[clouds.CloudImplementationFeatures.AUTOSTOP] = (
             'Auto-stop is not supported on Kubernetes.')
-        # Allow spot instances if supported by the cluster
-        try:
-            spot_label_key, _ = kubernetes_utils.get_spot_label(context)
-            if spot_label_key is not None:
-                unsupported_features.pop(
-                    clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
-        except exceptions.KubeAPIUnreachableError as e:
-            cls._log_unreachable_context(context, str(e))
+        for context in contexts:
+            # Allow spot instances if supported by the cluster
+            try:
+                # Run spot label check and network type detection concurrently
+                # as they are independent operations
+                with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=2) as executor:
+                    spot_future = executor.submit(
+                        kubernetes_utils.get_spot_label, context)
+                    network_future = executor.submit(cls._detect_network_type,
+                                                     context,
+                                                     resources.network_tier)
+
+                    spot_label_key, _ = spot_future.result()
+                    if spot_label_key is not None:
+                        unsupported_features.pop(
+                            clouds.CloudImplementationFeatures.SPOT_INSTANCE,
+                            None)
+
+                    # Allow custom network tier if supported by the cluster
+                    # (e.g., Nebius clusters with high performance networking)
+                    network_type, _ = network_future.result()
+                    if network_type.supports_high_performance_networking():
+                        unsupported_features.pop(
+                            clouds.CloudImplementationFeatures.
+                            CUSTOM_NETWORK_TIER, None)
+            except exceptions.KubeAPIUnreachableError as e:
+                cls._log_unreachable_context(context, str(e))
         return unsupported_features
 
     @classmethod
@@ -149,7 +165,7 @@ class Kubernetes(clouds.Cloud):
                 'Ignoring these contexts.')
 
     @classmethod
-    def existing_allowed_contexts(cls) -> List[str]:
+    def existing_allowed_contexts(cls, silent: bool = False) -> List[str]:
         """Get existing allowed contexts.
 
         If None is returned in the list, it means that we are running in a pod
@@ -162,15 +178,35 @@ class Kubernetes(clouds.Cloud):
 
         all_contexts = set(all_contexts)
 
-        allowed_contexts = skypilot_config.get_nested(
-            ('kubernetes', 'allowed_contexts'), None)
+        # Allowed_contexts specified for workspace should take precedence over
+        # the global allowed_contexts.
+        allowed_contexts = skypilot_config.get_workspace_cloud(
+            'kubernetes').get('allowed_contexts', None)
+        if allowed_contexts is None:
+            allowed_contexts = skypilot_config.get_effective_region_config(
+                cloud='kubernetes',
+                region=None,
+                keys=('allowed_contexts',),
+                default_value=None)
+
+        # Exclude contexts starting with `ssh-`
+        # TODO(romilb): Remove when SSH Node Pools use a separate kubeconfig.
+        all_contexts = [
+            ctx for ctx in all_contexts if not ctx.startswith('ssh-')
+        ]
+
+        allow_all_contexts = allowed_contexts == 'all' or (
+            allowed_contexts is None and
+            env_options.Options.ALLOW_ALL_KUBERNETES_CONTEXTS.get())
+        if allow_all_contexts:
+            allowed_contexts = all_contexts
 
         if allowed_contexts is None:
             # Try kubeconfig if present
             current_context = (
                 kubernetes_utils.get_current_kube_config_context_name())
-            if (current_context is None and
-                    kubernetes_utils.is_incluster_config_available()):
+            if ((current_context is None or current_context.startswith('ssh-'))
+                    and kubernetes_utils.is_incluster_config_available()):
                 # If no kubeconfig contexts found, use in-cluster if available
                 current_context = kubernetes.in_cluster_context_name()
             allowed_contexts = []
@@ -183,8 +219,12 @@ class Kubernetes(clouds.Cloud):
             if context in all_contexts:
                 existing_contexts.append(context)
             else:
+                # Skip SSH Node Pool contexts
+                if context.startswith('ssh-'):
+                    continue
                 skipped_contexts.append(context)
-        cls._log_skipped_contexts_once(tuple(skipped_contexts))
+        if not silent:
+            cls._log_skipped_contexts_once(tuple(skipped_contexts))
         return existing_contexts
 
     @classmethod
@@ -218,10 +258,15 @@ class Kubernetes(clouds.Cloud):
                 'refresh Kubernetes availability if permanent.')
 
     @classmethod
-    def regions_with_offering(cls, instance_type: Optional[str],
-                              accelerators: Optional[Dict[str, int]],
-                              use_spot: bool, region: Optional[str],
-                              zone: Optional[str]) -> List[clouds.Region]:
+    def regions_with_offering(
+        cls,
+        instance_type: Optional[str],
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
+    ) -> List[clouds.Region]:
         del accelerators, zone, use_spot  # unused
         existing_contexts = cls.existing_allowed_contexts()
 
@@ -231,6 +276,19 @@ class Kubernetes(clouds.Cloud):
 
         if region is not None:
             regions = [r for r in regions if r.name == region]
+        if resources is not None:
+            filtered_regions = []
+            resources_required_features = resources.get_required_cloud_features(
+            )
+            for r in regions:
+                try:
+                    cls.check_features_are_supported(
+                        resources, resources_required_features, r.name)
+                    filtered_regions.append(r)
+                except exceptions.NotSupportedError as e:
+                    logger.info(f'Filter out context: {r.name}, reason: {e}')
+                    continue
+            regions = filtered_regions
 
         # Check if requested instance type will fit in the cluster.
         # TODO(zhwu,romilb): autoscaler type needs to be regional (per
@@ -238,22 +296,6 @@ class Kubernetes(clouds.Cloud):
         if instance_type is None:
             return regions
 
-        autoscaler_type = kubernetes_utils.get_autoscaler_type()
-        if (autoscaler_type is not None and not kubernetes_utils.get_autoscaler(
-                autoscaler_type).can_query_backend):
-            # Unsupported autoscaler type. Rely on the autoscaler to
-            # provision the right instance type without running checks.
-            # Worst case, if autoscaling fails, the pod will be stuck in
-            # pending state until provision_timeout, after which failover
-            # will be triggered.
-            #
-            # Removing this if statement produces the same behavior,
-            # because can_create_new_instance_of_type() always returns True
-            # for unsupported autoscaler types.
-            # This check is here as a performance optimization to avoid
-            # further code executions that is known to return this result.
-            return regions
-
         regions_to_return = []
         for r in regions:
             context = r.name
@@ -270,9 +312,34 @@ class Kubernetes(clouds.Cloud):
                          'not fit in the existing Kubernetes cluster '
                          'with context: '
                          f'{context}. Reason: {reason}')
+
+            autoscaler_type = skypilot_config.get_effective_region_config(
+                cloud='kubernetes',
+                region=context,
+                keys=('autoscaler',),
+                default_value=None)
+            if (autoscaler_type is not None and
+                    not kubernetes_utils.get_autoscaler(
+                        kubernetes_enums.KubernetesAutoscalerType(
+                            autoscaler_type)).can_query_backend):
+                # Unsupported autoscaler type. Rely on the autoscaler to
+                # provision the right instance type without running checks.
+                # Worst case, if autoscaling fails, the pod will be stuck in
+                # pending state until provision_timeout, after which failover
+                # will be triggered.
+                #
+                # Removing this if statement produces the same behavior,
+                # because can_create_new_instance_of_type() always returns True
+                # for unsupported autoscaler types.
+                # This check is here as a performance optimization to avoid
+                # further code executions that is known to return this result.
+                regions_to_return.append(r)
+                continue
+
             if autoscaler_type is None:
                 continue
-            autoscaler = kubernetes_utils.get_autoscaler(autoscaler_type)
+            autoscaler = kubernetes_utils.get_autoscaler(
+                kubernetes_enums.KubernetesAutoscalerType(autoscaler_type))
             logger.debug(f'{context} has autoscaler of type: {autoscaler_type}')
             if autoscaler.can_create_new_instance_of_type(
                     context, instance_type):
@@ -312,10 +379,12 @@ class Kubernetes(clouds.Cloud):
             cls,
             cpus: Optional[str] = None,
             memory: Optional[str] = None,
-            disk_tier: Optional['resources_utils.DiskTier'] = None) -> str:
+            disk_tier: Optional['resources_utils.DiskTier'] = None,
+            region: Optional[str] = None,
+            zone: Optional[str] = None) -> str:
         # TODO(romilb): In the future, we may want to move the instance type
         #  selection + availability checking to a kubernetes_catalog module.
-        del disk_tier  # Unused.
+        del disk_tier, region, zone  # Unused.
         # We strip '+' from resource requests since Kubernetes can provision
         # exactly the requested resources.
         instance_cpus = float(
@@ -379,7 +448,11 @@ class Kubernetes(clouds.Cloud):
         return 0
 
     @staticmethod
-    def _calculate_provision_timeout(num_nodes: int) -> int:
+    def _calculate_provision_timeout(
+        num_nodes: int,
+        volume_mounts: Optional[List['volume_lib.VolumeMount']],
+        enable_flex_start: bool,
+    ) -> int:
         """Calculate provision timeout based on number of nodes.
 
         The timeout scales linearly with the number of nodes to account for
@@ -387,6 +460,8 @@ class Kubernetes(clouds.Cloud):
 
         Args:
             num_nodes: Number of nodes being provisioned
+            volume_mounts: Volume mounts for the pod
+            enable_flex_start: Whether flex start is enabled
 
         Returns:
             Timeout in seconds
@@ -394,19 +469,38 @@ class Kubernetes(clouds.Cloud):
         base_timeout = 10  # Base timeout for single node
         per_node_timeout = 0.2  # Additional seconds per node
         max_timeout = 60  # Cap at 1 minute
+        if enable_flex_start:
+            # Flex start takes longer to provision.
+            base_timeout = 1200
+            per_node_timeout = 10
+            max_timeout = 2400
+        elif volume_mounts is not None:
+            for volume_mount in volume_mounts:
+                if (volume_mount.volume_config.type ==
+                        volume_lib.VolumeType.PVC.value):
+                    if (volume_mount.volume_config.config.get(
+                            'access_mode', '') ==
+                            volume_lib.VolumeAccessMode.READ_WRITE_MANY.value):
+                        # GKE may take several minutes to provision a PV
+                        # supporting READ_WRITE_MANY with filestore.
+                        base_timeout = 180
+                        max_timeout = 240
+                        break
 
         return int(
             min(base_timeout + (per_node_timeout * (num_nodes - 1)),
                 max_timeout))
 
     def make_deploy_resources_variables(
-            self,
-            resources: 'resources_lib.Resources',
-            cluster_name: 'resources_utils.ClusterName',
-            region: Optional['clouds.Region'],
-            zones: Optional[List['clouds.Zone']],
-            num_nodes: int,
-            dryrun: bool = False) -> Dict[str, Optional[str]]:
+        self,
+        resources: 'resources_lib.Resources',
+        cluster_name: 'resources_utils.ClusterName',
+        region: Optional['clouds.Region'],
+        zones: Optional[List['clouds.Zone']],
+        num_nodes: int,
+        dryrun: bool = False,
+        volume_mounts: Optional[List['volume_lib.VolumeMount']] = None,
+    ) -> Dict[str, Optional[str]]:
         del cluster_name, zones, dryrun  # Unused.
         if region is None:
             context = kubernetes_utils.get_current_kube_config_context_name()
@@ -414,8 +508,9 @@ class Kubernetes(clouds.Cloud):
             context = region.name
         assert context is not None, 'No context found in kubeconfig'
 
-        r = resources
-        acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
+        resources = resources.assert_launchable()
+        acc_dict = self.get_accelerators_from_instance_type(
+            resources.instance_type)
         custom_resources = resources_utils.make_ray_custom_resources_str(
             acc_dict)
 
@@ -426,8 +521,12 @@ class Kubernetes(clouds.Cloud):
         cpus = k.cpus
         mem = k.memory
         # Optionally populate accelerator information.
-        acc_count = k.accelerator_count if k.accelerator_count else 0
-        acc_type = k.accelerator_type if k.accelerator_type else None
+        acc_type = k.accelerator_type
+        acc_count = k.accelerator_count
+        if acc_type is not None and is_tpu_on_gke(acc_type):
+            acc_type, acc_count = normalize_tpu_accelerator_name(acc_type)
+        else:
+            acc_count = acc_count or 0
 
         def _get_image_id(resources: 'resources_lib.Resources') -> str:
             image_id_dict = resources.image_id
@@ -444,15 +543,18 @@ class Kubernetes(clouds.Cloud):
                 # Select image based on whether we are using GPUs or not.
                 image_id = self.IMAGE_GPU if acc_count > 0 else self.IMAGE_CPU
                 # Get the container image ID from the service catalog.
-                image_id = service_catalog.get_image_id_from_tag(
-                    image_id, clouds='kubernetes')
+                image_id = catalog.get_image_id_from_tag(image_id,
+                                                         clouds='kubernetes')
             return image_id
 
         image_id = _get_image_id(resources)
-        # TODO(romilb): Create a lightweight image for SSH jump host
-        ssh_jump_image = service_catalog.get_image_id_from_tag(
-            self.IMAGE_CPU, clouds='kubernetes')
 
+        # Set environment variables for the pod. Note that SkyPilot env vars
+        # are set separately when the task is run. These env vars are
+        # independent of the SkyPilot task to be run.
+        k8s_env_vars = {kubernetes.IN_CLUSTER_CONTEXT_NAME_ENV_VAR: context}
+
+        # Setup GPU/TPU labels and resource keys.
         k8s_acc_label_key = None
         k8s_acc_label_values = None
         k8s_topology_label_key = None
@@ -472,17 +574,32 @@ class Kubernetes(clouds.Cloud):
                 tpu_requested = True
                 k8s_resource_key = kubernetes_utils.TPU_RESOURCE_KEY
             else:
-                k8s_resource_key = kubernetes_utils.get_gpu_resource_key()
+                k8s_resource_key = kubernetes_utils.get_gpu_resource_key(
+                    context)
         else:
+            # If no GPUs are requested, we set NVIDIA_VISIBLE_DEVICES=none to
+            # maintain GPU isolation. This is to override the default behavior
+            # of Nvidia device plugin which would expose all GPUs to the pod
+            # when no GPUs are requested.
+            # Note that NVIDIA_VISIBLE_DEVICES is different from
+            # CUDA_VISIBLE_DEVICES - the latter is used to control which GPUs
+            # are visible to the application and is set inside the pod, while
+            # the former is used to control which GPUs are visible to the pod
+            # through the nvidia runtime.
+            # See: https://github.com/NVIDIA/k8s-device-plugin/issues/61
+            k8s_env_vars['NVIDIA_VISIBLE_DEVICES'] = 'none'
             avoid_label_keys = kubernetes_utils.get_accelerator_label_keys(
                 context)
             if len(avoid_label_keys) == 0:
                 avoid_label_keys = None
-        port_mode = network_utils.get_port_mode(None)
+        port_mode = network_utils.get_port_mode(None, context)
 
-        remote_identity = skypilot_config.get_nested(
-            ('kubernetes', 'remote_identity'),
-            schemas.get_default_remote_identity('kubernetes'))
+        remote_identity = skypilot_config.get_effective_region_config(
+            # TODO(kyuds): Support SSH node pools as well.
+            cloud='kubernetes',
+            region=context,
+            keys=('remote_identity',),
+            default_value=schemas.get_default_remote_identity('kubernetes'))
 
         if isinstance(remote_identity, dict):
             # If remote_identity is a dict, use the service account for the
@@ -496,20 +613,17 @@ class Kubernetes(clouds.Cloud):
             # If remote_identity is not a dict, use
             k8s_service_account_name = remote_identity
 
-        if (k8s_service_account_name ==
-                schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value):
-            # SA name doesn't matter since automounting credentials is disabled
-            k8s_service_account_name = 'default'
-            k8s_automount_sa_token = 'false'
-        elif (k8s_service_account_name ==
-              schemas.RemoteIdentityOptions.SERVICE_ACCOUNT.value):
-            # Use the default service account
+        lc = schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value
+        sa = schemas.RemoteIdentityOptions.SERVICE_ACCOUNT.value
+
+        if k8s_service_account_name == lc or k8s_service_account_name == sa:
+            # Use the default service account if remote identity is not set.
+            # For LOCAL_CREDENTIALS, this is for in-cluster authentication
+            # which needs a serviceaccount (specifically for SSH node pools
+            # which uses in-cluster authentication internally, and we would
+            # like to support exec-auth when the user is also using SSH infra)
             k8s_service_account_name = (
                 kubernetes_utils.DEFAULT_SERVICE_ACCOUNT_NAME)
-            k8s_automount_sa_token = 'true'
-        else:
-            # User specified a custom service account
-            k8s_automount_sa_token = 'true'
 
         fuse_device_required = bool(resources.requires_fuse)
 
@@ -518,26 +632,22 @@ class Kubernetes(clouds.Cloud):
         if resources.use_spot:
             spot_label_key, spot_label_value = kubernetes_utils.get_spot_label()
 
-        # Timeout for resource provisioning. This timeout determines how long to
-        # wait for pod to be in pending status before giving up.
-        # Larger timeout may be required for autoscaling clusters, since
-        # autoscaler may take some time to provision new nodes.
-        # Note that this timeout includes time taken by the Kubernetes scheduler
-        # itself, which can be upto 2-3 seconds, and up to 10-15 seconds when
-        # scheduling 100s of pods.
-        # We use a linear scaling formula to determine the timeout based on the
-        # number of nodes.
-
-        timeout = self._calculate_provision_timeout(num_nodes)
-        timeout = skypilot_config.get_nested(
-            ('kubernetes', 'provision_timeout'),
-            timeout,
-            override_configs=resources.cluster_config_overrides)
-
-        # Set environment variables for the pod. Note that SkyPilot env vars
-        # are set separately when the task is run. These env vars are
-        # independent of the SkyPilot task to be run.
-        k8s_env_vars = {kubernetes.IN_CLUSTER_CONTEXT_NAME_ENV_VAR: context}
+        network_type, machine_type = self._detect_network_type(
+            context, resources.network_tier)
+
+        # Check if this cluster supports high performance networking and
+        # configure appropriate settings for different cluster types
+        if (resources.network_tier is not None and
+                resources.network_tier == resources_utils.NetworkTier.BEST):
+            # Only proceed if CUSTOM_NETWORK_TIER is supported by this cluster
+            unsupported_features = self._unsupported_features_for_resources(
+                resources)
+            if clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER \
+                    not in unsupported_features:
+                # Add high-performance networking environment variables for
+                # clusters with high performance networking
+                network_env_vars = network_type.get_network_env_vars()
+                k8s_env_vars.update(network_env_vars)
 
         # We specify object-store-memory to be 500MB to avoid taking up too
         # much memory on the head node. 'num-cpus' should be set to limit
@@ -551,9 +661,63 @@ class Kubernetes(clouds.Cloud):
         }
 
         # Get the storage class name for high availability controller's PVC
-        k8s_ha_storage_class_name = skypilot_config.get_nested(
-            ('kubernetes', 'high_availability', 'storage_class_name'),
-            None,
+        k8s_ha_storage_class_name = (
+            skypilot_config.get_effective_region_config(
+                cloud='kubernetes',
+                region=context,
+                keys=('high_availability', 'storage_class_name'),
+                default_value=None))
+
+        k8s_kueue_local_queue_name = (
+            skypilot_config.get_effective_region_config(
+                # TODO(kyuds): Support SSH node pools as well.
+                cloud='kubernetes',
+                region=context,
+                keys=('kueue', 'local_queue_name'),
+                default_value=None,
+                override_configs=resources.cluster_config_overrides))
+
+        # Check DWS configuration for GKE.
+        (enable_flex_start, enable_flex_start_queued_provisioning,
+         max_run_duration_seconds) = gcp_utils.get_dws_config(
+             context, k8s_kueue_local_queue_name,
+             resources.cluster_config_overrides)
+        if enable_flex_start_queued_provisioning or enable_flex_start:
+            # DWS is only supported in GKE, check the autoscaler type.
+            autoscaler_type = skypilot_config.get_effective_region_config(
+                # TODO(kyuds): Support SSH node pools as well.
+                cloud='kubernetes',
+                region=context,
+                keys=('autoscaler',),
+                default_value=None)
+            if (autoscaler_type !=
+                    kubernetes_enums.KubernetesAutoscalerType.GKE.value):
+                raise ValueError(
+                    f'DWS is only supported in GKE, but the autoscaler type '
+                    f'for context {context} is {autoscaler_type}')
+
+        # Timeout for resource provisioning. This timeout determines how long to
+        # wait for pod to be in pending status before giving up.
+        # Larger timeout may be required for autoscaling clusters, since
+        # autoscaler may take some time to provision new nodes.
+        # Note that this timeout includes time taken by the Kubernetes scheduler
+        # itself, which can be upto 2-3 seconds, and up to 10-15 seconds when
+        # scheduling 100s of pods.
+        # We use a linear scaling formula to determine the timeout based on the
+        # number of nodes.
+
+        timeout = self._calculate_provision_timeout(
+            num_nodes, volume_mounts, enable_flex_start or
+            enable_flex_start_queued_provisioning)
+
+        # Use _REPR, instead of directly using 'kubernetes' as the config key,
+        # because it could be SSH node pool as well.
+        cloud_config_str = self._REPR.lower()
+        timeout = skypilot_config.get_effective_region_config(
+            cloud=cloud_config_str,
+            region=context,
+            keys=('provision_timeout',),
+            default_value=timeout,
             override_configs=resources.cluster_config_overrides)
 
         deploy_vars = {
@@ -564,15 +728,12 @@ class Kubernetes(clouds.Cloud):
             'accelerator_count': str(acc_count),
             'timeout': str(timeout),
             'k8s_port_mode': port_mode.value,
-            'k8s_networking_mode': network_utils.get_networking_mode().value,
-            'k8s_ssh_key_secret_name': self.SKY_SSH_KEY_SECRET_NAME,
             'k8s_acc_label_key': k8s_acc_label_key,
             'k8s_acc_label_values': k8s_acc_label_values,
-            'k8s_ssh_jump_name': self.SKY_SSH_JUMP_NAME,
-            'k8s_ssh_jump_image': ssh_jump_image,
             'k8s_service_account_name': k8s_service_account_name,
-            'k8s_automount_sa_token': k8s_automount_sa_token,
+            'k8s_automount_sa_token': 'true',
             'k8s_fuse_device_required': fuse_device_required,
+            'k8s_kueue_local_queue_name': k8s_kueue_local_queue_name,
             # Namespace to run the fusermount-server daemonset in
             'k8s_skypilot_system_namespace': _SKYPILOT_SYSTEM_NAMESPACE,
             'k8s_fusermount_shared_dir': _FUSERMOUNT_SHARED_DIR,
@@ -600,9 +761,17 @@ class Kubernetes(clouds.Cloud):
                 (constants.PERSISTENT_SETUP_SCRIPT_PATH),
             'k8s_high_availability_deployment_run_script_dir':
                 (constants.PERSISTENT_RUN_SCRIPT_DIR),
+            'k8s_high_availability_restarting_signal_file':
+                (constants.PERSISTENT_RUN_RESTARTING_SIGNAL_FILE),
+            'ha_recovery_log_path':
+                constants.HA_PERSISTENT_RECOVERY_LOG_PATH.format(''),
+            'sky_python_cmd': constants.SKY_PYTHON_CMD,
             'k8s_high_availability_storage_class_name':
                 (k8s_ha_storage_class_name),
             'avoid_label_keys': avoid_label_keys,
+            'k8s_enable_flex_start': enable_flex_start,
+            'k8s_max_run_duration_seconds': max_run_duration_seconds,
+            'k8s_network_type': network_type.value,
         }
 
         # Add kubecontext if it is set. It may be None if SkyPilot is running
@@ -613,13 +782,43 @@ class Kubernetes(clouds.Cloud):
         namespace = kubernetes_utils.get_kube_config_context_namespace(context)
         deploy_vars['k8s_namespace'] = namespace
 
+        # Add backward compatibility template variables for GPUDirect variants
+        deploy_vars['k8s_enable_gpudirect_tcpx'] = (
+            network_type == KubernetesHighPerformanceNetworkType.GCP_TCPX)
+        deploy_vars['k8s_enable_gpudirect_tcpxo'] = (
+            network_type == KubernetesHighPerformanceNetworkType.GCP_TCPXO)
+        rdma_enabled = (network_type ==
+                        KubernetesHighPerformanceNetworkType.GCP_GPUDIRECT_RDMA)
+        deploy_vars['k8s_enable_gpudirect_rdma'] = rdma_enabled
+        if rdma_enabled and machine_type.startswith('a4'):
+            deploy_vars['k8s_enable_gpudirect_rdma_a4'] = True
+        else:
+            deploy_vars['k8s_enable_gpudirect_rdma_a4'] = False
+
+        deploy_vars['k8s_ipc_lock_capability'] = (
+            network_type.requires_ipc_lock_capability())
+
         return deploy_vars
 
+    @staticmethod
+    def _warn_on_disk_size(resources: 'resources_lib.Resources'):
+        if resources.disk_size != resources_lib.DEFAULT_DISK_SIZE_GB:
+            logger.info(f'{colorama.Style.DIM}Disk size {resources.disk_size} '
+                        'is not supported by Kubernetes. '
+                        'To add additional disk, use volumes.'
+                        f'{colorama.Style.RESET_ALL}')
+        if resources.disk_tier is not None:
+            logger.info(f'{colorama.Style.DIM}Disk tier {resources.disk_tier} '
+                        'is not supported by Kubernetes. '
+                        'To add additional disk, use volumes.'
+                        f'{colorama.Style.RESET_ALL}')
+
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
     ) -> 'resources_utils.FeasibleResources':
         # TODO(zhwu): This needs to be updated to return the correct region
         # (context) that has enough resources.
+        self._warn_on_disk_size(resources)
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
@@ -628,7 +827,8 @@ class Kubernetes(clouds.Cloud):
                 accelerators=resources.accelerators,
                 use_spot=resources.use_spot,
                 region=resources.region,
-                zone=resources.zone)
+                zone=resources.zone,
+                resources=resources)
             if not regions:
                 return resources_utils.FeasibleResources([], [], None)
             resources = resources.copy(accelerators=None)
@@ -639,7 +839,7 @@ class Kubernetes(clouds.Cloud):
             resource_list = []
             for instance_type in instance_list:
                 r = resources.copy(
-                    cloud=Kubernetes(),
+                    cloud=self.__class__(),
                     instance_type=instance_type,
                     accelerators=None,
                 )
@@ -652,7 +852,9 @@ class Kubernetes(clouds.Cloud):
         default_instance_type = Kubernetes.get_default_instance_type(
             cpus=resources.cpus,
             memory=resources.memory,
-            disk_tier=resources.disk_tier)
+            disk_tier=resources.disk_tier,
+            region=resources.region,
+            zone=resources.zone)
 
         if accelerators is None:
             # For CPU only clusters, need no special handling
@@ -661,12 +863,18 @@ class Kubernetes(clouds.Cloud):
             assert len(accelerators) == 1, resources
             # GPUs requested - build instance type.
             acc_type, acc_count = list(accelerators.items())[0]
+            # If acc_type contains spaces, return empty list since Kubernetes
+            # does not support spaces in label values
+            if ' ' in acc_type:
+                return resources_utils.FeasibleResources([], [], None)
 
             # Parse into KubernetesInstanceType
             k8s_instance_type = (kubernetes_utils.KubernetesInstanceType.
                                  from_instance_type(default_instance_type))
 
             gpu_task_cpus = k8s_instance_type.cpus
+            if resources.cpus is None:
+                gpu_task_cpus = self._DEFAULT_NUM_VCPUS_WITH_GPU * acc_count
             # Special handling to bump up memory multiplier for GPU instances
             gpu_task_memory = (float(resources.memory.strip('+')) if
                                resources.memory is not None else gpu_task_cpus *
@@ -680,7 +888,8 @@ class Kubernetes(clouds.Cloud):
             accelerators=None,
             use_spot=resources.use_spot,
             region=resources.region,
-            zone=resources.zone)
+            zone=resources.zone,
+            resources=resources)
         if not available_regions:
             return resources_utils.FeasibleResources([], [], None)
         # No fuzzy lists for Kubernetes
@@ -691,10 +900,47 @@ class Kubernetes(clouds.Cloud):
                                                  [], None)
 
     @classmethod
-    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_single_context(cls, context: str) -> Tuple[bool, str]:
+        """Check if the user has access credentials to a single SSH context."""
+
+        def _red_color(str_to_format: str) -> str:
+            return (f'{colorama.Fore.LIGHTRED_EX}'
+                    f'{str_to_format}'
+                    f'{colorama.Style.RESET_ALL}')
+
+        def _dim_color(str_to_format: str) -> str:
+            return (f'{colorama.Style.DIM}'
+                    f'{str_to_format}'
+                    f'{colorama.Style.RESET_ALL}')
+
+        def _bright_green_color(str_to_format: str) -> str:
+            return (f'{colorama.Fore.GREEN}'
+                    f'{str_to_format}'
+                    f'{colorama.Style.RESET_ALL}')
+
+        try:
+            check_result = kubernetes_utils.check_credentials(
+                context, run_optional_checks=True)
+            if check_result[0]:
+                if check_result[1] is not None:
+                    return True, (_bright_green_color('enabled.') +
+                                  _dim_color(f' Note: {check_result[1]}'))
+                else:
+                    return True, _bright_green_color('enabled.')
+            else:
+                assert check_result[1] is not None
+                return False, (_red_color('disabled.') +
+                               _dim_color(f' Reason: {check_result[1]}'))
+        except Exception as e:  # pylint: disable=broad-except
+            return False, _red_color(str(e))
+
+    @classmethod
+    def _check_compute_credentials(
+            cls) -> Tuple[bool, Optional[Union[str, Dict[str, str]]]]:
         """Checks if the user has access credentials to
         Kubernetes."""
         # Check for port forward dependencies
+        logger.debug(f'Checking compute credentials for {cls.canonical_name()}')
         reasons = kubernetes_utils.check_port_forward_mode_dependencies(False)
         if reasons is not None:
             formatted = '\n'.join(
@@ -718,26 +964,15 @@ class Kubernetes(clouds.Cloud):
             return (False, 'No available context found in kubeconfig. '
                     'Check if you have a valid kubeconfig file' +
                     check_skypilot_config_msg)
-        reasons = []
-        hints = []
+
+        ctx2text = {}
         success = False
         for context in existing_allowed_contexts:
-            try:
-                check_result = kubernetes_utils.check_credentials(
-                    context, run_optional_checks=True)
-                if check_result[0]:
-                    success = True
-                    if check_result[1] is not None:
-                        hints.append(f'Context {context}: {check_result[1]}')
-                else:
-                    reasons.append(f'Context {context}: {check_result[1]}')
-            except Exception as e:  # pylint: disable=broad-except
-                return (False, f'Credential check failed for {context}: '
-                        f'{common_utils.format_exception(e)}')
-        if success:
-            return (True, cls._format_credential_check_results(hints, reasons))
-        return (False, 'Failed to find available context with working '
-                'credentials. Details:\n' + '\n'.join(reasons))
+            suc, text = cls._check_single_context(context)
+            success = success or suc
+            ctx2text[context] = text
+
+        return success, ctx2text
 
     @classmethod
     def _format_credential_check_results(cls, hints: List[str],
@@ -768,10 +1003,28 @@ class Kubernetes(clouds.Cloud):
         return ''.join(message_parts)
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
-        if os.path.exists(os.path.expanduser(CREDENTIAL_PATH)):
+        credential_paths = kubernetes_utils.get_kubeconfig_paths()
+        if credential_paths:
+            # For single kubeconfig path, keep the original path.
+            kubeconfig_file = credential_paths[0]
+            if len(credential_paths) > 1:
+                # For multiple kubeconfig paths, merge them into a single file.
+                # TODO(aylei): GC merged kubeconfig files.
+                kubeconfig_file = tempfile.NamedTemporaryFile(
+                    prefix='merged-kubeconfig-', suffix='.yaml',
+                    delete=False).name
+                subprocess.run(
+                    'kubectl config view --flatten '
+                    f'> {kubeconfig_file}',
+                    shell=True,
+                    check=True)
+            if os.path.exists(kubeconfig_file):
+                # convert auth plugin paths (e.g.: gke-gcloud-auth-plugin)
+                kubeconfig_file = kubernetes_utils.format_kubeconfig_exec_auth_with_cache(kubeconfig_file)  # pylint: disable=line-too-long
+
             # Upload kubeconfig to the default path to avoid having to set
             # KUBECONFIG in the environment.
-            return {DEFAULT_KUBECONFIG_PATH: CREDENTIAL_PATH}
+            return {kubernetes.DEFAULT_KUBECONFIG_PATH: kubeconfig_file}
         else:
             return {}
 
@@ -787,7 +1040,7 @@ class Kubernetes(clouds.Cloud):
 
         all_contexts = kubernetes_utils.get_all_kube_context_names()
 
-        if region not in all_contexts:
+        if region and region not in all_contexts:
             raise ValueError(
                 f'Context {region} not found in kubeconfig. Kubernetes only '
                 'supports context names as regions. Available '
@@ -810,11 +1063,11 @@ class Kubernetes(clouds.Cloud):
 
     @classmethod
     def get_user_identities(cls) -> Optional[List[List[str]]]:
-        k8s = kubernetes.kubernetes
         identities = []
+        k8s = kubernetes.kubernetes
         try:
             all_contexts, current_context = (
-                k8s.config.list_kube_config_contexts())
+                kubernetes.list_kube_config_contexts())
         except k8s.config.config_exception.ConfigException:
             return None
         # Add current context at the head of the list
@@ -825,6 +1078,31 @@ class Kubernetes(clouds.Cloud):
             identities.append(identity)
         return identities
 
+    @classmethod
+    def is_volume_name_valid(cls,
+                             volume_name: str) -> Tuple[bool, Optional[str]]:
+        """Validates that the volume name is valid for this cloud.
+
+        Follows Kubernetes DNS-1123 subdomain rules:
+        - must be <= 253 characters
+        - must match: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' # pylint: disable=line-too-long
+        """
+        # Max length per DNS-1123 subdomain
+        if len(volume_name) > cls._MAX_VOLUME_NAME_LEN_LIMIT:
+            return (False, f'Volume name exceeds the maximum length of '
+                    f'{cls._MAX_VOLUME_NAME_LEN_LIMIT} characters '
+                    '(DNS-1123 subdomain).')
+
+        # DNS-1123 label: [a-z0-9]([-a-z0-9]*[a-z0-9])?
+        label = r'[a-z0-9]([-a-z0-9]*[a-z0-9])?'
+        # DNS-1123 subdomain: label(\.-separated label)*
+        subdomain_pattern = rf'^{label}(\.{label})*$'
+        if re.fullmatch(subdomain_pattern, volume_name) is None:
+            return (False, 'Volume name must be a valid DNS-1123 subdomain: '
+                    'lowercase alphanumeric, "-", and "."; start/end with '
+                    'alphanumeric.')
+        return True, None
+
     @classmethod
     def is_label_valid(cls, label_key: str,
                        label_value: str) -> Tuple[bool, Optional[str]]:
@@ -854,3 +1132,133 @@ class Kubernetes(clouds.Cloud):
         if not key_valid or not value_valid:
             return False, error_msg
         return True, None
+
+    @classmethod
+    def expand_infras(cls) -> List[str]:
+        return [
+            f'{cls.canonical_name()}/{c}'
+            for c in cls.existing_allowed_contexts(silent=True)
+        ]
+
+    @classmethod
+    def _detect_network_type(
+        cls,
+        context: str,
+        network_tier: Optional['resources_utils.NetworkTier'] = None
+    ) -> Tuple[KubernetesHighPerformanceNetworkType, str]:
+        """Detect the type of Kubernetes network based on node labels.
+
+        Args:
+            context: The Kubernetes context to check.
+            network_tier: The network tier requested. If None or not BEST,
+                         returns NONE (no high-performance networking).
+
+        Returns:
+            A tuple of the detected network type and the instance type.
+        """
+        # If network_tier is None or not BEST, return NONE
+        if (network_tier is None or
+                network_tier != resources_utils.NetworkTier.BEST):
+            return KubernetesHighPerformanceNetworkType.NONE, ''
+
+        try:
+            nodes = kubernetes_utils.get_kubernetes_nodes(context=context)
+            for node in nodes:
+                if node.metadata.labels:
+                    # Check for Nebius clusters
+                    for label_key, _ in node.metadata.labels.items():
+                        if label_key.startswith('nebius.com/'):
+                            return (KubernetesHighPerformanceNetworkType.NEBIUS,
+                                    '')
+                        if label_key.startswith('ib.coreweave.cloud/'):
+                            return (
+                                KubernetesHighPerformanceNetworkType.COREWEAVE,
+                                '')
+
+                    # Check for GKE clusters with specific GPUDirect variants
+                    machine_family = node.metadata.labels.get(
+                        'cloud.google.com/machine-family', '')
+                    instance_type = node.metadata.labels.get(
+                        'node.kubernetes.io/instance-type', '')
+                    gke_accelerator = node.metadata.labels.get(
+                        'cloud.google.com/gke-accelerator', '')
+
+                    # Check if this is a GKE cluster with A3/A4 machine family
+                    if machine_family in ['a3', 'a4']:
+                        # Check instance type to determine specific GPUDirect
+                        # variant
+                        if 'a3-highgpu-8g' in instance_type:
+                            return (
+                                KubernetesHighPerformanceNetworkType.GCP_TCPX,
+                                'a3-highgpu-8g')
+                        elif 'a3-edgegpu-8g' in instance_type:
+                            return (
+                                KubernetesHighPerformanceNetworkType.GCP_TCPX,
+                                'a3-edgegpu-8g')
+                        elif 'a3-megagpu-8g' in instance_type:
+                            return (
+                                KubernetesHighPerformanceNetworkType.GCP_TCPXO,
+                                'a3-megagpu-8g')
+                        elif 'a4-highgpu-8g' in instance_type:
+                            return (KubernetesHighPerformanceNetworkType.
+                                    GCP_GPUDIRECT_RDMA, 'a4-highgpu-8g')
+                        elif 'a3-ultragpu-8g' in instance_type:
+                            return (KubernetesHighPerformanceNetworkType.
+                                    GCP_GPUDIRECT_RDMA, 'a3-ultragpu-8g')
+                        # Generic A3/A4 detection as fallback
+                        elif machine_family == 'a4':
+                            return (KubernetesHighPerformanceNetworkType.
+                                    GCP_GPUDIRECT_RDMA, 'a4')
+
+                    # Fallback: Check for GPU Direct TCPX capable instance
+                    # types with high-perf GPUs
+                    is_gpu_direct_tcpx_instance = (
+                        instance_type
+                        in gcp_constants.GPU_DIRECT_TCPX_INSTANCE_TYPES)
+                    has_high_perf_gpu = ('nvidia-h100' in gke_accelerator or
+                                         'nvidia-h200' in gke_accelerator or
+                                         'nvidia-b200' in gke_accelerator)
+
+                    if is_gpu_direct_tcpx_instance and has_high_perf_gpu:
+                        # Default to TCPX if we can't determine the specific
+                        # variant
+                        return (KubernetesHighPerformanceNetworkType.GCP_TCPX,
+                                instance_type)
+
+        except exceptions.KubeAPIUnreachableError:
+            # If we can't reach the cluster, assume no high perf networking
+            pass
+
+        # If we cannot determine the network type based on nodes
+        # Check if the cluster has any node pools with autoscaling enabled
+        # with machine types that support high perf networking for GKE.
+        autoscaler_type = skypilot_config.get_effective_region_config(
+            cloud='kubernetes',
+            region=context,
+            keys=('autoscaler',),
+            default_value=None)
+        if (autoscaler_type !=
+                kubernetes_enums.KubernetesAutoscalerType.GKE.value):
+            return KubernetesHighPerformanceNetworkType.NONE, ''
+        autoscaler = kubernetes_utils.get_autoscaler(
+            kubernetes_enums.KubernetesAutoscalerType(autoscaler_type))
+        logger.debug(f'{context} has autoscaler of type: {autoscaler_type}')
+        machine_types = autoscaler.get_available_machine_types(context)
+        # Check if any machine type supports high perf networking for GKE.
+        if 'a3-highgpu-8g' in machine_types:
+            return (KubernetesHighPerformanceNetworkType.GCP_TCPX,
+                    'a3-highgpu-8g')
+        elif 'a3-edgegpu-8g' in machine_types:
+            return (KubernetesHighPerformanceNetworkType.GCP_TCPX,
+                    'a3-edgegpu-8g')
+        elif 'a3-megagpu-8g' in machine_types:
+            return (KubernetesHighPerformanceNetworkType.GCP_TCPXO,
+                    'a3-megagpu-8g')
+        elif 'a4-highgpu-8g' in machine_types:
+            return (KubernetesHighPerformanceNetworkType.GCP_GPUDIRECT_RDMA,
+                    'a4-highgpu-8g')
+        elif 'a3-ultragpu-8g' in machine_types:
+            return (KubernetesHighPerformanceNetworkType.GCP_GPUDIRECT_RDMA,
+                    'a3-ultragpu-8g')
+
+        return KubernetesHighPerformanceNetworkType.NONE, ''