skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/clouds/kubernetes.py

@@ -1,4 +1,5 @@
 """Kubernetes."""
+import concurrent.futures
 import os
 import re
 import subprocess
@@ -25,6 +26,7 @@ from sky.provision.kubernetes.utils import normalize_tpu_accelerator_name
 from sky.skylet import constants
 from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import env_options
 from sky.utils import kubernetes_enums
 from sky.utils import registry
 from sky.utils import resources_utils
@@ -47,9 +49,6 @@ _FUSERMOUNT_SHARED_DIR = '/var/run/fusermount'
 class Kubernetes(clouds.Cloud):
     """Kubernetes."""
 
-    SKY_SSH_KEY_SECRET_NAME = 'sky-ssh-keys'
-    SKY_SSH_JUMP_NAME = 'sky-ssh-jump-pod'
-
     # Limit the length of the cluster name to avoid exceeding the limit of 63
     # characters for Kubernetes resources. We limit to 42 characters (63-21) to
     # allow additional characters for creating ingress services to expose ports.
@@ -62,6 +61,7 @@ class Kubernetes(clouds.Cloud):
     _SUPPORTS_SERVICE_ACCOUNT_ON_REMOTE = True
 
     _DEFAULT_NUM_VCPUS = 2
+    _DEFAULT_NUM_VCPUS_WITH_GPU = 4
     _DEFAULT_MEMORY_CPU_RATIO = 1
     _DEFAULT_MEMORY_CPU_RATIO_WITH_GPU = 4  # Allocate more memory for GPU tasks
     _REPR = 'Kubernetes'
@@ -97,44 +97,52 @@ class Kubernetes(clouds.Cloud):
     # Set of contexts that has logged as temporarily unreachable
     logged_unreachable_contexts: Set[str] = set()
 
-    @property
-    def ssh_key_secret_field_name(self):
-        # Use a fresh user hash to avoid conflicts in the secret object naming.
-        # This can happen when the controller is reusing the same user hash
-        # through USER_ID_ENV_VAR but has a different SSH key.
-        fresh_user_hash = common_utils.generate_user_hash()
-        return f'ssh-publickey-{fresh_user_hash}'
-
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         # TODO(aylei): features need to be regional (per context) to make
         # multi-kubernetes selection/failover work.
         unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
-        context = resources.region
+        context = region if region is not None else resources.region
         if context is None:
-            context = kubernetes_utils.get_current_kube_config_context_name()
+            contexts = cls.existing_allowed_contexts()
+        else:
+            contexts = [context]
         unsupported_features[clouds.CloudImplementationFeatures.STOP] = (
             'Stopping clusters is not supported on Kubernetes.')
         unsupported_features[clouds.CloudImplementationFeatures.AUTOSTOP] = (
             'Auto-stop is not supported on Kubernetes.')
-        # Allow spot instances if supported by the cluster
-        try:
-            spot_label_key, _ = kubernetes_utils.get_spot_label(context)
-            if spot_label_key is not None:
-                unsupported_features.pop(
-                    clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
-            # Allow custom network tier if supported by the cluster
-            # (e.g., Nebius clusters with high performance networking)
-            network_type, _ = cls._detect_network_type(context,
-                                                       resources.network_tier)
-            if network_type.supports_high_performance_networking():
-                unsupported_features.pop(
-                    clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER,
-                    None)
-        except exceptions.KubeAPIUnreachableError as e:
-            cls._log_unreachable_context(context, str(e))
+        for context in contexts:
+            # Allow spot instances if supported by the cluster
+            try:
+                # Run spot label check and network type detection concurrently
+                # as they are independent operations
+                with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=2) as executor:
+                    spot_future = executor.submit(
+                        kubernetes_utils.get_spot_label, context)
+                    network_future = executor.submit(cls._detect_network_type,
+                                                     context,
+                                                     resources.network_tier)
+
+                    spot_label_key, _ = spot_future.result()
+                    if spot_label_key is not None:
+                        unsupported_features.pop(
+                            clouds.CloudImplementationFeatures.SPOT_INSTANCE,
+                            None)
+
+                    # Allow custom network tier if supported by the cluster
+                    # (e.g., Nebius clusters with high performance networking)
+                    network_type, _ = network_future.result()
+                    if network_type.supports_high_performance_networking():
+                        unsupported_features.pop(
+                            clouds.CloudImplementationFeatures.
+                            CUSTOM_NETWORK_TIER, None)
+            except exceptions.KubeAPIUnreachableError as e:
+                cls._log_unreachable_context(context, str(e))
         return unsupported_features
 
     @classmethod
@@ -187,6 +195,12 @@ class Kubernetes(clouds.Cloud):
             ctx for ctx in all_contexts if not ctx.startswith('ssh-')
         ]
 
+        allow_all_contexts = allowed_contexts == 'all' or (
+            allowed_contexts is None and
+            env_options.Options.ALLOW_ALL_KUBERNETES_CONTEXTS.get())
+        if allow_all_contexts:
+            allowed_contexts = all_contexts
+
         if allowed_contexts is None:
             # Try kubeconfig if present
             current_context = (
@@ -244,10 +258,15 @@ class Kubernetes(clouds.Cloud):
                 'refresh Kubernetes availability if permanent.')
 
     @classmethod
-    def regions_with_offering(cls, instance_type: Optional[str],
-                              accelerators: Optional[Dict[str, int]],
-                              use_spot: bool, region: Optional[str],
-                              zone: Optional[str]) -> List[clouds.Region]:
+    def regions_with_offering(
+        cls,
+        instance_type: Optional[str],
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
+    ) -> List[clouds.Region]:
         del accelerators, zone, use_spot  # unused
         existing_contexts = cls.existing_allowed_contexts()
 
@@ -257,6 +276,19 @@ class Kubernetes(clouds.Cloud):
 
         if region is not None:
             regions = [r for r in regions if r.name == region]
+        if resources is not None:
+            filtered_regions = []
+            resources_required_features = resources.get_required_cloud_features(
+            )
+            for r in regions:
+                try:
+                    cls.check_features_are_supported(
+                        resources, resources_required_features, r.name)
+                    filtered_regions.append(r)
+                except exceptions.NotSupportedError as e:
+                    logger.info(f'Filter out context: {r.name}, reason: {e}')
+                    continue
+            regions = filtered_regions
 
         # Check if requested instance type will fit in the cluster.
         # TODO(zhwu,romilb): autoscaler type needs to be regional (per
@@ -516,9 +548,6 @@ class Kubernetes(clouds.Cloud):
             return image_id
 
         image_id = _get_image_id(resources)
-        # TODO(romilb): Create a lightweight image for SSH jump host
-        ssh_jump_image = catalog.get_image_id_from_tag(self.IMAGE_CPU,
-                                                       clouds='kubernetes')
 
         # Set environment variables for the pod. Note that SkyPilot env vars
         # are set separately when the task is run. These env vars are
@@ -566,6 +595,7 @@ class Kubernetes(clouds.Cloud):
         port_mode = network_utils.get_port_mode(None, context)
 
         remote_identity = skypilot_config.get_effective_region_config(
+            # TODO(kyuds): Support SSH node pools as well.
             cloud='kubernetes',
             region=context,
             keys=('remote_identity',),
@@ -640,6 +670,7 @@ class Kubernetes(clouds.Cloud):
 
         k8s_kueue_local_queue_name = (
             skypilot_config.get_effective_region_config(
+                # TODO(kyuds): Support SSH node pools as well.
                 cloud='kubernetes',
                 region=context,
                 keys=('kueue', 'local_queue_name'),
@@ -654,6 +685,7 @@ class Kubernetes(clouds.Cloud):
         if enable_flex_start_queued_provisioning or enable_flex_start:
             # DWS is only supported in GKE, check the autoscaler type.
             autoscaler_type = skypilot_config.get_effective_region_config(
+                # TODO(kyuds): Support SSH node pools as well.
                 cloud='kubernetes',
                 region=context,
                 keys=('autoscaler',),
@@ -677,8 +709,12 @@ class Kubernetes(clouds.Cloud):
         timeout = self._calculate_provision_timeout(
             num_nodes, volume_mounts, enable_flex_start or
             enable_flex_start_queued_provisioning)
+
+        # Use _REPR, instead of directly using 'kubernetes' as the config key,
+        # because it could be SSH node pool as well.
+        cloud_config_str = self._REPR.lower()
         timeout = skypilot_config.get_effective_region_config(
-            cloud='kubernetes',
+            cloud=cloud_config_str,
             region=context,
             keys=('provision_timeout',),
             default_value=timeout,
@@ -692,13 +728,8 @@ class Kubernetes(clouds.Cloud):
             'accelerator_count': str(acc_count),
             'timeout': str(timeout),
             'k8s_port_mode': port_mode.value,
-            'k8s_networking_mode': network_utils.get_networking_mode(
-                None, context=context).value,
-            'k8s_ssh_key_secret_name': self.SKY_SSH_KEY_SECRET_NAME,
             'k8s_acc_label_key': k8s_acc_label_key,
             'k8s_acc_label_values': k8s_acc_label_values,
-            'k8s_ssh_jump_name': self.SKY_SSH_JUMP_NAME,
-            'k8s_ssh_jump_image': ssh_jump_image,
             'k8s_service_account_name': k8s_service_account_name,
             'k8s_automount_sa_token': 'true',
             'k8s_fuse_device_required': fuse_device_required,
@@ -796,7 +827,8 @@ class Kubernetes(clouds.Cloud):
                 accelerators=resources.accelerators,
                 use_spot=resources.use_spot,
                 region=resources.region,
-                zone=resources.zone)
+                zone=resources.zone,
+                resources=resources)
             if not regions:
                 return resources_utils.FeasibleResources([], [], None)
             resources = resources.copy(accelerators=None)
@@ -841,6 +873,8 @@ class Kubernetes(clouds.Cloud):
                                  from_instance_type(default_instance_type))
 
             gpu_task_cpus = k8s_instance_type.cpus
+            if resources.cpus is None:
+                gpu_task_cpus = self._DEFAULT_NUM_VCPUS_WITH_GPU * acc_count
             # Special handling to bump up memory multiplier for GPU instances
             gpu_task_memory = (float(resources.memory.strip('+')) if
                                resources.memory is not None else gpu_task_cpus *
@@ -854,7 +888,8 @@ class Kubernetes(clouds.Cloud):
             accelerators=None,
             use_spot=resources.use_spot,
             region=resources.region,
-            zone=resources.zone)
+            zone=resources.zone,
+            resources=resources)
         if not available_regions:
             return resources_utils.FeasibleResources([], [], None)
         # No fuzzy lists for Kubernetes

sky/clouds/lambda_cloud.py

@@ -59,7 +59,9 @@ class Lambda(clouds.Cloud):
 
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         del resources  # unused
         return cls._CLOUD_UNSUPPORTED_FEATURES
@@ -69,10 +71,15 @@ class Lambda(clouds.Cloud):
         return cls._MAX_CLUSTER_NAME_LEN_LIMIT
 
     @classmethod
-    def regions_with_offering(cls, instance_type: str,
-                              accelerators: Optional[Dict[str, int]],
-                              use_spot: bool, region: Optional[str],
-                              zone: Optional[str]) -> List[clouds.Region]:
+    def regions_with_offering(
+        cls,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
+    ) -> List[clouds.Region]:
         assert zone is None, 'Lambda does not support zones.'
         del accelerators, zone  # unused
         if use_spot:

sky/clouds/nebius.py

@@ -78,7 +78,9 @@ class Nebius(clouds.Cloud):
 
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         unsupported = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
 
@@ -101,10 +103,15 @@ class Nebius(clouds.Cloud):
         return cls._MAX_CLUSTER_NAME_LEN_LIMIT
 
     @classmethod
-    def regions_with_offering(cls, instance_type: str,
-                              accelerators: Optional[Dict[str, int]],
-                              use_spot: bool, region: Optional[str],
-                              zone: Optional[str]) -> List[clouds.Region]:
+    def regions_with_offering(
+        cls,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
+    ) -> List[clouds.Region]:
         assert zone is None, 'Nebius does not support zones.'
         del accelerators, zone  # unused
         regions = catalog.get_region_zones_for_instance_type(
@@ -245,9 +252,12 @@ class Nebius(clouds.Cloud):
                 'filesystem_mount_tag': f'filesystem-skypilot-{i+1}'
             })
 
+        use_static_ip_address = skypilot_config.get_nested(
+            ('nebius', 'use_static_ip_address'), default_value=False)
         resources_vars: Dict[str, Any] = {
             'instance_type': resources.instance_type,
             'custom_resources': custom_resources,
+            'use_static_ip_address': use_static_ip_address,
             'region': region.name,
             'image_id': image_family,
             # Nebius does not support specific zones.
@@ -364,10 +374,10 @@ class Nebius(clouds.Cloud):
                       f'{_INDENT_PREFIX}  $ nebius --format json iam whoami|jq -r \'.user_profile.tenants[0].tenant_id\' > {nebius.tenant_id_path()} \n')  # pylint: disable=line-too-long
         if not nebius.is_token_or_cred_file_exist():
             return False, f'{token_cred_msg}'
-        sdk = nebius.sdk()
         tenant_id = nebius.get_tenant_id()
         if tenant_id is None:
             return False, f'{tenant_msg}'
+        sdk = nebius.sdk()
         try:
             service = nebius.iam().ProjectServiceClient(sdk)
             service.list(
@@ -445,9 +455,13 @@ class Nebius(clouds.Cloud):
         del workspace_config  # Unused
         sdk = nebius.sdk()
         profile_client = nebius.iam().ProfileServiceClient(sdk)
-        profile = nebius.sync_call(
-            profile_client.get(nebius.iam().GetProfileRequest(),
-                               timeout=nebius.READ_TIMEOUT))
+        try:
+            profile = nebius.sync_call(
+                profile_client.get(nebius.iam().GetProfileRequest(),
+                                   timeout=nebius.READ_TIMEOUT))
+        except Exception as e:
+            raise exceptions.CloudUserIdentityError(
+                f'Error getting Nebius profile: {e}')
         if profile.user_profile is not None:
             if profile.user_profile.attributes is None:
                 raise exceptions.CloudUserIdentityError(

sky/clouds/oci.py

@@ -28,6 +28,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 from sky import catalog
 from sky import clouds
 from sky import exceptions
+from sky.adaptors import common
 from sky.adaptors import oci as oci_adaptor
 from sky.clouds.utils import oci_utils
 from sky.provision.oci.query_utils import query_helper
@@ -68,7 +69,9 @@ class OCI(clouds.Cloud):
 
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         unsupported_features = {
             clouds.CloudImplementationFeatures.CLONE_DISK_FROM_CLUSTER:
@@ -95,10 +98,15 @@ class OCI(clouds.Cloud):
         return cls._MAX_CLUSTER_NAME_LEN_LIMIT
 
     @classmethod
-    def regions_with_offering(cls, instance_type: str,
-                              accelerators: Optional[Dict[str, int]],
-                              use_spot: bool, region: Optional[str],
-                              zone: Optional[str]) -> List[clouds.Region]:
+    def regions_with_offering(
+        cls,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
+    ) -> List[clouds.Region]:
         del accelerators  # unused
 
         regions = catalog.get_region_zones_for_instance_type(
@@ -454,13 +462,12 @@ class OCI(clouds.Cloud):
             f'{cls._INDENT_PREFIX}  region=us-sanjose-1\n'
             f'{cls._INDENT_PREFIX}  key_file=~/.oci/oci_api_key.pem')
 
-        try:
-            # pylint: disable=import-outside-toplevel,unused-import
-            import oci
-        except ImportError:
-            return False, ('`oci` is not installed. Install it with: '
-                           'pip install oci\n'
-                           f'{cls._INDENT_PREFIX}{short_credential_help_str}')
+        dependency_error_msg = (
+            '`oci` is not installed. Install it with: '
+            'pip install oci\n'
+            f'{cls._INDENT_PREFIX}{short_credential_help_str}')
+        if not common.can_import_modules(['oci']):
+            return False, dependency_error_msg
 
         conf_file = oci_adaptor.get_config_file()
 

sky/clouds/paperspace.py

@@ -60,7 +60,9 @@ class Paperspace(clouds.Cloud):
 
     @classmethod
     def _unsupported_features_for_resources(
-        cls, resources: 'resources_lib.Resources'
+        cls,
+        resources: 'resources_lib.Resources',
+        region: Optional[str] = None,
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         """The features not supported based on the resources provided.
 
@@ -86,6 +88,7 @@ class Paperspace(clouds.Cloud):
         use_spot: bool,
         region: Optional[str],
         zone: Optional[str],
+        resources: Optional['resources_lib.Resources'] = None,
     ) -> List[clouds.Region]:
         assert zone is None, 'Paperspace does not support zones.'
         del accelerators, zone  # unused