skypilot-nightly 1.0.0.dev20250320__py3-none-any.whl → 1.0.0.dev20250321__py3-none-any.whl

sky/clouds/cloud.py

@@ -13,6 +13,8 @@ import math
 import typing
 from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
 
+from typing_extensions import assert_never
+
 from sky import exceptions
 from sky import skypilot_config
 from sky.clouds import service_catalog
@@ -48,6 +50,16 @@ class CloudImplementationFeatures(enum.Enum):
     AUTO_TERMINATE = 'auto_terminate'  # Pod/VM can stop or down itself
 
 
+class CloudCapability(enum.Enum):
+    # Compute capability.
+    COMPUTE = 'compute'
+    # Storage capability.
+    STORAGE = 'storage'
+
+
+ALL_CAPABILITIES = [CloudCapability.COMPUTE, CloudCapability.STORAGE]
+
+
 class Region(collections.namedtuple('Region', ['name'])):
     """A region."""
     name: str
@@ -435,25 +447,36 @@ class Cloud:
         return {reservation: 0 for reservation in specific_reservations}
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def check_credentials(
+            cls,
+            cloud_capability: CloudCapability) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to this cloud.
 
         Returns a boolean of whether the user can access this cloud, and a
         string describing the reason if the user cannot access.
+
+        Raises NotSupportedError if the capability is
+        not supported by this cloud.
         """
-        raise NotImplementedError
+        if cloud_capability == CloudCapability.COMPUTE:
+            return cls._check_compute_credentials()
+        elif cloud_capability == CloudCapability.STORAGE:
+            return cls._check_storage_credentials()
+        assert_never(cloud_capability)
 
     @classmethod
-    def check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Checks if the user has access credentials to this cloud's storage.
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        this cloud's compute service."""
+        raise exceptions.NotSupportedError(
+            f'{cls._REPR} does not support {CloudCapability.COMPUTE.value}.')
 
-        Returns a boolean of whether the user can access this cloud's storage,
-        and a string describing the reason if the user cannot access.
-        """
-        # A given cloud does not support storage
-        # unless it overrides this method.
+    @classmethod
+    def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        this cloud's storage service."""
         raise exceptions.NotSupportedError(
-            f'{cls._REPR} does not support storage.')
+            f'{cls._REPR} does not support {CloudCapability.STORAGE.value}.')
 
     # TODO(zhwu): Make the return type immutable.
     @classmethod

sky/clouds/cudo.py

@@ -267,8 +267,9 @@ class Cudo(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        Cudo's compute service."""
         try:
             # pylint: disable=import-outside-toplevel,unused-import
             from cudo_compute import cudo_api

sky/clouds/do.py

@@ -259,8 +259,9 @@ class DO(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Verify that the user has valid credentials for DO."""
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Verify that the user has valid credentials for
+        DO's compute service."""
 
         try:
             do.exceptions()

sky/clouds/fluidstack.py

@@ -255,8 +255,9 @@ class Fluidstack(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        FluidStack's compute service."""
         try:
             assert os.path.exists(
                 os.path.expanduser(fluidstack_utils.FLUIDSTACK_API_KEY_PATH))

sky/clouds/gcp.py

@@ -719,9 +719,9 @@ class GCP(clouds.Cloud):
         return DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Checks if the user has compute access credentials to this cloud."""
-        return cls._check_credentials(  # Check APIs.
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to this cloud's compute service."""
+        return cls._check_credentials(
             [
                 ('compute', 'Compute Engine'),
                 ('cloudresourcemanager', 'Cloud Resource Manager'),
@@ -731,12 +731,11 @@ class GCP(clouds.Cloud):
             gcp_utils.get_minimal_compute_permissions())
 
     @classmethod
-    def check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Checks if the user has compute access credentials to this cloud."""
-        return cls._check_credentials(  # Check APIs.
-            [
-                ('storage', 'Cloud Storage'),
-            ], gcp_utils.get_minimal_storage_permissions())
+    def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to this cloud's storage service."""
+        return cls._check_credentials(
+            [('storage', 'Cloud Storage')],
+            gcp_utils.get_minimal_storage_permissions())
 
     @classmethod
     def _check_credentials(

sky/clouds/ibm.py

@@ -395,7 +395,21 @@ class IBM(clouds.Cloud):
         return image_size
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        IBM's compute service."""
+        return cls._check_credentials()
+
+    @classmethod
+    def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        IBM's storage service."""
+        # TODO(seungjin): Implement separate check for
+        # if the user has access to IBM COS.
+        return cls._check_credentials()
+
+    @classmethod
+    def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to this cloud."""
 
         required_fields = ['iam_api_key', 'resource_group_id']
@@ -433,11 +447,6 @@ class IBM(clouds.Cloud):
         except Exception as e:
             return (False, f'{str(e)}' + help_str)
 
-    @classmethod
-    def check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
-        # TODO(seungjin): Check if the user has access to IBM COS.
-        return cls.check_credentials()
-
     def get_credential_file_mounts(self) -> Dict[str, str]:
         """Returns a {remote:local} credential path mapping
          written to the cluster's file_mounts segment

sky/clouds/kubernetes.py

@@ -655,7 +655,9 @@ class Kubernetes(clouds.Cloud):
                                                  [], None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        Kubernetes."""
         # Test using python API
         try:
             existing_allowed_contexts = cls.existing_allowed_contexts()

sky/clouds/lambda_cloud.py

@@ -241,7 +241,9 @@ class Lambda(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        Lambda's compute service."""
         try:
             lambda_utils.LambdaCloudClient().list_instances()
         except (AssertionError, KeyError, lambda_utils.LambdaCloudError):

sky/clouds/nebius.py

@@ -250,8 +250,9 @@ class Nebius(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """ Verify that the user has valid credentials for Nebius. """
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        Nebius's compute service."""
         logging.debug('Nebius cloud check credentials')
         token_cred_msg = ('    Credentials can be set up by running: \n'\
                     f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n'\
@@ -279,10 +280,13 @@ class Nebius(clouds.Cloud):
         return True, None
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
-        return {
+        credential_file_mounts = {
             f'~/.nebius/{filename}': f'~/.nebius/{filename}'
             for filename in _CREDENTIAL_FILES
         }
+        credential_file_mounts['~/.aws/credentials'] = '~/.aws/credentials'
+
+        return credential_file_mounts
 
     @classmethod
     def get_current_user_identity(cls) -> Optional[List[str]]:

sky/clouds/oci.py

@@ -396,7 +396,21 @@ class OCI(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        OCI's compute service."""
+        return cls._check_credentials()
+
+    @classmethod
+    def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        OCI's storage service."""
+        # TODO(seungjin): Implement separate check for
+        # if the user has access to OCI Object Storage.
+        return cls._check_credentials()
+
+    @classmethod
+    def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to this cloud."""
 
         short_credential_help_str = (
@@ -456,11 +470,6 @@ class OCI(clouds.Cloud):
                 f'{cls._INDENT_PREFIX}Error details: '
                 f'{common_utils.format_exception(e, use_bracket=True)}')
 
-    @classmethod
-    def check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
-        # TODO(seungjin): Check if the user has access to OCI Object Storage.
-        return cls.check_credentials()
-
     @classmethod
     def check_disk_tier(
             cls, instance_type: Optional[str],

sky/clouds/paperspace.py

@@ -249,8 +249,9 @@ class Paperspace(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Verify that the user has valid credentials for Paperspace."""
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        Paperspace's compute service."""
         try:
             # attempt to make a CURL request for listing instances
             utils.PaperspaceCloudClient().list_instances()

sky/clouds/runpod.py

@@ -248,7 +248,13 @@ class RunPod(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        RunPod's compute service."""
+        return cls._check_credentials()
+
+    @classmethod
+    def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """ Verify that the user has valid credentials for RunPod. """
         try:
             import runpod  # pylint: disable=import-outside-toplevel

sky/clouds/scp.py

@@ -312,7 +312,9 @@ class SCP(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        SCP's compute service."""
         try:
             scp_utils.SCPClient().list_instances()
         except (AssertionError, KeyError, scp_utils.SCPClientError,

sky/clouds/service_catalog/kubernetes_catalog.py

@@ -12,6 +12,7 @@ from sky import clouds as sky_clouds
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
 from sky.adaptors import kubernetes
+from sky.clouds import cloud
 from sky.clouds.service_catalog import CloudFilter
 from sky.clouds.service_catalog import common
 from sky.provision.kubernetes import utils as kubernetes_utils
@@ -132,7 +133,8 @@ def _list_accelerators(
 
     # First check if Kubernetes is enabled. This ensures k8s python client is
     # installed. Do not put any k8s-specific logic before this check.
-    enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh()
+    enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+        cloud.CloudCapability.COMPUTE)
     if not sky_clouds.cloud_in_iterable(sky_clouds.Kubernetes(),
                                         enabled_clouds):
         return {}, {}, {}

sky/clouds/vast.py

@@ -234,8 +234,9 @@ class Vast(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """ Verify that the user has valid credentials for Vast. """
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has valid credentials for
+        Vast's compute service. """
         try:
             import vastai_sdk as _vast  # pylint: disable=import-outside-toplevel
             vast = _vast.VastAI()

sky/clouds/vsphere.py

@@ -254,8 +254,9 @@ class Vsphere(clouds.Cloud):
                                                  fuzzy_candidate_list, None)
 
     @classmethod
-    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
-        """Checks if the user has access credentials to this cloud."""
+    def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Checks if the user has access credentials to
+        vSphere's compute service."""
 
         try:
             # pylint: disable=import-outside-toplevel,unused-import

sky/core.py

@@ -19,6 +19,7 @@ from sky import optimizer
 from sky import sky_logging
 from sky import task as task_lib
 from sky.backends import backend_utils
+from sky.clouds import cloud as sky_cloud
 from sky.clouds import service_catalog
 from sky.jobs.server import core as managed_jobs_core
 from sky.provision.kubernetes import constants as kubernetes_constants
@@ -1001,7 +1002,8 @@ def storage_delete(name: str) -> None:
 # ===================
 @usage_lib.entrypoint
 def enabled_clouds() -> List[clouds.Cloud]:
-    return global_user_state.get_cached_enabled_clouds()
+    return global_user_state.get_cached_enabled_clouds(
+        sky_cloud.CloudCapability.COMPUTE)
 
 
 @usage_lib.entrypoint
@@ -1136,7 +1138,7 @@ def local_down() -> None:
                 ux_utils.spinner_message('Running sky check...')):
             sky_check.check(clouds=['kubernetes'],
                             quiet=True,
-                            capability=sky_check.CloudCapability.COMPUTE)
+                            capability=sky_cloud.CloudCapability.COMPUTE)
         logger.info(
             ux_utils.finishing_message('Local cluster removed.',
                                        log_path=log_path,

sky/data/data_transfer.py

@@ -14,6 +14,7 @@ Currently implemented:
 TODO:
 - All combinations of Azure Transfer
 - All combinations of R2 Transfer
+- All combinations of Nebius Transfer
 - GCS -> S3
 """
 import json
@@ -135,6 +136,35 @@ def s3_to_r2(s3_bucket_name: str, r2_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def s3_to_nebius(s3_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Amazon S3 to Nebius Object Storage.
+
+    it will block until the transfer is complete.
+
+    Args:
+      s3_bucket_name: str; Name of the Amazon S3 Bucket
+      nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
+def r2_to_nebius(r2_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Cloudflare R2 Bucket to
+     Nebius Object Storage.
+
+    it will block until the transfer is complete.
+
+    Args:
+      r2_bucket_name: str; Name of the Amazon S3 Bucket
+      nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def gcs_to_s3(gs_bucket_name: str, s3_bucket_name: str) -> None:
     """Creates a one-time transfer from Google Cloud Storage to Amazon S3.
 
@@ -160,6 +190,19 @@ def gcs_to_r2(gs_bucket_name: str, r2_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def gcs_to_nebius(gs_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Google Cloud Storage to
+     Nebius Object Storage.
+
+     Args:
+      gs_bucket_name: str; Name of the Google Cloud Storage Bucket
+      nebius_bucket_name: str; Name of the Nebius Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
     """Creates a one-time transfer from Cloudflare R2 to Google Cloud Storage.
 
@@ -175,6 +218,22 @@ def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def nebius_to_gcs(nebius_bucket_name: str, gs_bucket_name: str) -> None:
+    """Creates a one-time transfer from Nebius Object Storage
+     to Google Cloud Storage.
+
+    Can be viewed from: https://console.cloud.google.com/transfer/cloud
+    it will block until the transfer is complete.
+
+    Args:
+      nebius_bucket_name: str; Name of the Nebius Object Storage
+      gs_bucket_name: str; Name of the Google Cloud Storage Bucket
+    """
+    raise NotImplementedError('Moving data directly from Nebius to clouds is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
     """Creates a one-time transfer from Amazon S3 to Google Cloud Storage.
 
@@ -190,6 +249,22 @@ def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def nebius_to_s3(nebius_bucket_name: str, s3_bucket_name: str) -> None:
+    """Creates a one-time transfer from Nebius Object Storage
+    to Amazon S3 Bucket.
+
+    Can be viewed from: https://console.cloud.google.com/transfer/cloud
+    it will block until the transfer is complete.
+
+    Args:
+      nebius_bucket_name: str; Name of the Nebius Object Storage\
+      s3_bucket_name: str; Name of the Amazon S3 Bucket
+    """
+    raise NotImplementedError('Moving data directly from R2 to clouds is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def _add_bucket_iam_member(bucket_name: str, role: str, member: str) -> None:
     storage_client = gcp.storage_client()
     bucket = storage_client.bucket(bucket_name)

sky/data/data_utils.py

@@ -20,6 +20,7 @@ from sky.adaptors import azure
 from sky.adaptors import cloudflare
 from sky.adaptors import gcp
 from sky.adaptors import ibm
+from sky.adaptors import nebius
 from sky.skylet import log_lib
 from sky.utils import common_utils
 from sky.utils import ux_utils
@@ -94,6 +95,18 @@ def split_r2_path(r2_path: str) -> Tuple[str, str]:
     return bucket, key
 
 
+def split_nebius_path(nebius_path: str) -> Tuple[str, str]:
+    """Splits Nebius Path into Bucket name and Relative Path to Bucket
+
+    Args:
+      nebius_path: str; Nebius Path, e.g. nebius://imagenet/train/
+    """
+    path_parts = nebius_path.replace('nebius://', '').split('/')
+    bucket = path_parts.pop(0)
+    key = '/'.join(path_parts)
+    return bucket, key
+
+
 def split_cos_path(s3_path: str) -> Tuple[str, str, str]:
     """returns extracted region, bucket name and bucket path to data
         from the specified cos bucket's url.
@@ -307,6 +320,16 @@ def create_r2_client(region: str = 'auto') -> Client:
     return cloudflare.client('s3', region)
 
 
+def create_nebius_client(region: Optional[str]) -> Client:
+    """Helper method that connects to Boto3 client for Nebius Object Storage
+
+    Args:
+      region: str; Region for Nebius Object Storage
+    """
+    region = region if region is not None else nebius.DEFAULT_REGION
+    return nebius.client('s3', region)
+
+
 def verify_r2_bucket(name: str) -> bool:
     """Helper method that checks if the R2 bucket exists
 
@@ -318,6 +341,17 @@ def verify_r2_bucket(name: str) -> bool:
     return bucket in r2.buckets.all()
 
 
+def verify_nebius_bucket(name: str) -> bool:
+    """Helper method that checks if the Nebius bucket exists
+
+    Args:
+      name: str; Name of Nebius Object Storage (without nebius:// prefix)
+    """
+    nebius_s = nebius.resource('s3')
+    bucket = nebius_s.Bucket(name)
+    return bucket in nebius_s.buckets.all()
+
+
 def verify_ibm_cos_bucket(name: str) -> bool:
     """Helper method that checks if the cos bucket exists
 

sky/data/mounting_utils.py

@@ -54,6 +54,24 @@ def get_s3_mount_cmd(bucket_name: str,
     return mount_cmd
 
 
+def get_nebius_mount_cmd(nebius_profile_name: str,
+                         endpoint_url: str,
+                         bucket_name: str,
+                         mount_path: str,
+                         _bucket_sub_path: Optional[str] = None) -> str:
+    """Returns a command to install Nebius mount utility goofys."""
+    if _bucket_sub_path is None:
+        _bucket_sub_path = ''
+    else:
+        _bucket_sub_path = f':{_bucket_sub_path}'
+    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} goofys -o allow_other '
+                 f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                 f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                 f'--endpoint {endpoint_url} '
+                 f'{bucket_name}{_bucket_sub_path} {mount_path}')
+    return mount_cmd
+
+
 def get_gcs_mount_install_cmd() -> str:
     """Returns a command to install GCS mount utility gcsfuse."""
     install_cmd = ('ARCH=$(uname -m) && '