skypilot-nightly 1.0.0.dev20250320__py3-none-any.whl → 1.0.0.dev20250322__py3-none-any.whl

sky/data/data_transfer.py

@@ -14,6 +14,7 @@ Currently implemented:
 TODO:
 - All combinations of Azure Transfer
 - All combinations of R2 Transfer
+- All combinations of Nebius Transfer
 - GCS -> S3
 """
 import json
@@ -135,6 +136,35 @@ def s3_to_r2(s3_bucket_name: str, r2_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def s3_to_nebius(s3_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Amazon S3 to Nebius Object Storage.
+
+    it will block until the transfer is complete.
+
+    Args:
+      s3_bucket_name: str; Name of the Amazon S3 Bucket
+      nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
+def r2_to_nebius(r2_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Cloudflare R2 Bucket to
+     Nebius Object Storage.
+
+    it will block until the transfer is complete.
+
+    Args:
+      r2_bucket_name: str; Name of the Amazon S3 Bucket
+      nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def gcs_to_s3(gs_bucket_name: str, s3_bucket_name: str) -> None:
     """Creates a one-time transfer from Google Cloud Storage to Amazon S3.
 
@@ -160,6 +190,19 @@ def gcs_to_r2(gs_bucket_name: str, r2_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def gcs_to_nebius(gs_bucket_name: str, nebius_bucket_name: str) -> None:
+    """Creates a one-time transfer from Google Cloud Storage to
+     Nebius Object Storage.
+
+     Args:
+      gs_bucket_name: str; Name of the Google Cloud Storage Bucket
+      nebius_bucket_name: str; Name of the Nebius Bucket
+    """
+    raise NotImplementedError('Moving data directly from clouds to Nebius is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
     """Creates a one-time transfer from Cloudflare R2 to Google Cloud Storage.
 
@@ -175,6 +218,22 @@ def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def nebius_to_gcs(nebius_bucket_name: str, gs_bucket_name: str) -> None:
+    """Creates a one-time transfer from Nebius Object Storage
+     to Google Cloud Storage.
+
+    Can be viewed from: https://console.cloud.google.com/transfer/cloud
+    it will block until the transfer is complete.
+
+    Args:
+      nebius_bucket_name: str; Name of the Nebius Object Storage
+      gs_bucket_name: str; Name of the Google Cloud Storage Bucket
+    """
+    raise NotImplementedError('Moving data directly from Nebius to clouds is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
     """Creates a one-time transfer from Amazon S3 to Google Cloud Storage.
 
@@ -190,6 +249,22 @@ def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
                               'a local source for the storage object.')
 
 
+def nebius_to_s3(nebius_bucket_name: str, s3_bucket_name: str) -> None:
+    """Creates a one-time transfer from Nebius Object Storage
+    to Amazon S3 Bucket.
+
+    Can be viewed from: https://console.cloud.google.com/transfer/cloud
+    it will block until the transfer is complete.
+
+    Args:
+      nebius_bucket_name: str; Name of the Nebius Object Storage\
+      s3_bucket_name: str; Name of the Amazon S3 Bucket
+    """
+    raise NotImplementedError('Moving data directly from R2 to clouds is '
+                              'currently not supported. Please specify '
+                              'a local source for the storage object.')
+
+
 def _add_bucket_iam_member(bucket_name: str, role: str, member: str) -> None:
     storage_client = gcp.storage_client()
     bucket = storage_client.bucket(bucket_name)

sky/data/data_utils.py

@@ -20,6 +20,7 @@ from sky.adaptors import azure
 from sky.adaptors import cloudflare
 from sky.adaptors import gcp
 from sky.adaptors import ibm
+from sky.adaptors import nebius
 from sky.skylet import log_lib
 from sky.utils import common_utils
 from sky.utils import ux_utils
@@ -94,6 +95,18 @@ def split_r2_path(r2_path: str) -> Tuple[str, str]:
     return bucket, key
 
 
+def split_nebius_path(nebius_path: str) -> Tuple[str, str]:
+    """Splits Nebius Path into Bucket name and Relative Path to Bucket
+
+    Args:
+      nebius_path: str; Nebius Path, e.g. nebius://imagenet/train/
+    """
+    path_parts = nebius_path.replace('nebius://', '').split('/')
+    bucket = path_parts.pop(0)
+    key = '/'.join(path_parts)
+    return bucket, key
+
+
 def split_cos_path(s3_path: str) -> Tuple[str, str, str]:
     """returns extracted region, bucket name and bucket path to data
         from the specified cos bucket's url.
@@ -307,6 +320,16 @@ def create_r2_client(region: str = 'auto') -> Client:
     return cloudflare.client('s3', region)
 
 
+def create_nebius_client(region: Optional[str]) -> Client:
+    """Helper method that connects to Boto3 client for Nebius Object Storage
+
+    Args:
+      region: str; Region for Nebius Object Storage
+    """
+    region = region if region is not None else nebius.DEFAULT_REGION
+    return nebius.client('s3', region)
+
+
 def verify_r2_bucket(name: str) -> bool:
     """Helper method that checks if the R2 bucket exists
 
@@ -318,6 +341,17 @@ def verify_r2_bucket(name: str) -> bool:
     return bucket in r2.buckets.all()
 
 
+def verify_nebius_bucket(name: str) -> bool:
+    """Helper method that checks if the Nebius bucket exists
+
+    Args:
+      name: str; Name of Nebius Object Storage (without nebius:// prefix)
+    """
+    nebius_s = nebius.resource('s3')
+    bucket = nebius_s.Bucket(name)
+    return bucket in nebius_s.buckets.all()
+
+
 def verify_ibm_cos_bucket(name: str) -> bool:
     """Helper method that checks if the cos bucket exists
 

sky/data/mounting_utils.py

@@ -54,6 +54,24 @@ def get_s3_mount_cmd(bucket_name: str,
     return mount_cmd
 
 
+def get_nebius_mount_cmd(nebius_profile_name: str,
+                         endpoint_url: str,
+                         bucket_name: str,
+                         mount_path: str,
+                         _bucket_sub_path: Optional[str] = None) -> str:
+    """Returns a command to install Nebius mount utility goofys."""
+    if _bucket_sub_path is None:
+        _bucket_sub_path = ''
+    else:
+        _bucket_sub_path = f':{_bucket_sub_path}'
+    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} goofys -o allow_other '
+                 f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                 f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                 f'--endpoint {endpoint_url} '
+                 f'{bucket_name}{_bucket_sub_path} {mount_path}')
+    return mount_cmd
+
+
 def get_gcs_mount_install_cmd() -> str:
     """Returns a command to install GCS mount utility gcsfuse."""
     install_cmd = ('ARCH=$(uname -m) && '