skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/cloud_stores.py

@@ -18,6 +18,7 @@ from sky import sky_logging
 from sky.adaptors import aws
 from sky.adaptors import azure
 from sky.adaptors import cloudflare
+from sky.adaptors import coreweave
 from sky.adaptors import ibm
 from sky.adaptors import nebius
 from sky.adaptors import oci
@@ -197,9 +198,15 @@ class AzureBlobCloudStorage(CloudStorage):
     _GET_AZCOPY = [
         'azcopy --version > /dev/null 2>&1 || '
         '(mkdir -p /usr/local/bin; '
-        'curl -L https://aka.ms/downloadazcopy-v10-linux -o azcopy.tar.gz; '
-        'sudo tar -xvzf azcopy.tar.gz --strip-components=1 -C /usr/local/bin --exclude=*.txt; '  # pylint: disable=line-too-long
-        'sudo chmod +x /usr/local/bin/azcopy; '
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  ARCH="arm64"; '
+        'elif [ "$ARCH" = "x86_64" ]; then '
+        '  ARCH="amd64"; '
+        'fi && '
+        'curl -fL https://github.com/Azure/azure-storage-azcopy/releases/download/v10.30.1/azcopy_linux_${ARCH}_10.30.1.tar.gz -o azcopy.tar.gz && '  # pylint: disable=line-too-long
+        'sudo tar -xvzf azcopy.tar.gz --strip-components=1 -C /usr/local/bin --exclude=*.txt && '  # pylint: disable=line-too-long
+        'sudo chmod +x /usr/local/bin/azcopy && '
         'rm azcopy.tar.gz)'
     ]
 
@@ -602,6 +609,77 @@ class NebiusCloudStorage(CloudStorage):
         return ' && '.join(all_commands)
 
 
+class CoreWeaveCloudStorage(CloudStorage):
+    """CoreWeave Cloud Storage."""
+
+    # List of commands to install AWS CLI
+    _GET_AWSCLI = [
+        'aws --version >/dev/null 2>&1 || '
+        f'{constants.SKY_UV_PIP_CMD} install awscli',
+    ]
+
+    def is_directory(self, url: str) -> bool:
+        """Checks if the coreweave object is a directory.
+
+        In cloud object stores, a "directory" refers to a regular object whose
+        name is a prefix of other objects.
+
+        Args:
+            url: coreweave object URL.
+        """
+        cw = coreweave.resource('s3')
+        bucket_name, path = data_utils.split_coreweave_path(url)
+        bucket = cw.Bucket(bucket_name)
+
+        num_objects = 0
+        for obj in bucket.objects.filter(Prefix=path):
+            num_objects += 1
+            if obj.key == path:
+                return False
+            # If there are more than 1 object in filter, then it is a directory
+            if num_objects == 3:
+                return True
+
+        # A directory with few or no items
+        return True
+
+    def make_sync_dir_command(self, source: str, destination: str) -> str:
+        """Downloads using AWS CLI."""
+        # AWS Sync by default uses 10 threads to upload files to the bucket.
+        # To increase parallelism, modify max_concurrent_requests in your
+        # aws config file (Default path: ~/.coreweave/cw.config).
+        assert 'cw://' in source, 'cw:// is not in source'
+        source = source.replace('cw://', 's3://')
+        download_via_awscli = (
+            'AWS_SHARED_CREDENTIALS_FILE='
+            f'{coreweave.COREWEAVE_CREDENTIALS_PATH} '
+            f'AWS_CONFIG_FILE={coreweave.COREWEAVE_CONFIG_PATH} '
+            f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
+            'sync --no-follow-symlinks '
+            f'{source} {destination} '
+            f'--profile={coreweave.COREWEAVE_PROFILE_NAME}')
+
+        all_commands = list(self._GET_AWSCLI)
+        all_commands.append(download_via_awscli)
+        return ' && '.join(all_commands)
+
+    def make_sync_file_command(self, source: str, destination: str) -> str:
+        """Downloads a file using AWS CLI."""
+        assert 'cw://' in source, 'cw:// is not in source'
+        source = source.replace('cw://', 's3://')
+        download_via_awscli = (
+            'AWS_SHARED_CREDENTIALS_FILE='
+            f'{coreweave.COREWEAVE_CREDENTIALS_PATH} '
+            f'AWS_CONFIG_FILE={coreweave.COREWEAVE_CONFIG_PATH} '
+            f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
+            f'cp {source} {destination} '
+            f'--profile={coreweave.COREWEAVE_PROFILE_NAME}')
+
+        all_commands = list(self._GET_AWSCLI)
+        all_commands.append(download_via_awscli)
+        return ' && '.join(all_commands)
+
+
 def get_storage_from_path(url: str) -> CloudStorage:
     """Returns a CloudStorage by identifying the scheme:// in a URL."""
     result = urllib.parse.urlsplit(url)
@@ -619,6 +697,7 @@ _REGISTRY = {
     'cos': IBMCosCloudStorage(),
     'oci': OciCloudStorage(),
     'nebius': NebiusCloudStorage(),
+    'cw': CoreWeaveCloudStorage(),
     # TODO: This is a hack, as Azure URL starts with https://, we should
     # refactor the registry to be able to take regex, so that Azure blob can
     # be identified with `https://(.*?)\.blob\.core\.windows\.net`

sky/clouds/__init__.py

@@ -26,8 +26,12 @@ from sky.clouds.lambda_cloud import Lambda
 from sky.clouds.nebius import Nebius
 from sky.clouds.oci import OCI
 from sky.clouds.paperspace import Paperspace
+from sky.clouds.primeintellect import PrimeIntellect
 from sky.clouds.runpod import RunPod
 from sky.clouds.scp import SCP
+from sky.clouds.seeweb import Seeweb
+from sky.clouds.shadeform import Shadeform
+from sky.clouds.slurm import Slurm
 from sky.clouds.ssh import SSH
 from sky.clouds.vast import Vast
 from sky.clouds.vsphere import Vsphere
@@ -43,8 +47,11 @@ __all__ = [
     'Lambda',
     'DO',
     'Paperspace',
+    'PrimeIntellect',
     'SCP',
+    'Slurm',
     'RunPod',
+    'Shadeform',
     'Vast',
     'OCI',
     'Vsphere',
@@ -58,6 +65,7 @@ __all__ = [
     'Fluidstack',
     'Nebius',
     'Hyperbolic',
+    'Seeweb',
     # Utility functions
     'cloud_in_iterable',
 ]