skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/client/service_account_auth.py

@@ -0,0 +1,47 @@
+"""Service account token authentication for SkyPilot client."""
+
+import os
+from typing import Optional
+
+from sky import skypilot_config
+from sky.skylet import constants
+
+
+def _get_service_account_token() -> Optional[str]:
+    """Get service account token from environment variable or config file.
+
+    Priority order:
+    1. SKYPILOT_SERVICE_ACCOUNT_TOKEN environment variable
+    2. ~/.sky/config.yaml service_account_token field
+
+    Returns:
+        The service account token if found, None otherwise.
+    """
+    # Check environment variable first
+    token = os.environ.get(constants.SERVICE_ACCOUNT_TOKEN_ENV_VAR)
+    if token:
+        if not token.startswith('sky_'):
+            raise ValueError('Invalid service account token format. '
+                             'Token must start with "sky_"')
+        return token
+
+    # Check config file
+    token = skypilot_config.get_nested(('api_server', 'service_account_token'),
+                                       default_value=None)
+    if token and not token.startswith('sky_'):
+        raise ValueError('Invalid service account token format in config. '
+                         'Token must start with "sky_"')
+    return token
+
+
+def get_service_account_headers() -> dict:
+    """Get headers for service account authentication.
+
+    Returns:
+        Dictionary with Authorization header if token is available,
+        empty dict otherwise.
+    """
+    token = _get_service_account_token()
+    if token:
+        return {'Authorization': f'Bearer {token}'}
+    return {}

sky/cloud_stores.py

@@ -18,6 +18,7 @@ from sky import sky_logging
 from sky.adaptors import aws
 from sky.adaptors import azure
 from sky.adaptors import cloudflare
+from sky.adaptors import coreweave
 from sky.adaptors import ibm
 from sky.adaptors import nebius
 from sky.adaptors import oci
@@ -197,9 +198,15 @@ class AzureBlobCloudStorage(CloudStorage):
     _GET_AZCOPY = [
         'azcopy --version > /dev/null 2>&1 || '
         '(mkdir -p /usr/local/bin; '
-        'curl -L https://aka.ms/downloadazcopy-v10-linux -o azcopy.tar.gz; '
-        'sudo tar -xvzf azcopy.tar.gz --strip-components=1 -C /usr/local/bin --exclude=*.txt; '  # pylint: disable=line-too-long
-        'sudo chmod +x /usr/local/bin/azcopy; '
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  ARCH="arm64"; '
+        'elif [ "$ARCH" = "x86_64" ]; then '
+        '  ARCH="amd64"; '
+        'fi && '
+        'curl -fL https://github.com/Azure/azure-storage-azcopy/releases/download/v10.30.1/azcopy_linux_${ARCH}_10.30.1.tar.gz -o azcopy.tar.gz && '  # pylint: disable=line-too-long
+        'sudo tar -xvzf azcopy.tar.gz --strip-components=1 -C /usr/local/bin --exclude=*.txt && '  # pylint: disable=line-too-long
+        'sudo chmod +x /usr/local/bin/azcopy && '
         'rm azcopy.tar.gz)'
     ]
 
@@ -602,6 +609,77 @@ class NebiusCloudStorage(CloudStorage):
         return ' && '.join(all_commands)
 
 
+class CoreWeaveCloudStorage(CloudStorage):
+    """CoreWeave Cloud Storage."""
+
+    # List of commands to install AWS CLI
+    _GET_AWSCLI = [
+        'aws --version >/dev/null 2>&1 || '
+        f'{constants.SKY_UV_PIP_CMD} install awscli',
+    ]
+
+    def is_directory(self, url: str) -> bool:
+        """Checks if the coreweave object is a directory.
+
+        In cloud object stores, a "directory" refers to a regular object whose
+        name is a prefix of other objects.
+
+        Args:
+            url: coreweave object URL.
+        """
+        cw = coreweave.resource('s3')
+        bucket_name, path = data_utils.split_coreweave_path(url)
+        bucket = cw.Bucket(bucket_name)
+
+        num_objects = 0
+        for obj in bucket.objects.filter(Prefix=path):
+            num_objects += 1
+            if obj.key == path:
+                return False
+            # If there are more than 1 object in filter, then it is a directory
+            if num_objects == 3:
+                return True
+
+        # A directory with few or no items
+        return True
+
+    def make_sync_dir_command(self, source: str, destination: str) -> str:
+        """Downloads using AWS CLI."""
+        # AWS Sync by default uses 10 threads to upload files to the bucket.
+        # To increase parallelism, modify max_concurrent_requests in your
+        # aws config file (Default path: ~/.coreweave/cw.config).
+        assert 'cw://' in source, 'cw:// is not in source'
+        source = source.replace('cw://', 's3://')
+        download_via_awscli = (
+            'AWS_SHARED_CREDENTIALS_FILE='
+            f'{coreweave.COREWEAVE_CREDENTIALS_PATH} '
+            f'AWS_CONFIG_FILE={coreweave.COREWEAVE_CONFIG_PATH} '
+            f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
+            'sync --no-follow-symlinks '
+            f'{source} {destination} '
+            f'--profile={coreweave.COREWEAVE_PROFILE_NAME}')
+
+        all_commands = list(self._GET_AWSCLI)
+        all_commands.append(download_via_awscli)
+        return ' && '.join(all_commands)
+
+    def make_sync_file_command(self, source: str, destination: str) -> str:
+        """Downloads a file using AWS CLI."""
+        assert 'cw://' in source, 'cw:// is not in source'
+        source = source.replace('cw://', 's3://')
+        download_via_awscli = (
+            'AWS_SHARED_CREDENTIALS_FILE='
+            f'{coreweave.COREWEAVE_CREDENTIALS_PATH} '
+            f'AWS_CONFIG_FILE={coreweave.COREWEAVE_CONFIG_PATH} '
+            f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
+            f'cp {source} {destination} '
+            f'--profile={coreweave.COREWEAVE_PROFILE_NAME}')
+
+        all_commands = list(self._GET_AWSCLI)
+        all_commands.append(download_via_awscli)
+        return ' && '.join(all_commands)
+
+
 def get_storage_from_path(url: str) -> CloudStorage:
     """Returns a CloudStorage by identifying the scheme:// in a URL."""
     result = urllib.parse.urlsplit(url)
@@ -619,6 +697,7 @@ _REGISTRY = {
     'cos': IBMCosCloudStorage(),
     'oci': OciCloudStorage(),
     'nebius': NebiusCloudStorage(),
+    'cw': CoreWeaveCloudStorage(),
     # TODO: This is a hack, as Azure URL starts with https://, we should
     # refactor the registry to be able to take regex, so that Azure blob can
     # be identified with `https://(.*?)\.blob\.core\.windows\.net`

sky/clouds/__init__.py

@@ -2,7 +2,9 @@
 
 from sky.clouds.cloud import Cloud
 from sky.clouds.cloud import cloud_in_iterable
+from sky.clouds.cloud import CloudCapability
 from sky.clouds.cloud import CloudImplementationFeatures
+from sky.clouds.cloud import DummyCloud
 from sky.clouds.cloud import OpenPortsVersion
 from sky.clouds.cloud import ProvisionerVersion
 from sky.clouds.cloud import Region
@@ -17,14 +19,19 @@ from sky.clouds.cudo import Cudo
 from sky.clouds.do import DO
 from sky.clouds.fluidstack import Fluidstack
 from sky.clouds.gcp import GCP
+from sky.clouds.hyperbolic import Hyperbolic
 from sky.clouds.ibm import IBM
 from sky.clouds.kubernetes import Kubernetes
 from sky.clouds.lambda_cloud import Lambda
 from sky.clouds.nebius import Nebius
 from sky.clouds.oci import OCI
 from sky.clouds.paperspace import Paperspace
+from sky.clouds.primeintellect import PrimeIntellect
 from sky.clouds.runpod import RunPod
 from sky.clouds.scp import SCP
+from sky.clouds.seeweb import Seeweb
+from sky.clouds.shadeform import Shadeform
+from sky.clouds.ssh import SSH
 from sky.clouds.vast import Vast
 from sky.clouds.vsphere import Vsphere
 
@@ -34,16 +41,20 @@ __all__ = [
     'Azure',
     'Cloud',
     'Cudo',
+    'DummyCloud',
     'GCP',
     'Lambda',
     'DO',
     'Paperspace',
+    'PrimeIntellect',
     'SCP',
     'RunPod',
+    'Shadeform',
     'Vast',
     'OCI',
     'Vsphere',
     'Kubernetes',
+    'SSH',
     'CloudImplementationFeatures',
     'Region',
     'Zone',
@@ -51,6 +62,8 @@ __all__ = [
     'StatusVersion',
     'Fluidstack',
     'Nebius',
+    'Hyperbolic',
+    'Seeweb',
     # Utility functions
     'cloud_in_iterable',
 ]