skypilot-nightly 1.0.0.dev20241024__tar.gz → 1.0.0.dev20241025__tar.gz

{skypilot_nightly-1.0.0.dev20241024/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20241025}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241024
+Version: 1.0.0.dev20241025
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'cbf5c0022ad920edb4f41cfad65a2cf4909d5930'
+_SKYPILOT_COMMIT_SHA = '057bc4b44755ac1e9dadc680e022c369e8ddff52'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241024'
+__version__ = '1.0.0.dev20241025'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/check.py

@@ -1,4 +1,5 @@
 """Credential checks: check cloud credentials and enable clouds."""
+import os
 import traceback
 from types import ModuleType
 from typing import Dict, Iterable, List, Optional, Tuple, Union
@@ -194,19 +195,25 @@ def get_cached_enabled_clouds_or_refresh(
 def get_cloud_credential_file_mounts(
         excluded_clouds: Optional[Iterable[sky_clouds.Cloud]]
 ) -> Dict[str, str]:
-    """Returns the files necessary to access all enabled clouds.
+    """Returns the files necessary to access all clouds.
 
     Returns a dictionary that will be added to a task's file mounts
     and a list of patterns that will be excluded (used as rsync_exclude).
     """
-    enabled_clouds = get_cached_enabled_clouds_or_refresh()
+    # Uploading credentials for all clouds instead of only sky check
+    # enabled clouds because users may have partial credentials for some
+    # clouds to access their specific resources (e.g. cloud storage) but
+    # not have the complete credentials to pass sky check.
+    clouds = sky_clouds.CLOUD_REGISTRY.values()
     file_mounts = {}
-    for cloud in enabled_clouds:
+    for cloud in clouds:
         if (excluded_clouds is not None and
                 sky_clouds.cloud_in_iterable(cloud, excluded_clouds)):
             continue
         cloud_file_mounts = cloud.get_credential_file_mounts()
-        file_mounts.update(cloud_file_mounts)
+        for remote_path, local_path in cloud_file_mounts.items():
+            if os.path.exists(os.path.expanduser(local_path)):
+                file_mounts[remote_path] = local_path
     # Currently, get_cached_enabled_clouds_or_refresh() does not support r2 as
     # only clouds with computing instances are marked as enabled by skypilot.
     # This will be removed when cloudflare/r2 is added as a 'cloud'.

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/cli.py

@@ -3085,7 +3085,7 @@ def show_gpus(
             qty_header = 'QTY_FILTER'
             free_header = 'FILTERED_FREE_GPUS'
         else:
-            qty_header = 'QTY_PER_NODE'
+            qty_header = 'REQUESTABLE_QTY_PER_NODE'
             free_header = 'TOTAL_FREE_GPUS'
         realtime_gpu_table = log_utils.create_table(
             ['GPU', qty_header, 'TOTAL_GPUS', free_header])

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/clouds/azure.py

@@ -15,6 +15,7 @@ from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import azure
 from sky.clouds import service_catalog
+from sky.clouds.utils import azure_utils
 from sky.utils import common_utils
 from sky.utils import resources_utils
 from sky.utils import ux_utils
@@ -36,6 +37,15 @@ _MAX_IDENTITY_FETCH_RETRY = 10
 
 _DEFAULT_AZURE_UBUNTU_HPC_IMAGE_GB = 30
 _DEFAULT_AZURE_UBUNTU_2004_IMAGE_GB = 150
+_DEFAULT_SKYPILOT_IMAGE_GB = 30
+
+_DEFAULT_CPU_IMAGE_ID = 'skypilot:gpu-ubuntu-2204'
+_DEFAULT_GPU_IMAGE_ID = 'skypilot:gpu-ubuntu-2204'
+_DEFAULT_V1_IMAGE_ID = 'skypilot:v1-ubuntu-2004'
+_DEFAULT_GPU_K80_IMAGE_ID = 'skypilot:k80-ubuntu-2004'
+_FALLBACK_IMAGE_ID = 'skypilot:gpu-ubuntu-2204'
+
+_COMMUNITY_IMAGE_PREFIX = '/CommunityGalleries'
 
 
 def _run_output(cmd):
@@ -132,29 +142,56 @@ class Azure(clouds.Cloud):
         cost += 0.0
         return cost
 
+    @classmethod
+    def get_default_instance_type(
+            cls,
+            cpus: Optional[str] = None,
+            memory: Optional[str] = None,
+            disk_tier: Optional[resources_utils.DiskTier] = None
+    ) -> Optional[str]:
+        return service_catalog.get_default_instance_type(cpus=cpus,
+                                                         memory=memory,
+                                                         disk_tier=disk_tier,
+                                                         clouds='azure')
+
     @classmethod
     def get_image_size(cls, image_id: str, region: Optional[str]) -> float:
-        if region is None:
-            # The region used here is only for where to send the query,
-            # not the image location. Azure's image is globally available.
-            region = 'eastus'
-        is_skypilot_image_tag = False
+        # Process skypilot images.
         if image_id.startswith('skypilot:'):
-            is_skypilot_image_tag = True
             image_id = service_catalog.get_image_id_from_tag(image_id,
                                                              clouds='azure')
-        image_id_splitted = image_id.split(':')
-        if len(image_id_splitted) != 4:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(f'Invalid image id: {image_id}. Expected '
-                                 'format: <publisher>:<offer>:<sku>:<version>')
-        publisher, offer, sku, version = image_id_splitted
-        if is_skypilot_image_tag:
-            if offer == 'ubuntu-hpc':
-                return _DEFAULT_AZURE_UBUNTU_HPC_IMAGE_GB
+            if image_id.startswith(_COMMUNITY_IMAGE_PREFIX):
+                # Avoid querying the image size from Azure as
+                # all skypilot custom images have the same size.
+                return _DEFAULT_SKYPILOT_IMAGE_GB
             else:
-                return _DEFAULT_AZURE_UBUNTU_2004_IMAGE_GB
+                publisher, offer, sku, version = image_id.split(':')
+                if offer == 'ubuntu-hpc':
+                    return _DEFAULT_AZURE_UBUNTU_HPC_IMAGE_GB
+                else:
+                    return _DEFAULT_AZURE_UBUNTU_2004_IMAGE_GB
+
+        # Process user-specified images.
+        azure_utils.validate_image_id(image_id)
         compute_client = azure.get_client('compute', cls.get_project_id())
+
+        # Community gallery image.
+        if image_id.startswith(_COMMUNITY_IMAGE_PREFIX):
+            if region is None:
+                return 0.0
+            _, _, gallery_name, _, image_name = image_id.split('/')
+            try:
+                return azure_utils.get_community_image_size(
+                    compute_client, gallery_name, image_name, region)
+            except exceptions.ResourcesUnavailableError:
+                return 0.0
+
+        # Marketplace image
+        if region is None:
+            # The region used here is only for where to send the query,
+            # not the image location. Marketplace image is globally available.
+            region = 'eastus'
+        publisher, offer, sku, version = image_id.split(':')
         try:
             image = compute_client.virtual_machine_images.get(
                 region, publisher, offer, sku, version)
@@ -176,40 +213,23 @@ class Azure(clouds.Cloud):
         size_in_gb = size_in_bytes / (1024**3)
         return size_in_gb
 
-    @classmethod
-    def get_default_instance_type(
-            cls,
-            cpus: Optional[str] = None,
-            memory: Optional[str] = None,
-            disk_tier: Optional[resources_utils.DiskTier] = None
-    ) -> Optional[str]:
-        return service_catalog.get_default_instance_type(cpus=cpus,
-                                                         memory=memory,
-                                                         disk_tier=disk_tier,
-                                                         clouds='azure')
-
     def _get_default_image_tag(self, gen_version, instance_type) -> str:
         # ubuntu-2004 v21.08.30, K80 requires image with old NVIDIA driver version
         acc = self.get_accelerators_from_instance_type(instance_type)
         if acc is not None:
             acc_name = list(acc.keys())[0]
             if acc_name == 'K80':
-                return 'skypilot:k80-ubuntu-2004'
-
-        # ubuntu-2004 v21.11.04, the previous image we used in the past for
-        # V1 HyperV instance before we change default image to ubuntu-hpc.
+                return _DEFAULT_GPU_K80_IMAGE_ID
+        # About Gen V1 vs V2:
         # In Azure, all instances with K80 (Standard_NC series), some
         # instances with M60 (Standard_NV series) and some cpu instances
-        # (Basic_A, Standard_D, ...) are V1 instance. For these instances,
-        # we use the previous image.
+        # (Basic_A, Standard_D, ...) are V1 instance.
+        # All A100 instances are V2.
         if gen_version == 'V1':
-            return 'skypilot:v1-ubuntu-2004'
-
-        # nvidia-driver: 535.54.03, cuda: 12.2
-        # see: https://github.com/Azure/azhpc-images/releases/tag/ubuntu-hpc-20230803
-        # All A100 instances is of gen2, so it will always use
-        # the latest ubuntu-hpc:2204 image.
-        return 'skypilot:gpu-ubuntu-2204'
+            return _DEFAULT_V1_IMAGE_ID
+        if acc is None:
+            return _DEFAULT_CPU_IMAGE_ID
+        return _DEFAULT_GPU_IMAGE_ID
 
     @classmethod
     def regions_with_offering(cls, instance_type: str,
@@ -302,17 +322,34 @@ class Azure(clouds.Cloud):
             else:
                 assert region_name in resources.image_id, resources.image_id
                 image_id = resources.image_id[region_name]
+
+        # Checked basic image syntax in resources.py
         if image_id.startswith('skypilot:'):
             image_id = service_catalog.get_image_id_from_tag(image_id,
                                                              clouds='azure')
-        # Already checked in resources.py
-        publisher, offer, sku, version = image_id.split(':')
-        image_config = {
-            'image_publisher': publisher,
-            'image_offer': offer,
-            'image_sku': sku,
-            'image_version': version,
-        }
+            # Fallback if image does not exist in the specified region.
+            # Putting fallback here instead of at image validation
+            # when creating the resource because community images are
+            # regional so we need the correct region when we check whether
+            # the image exists.
+            if image_id.startswith(
+                    _COMMUNITY_IMAGE_PREFIX
+            ) and region_name not in azure_catalog.COMMUNITY_IMAGE_AVAILABLE_REGIONS:
+                logger.info(f'Azure image {image_id} does not exist in region '
+                            f'{region_name} so use the fallback image instead.')
+                image_id = service_catalog.get_image_id_from_tag(
+                    _FALLBACK_IMAGE_ID, clouds='azure')
+
+        if image_id.startswith(_COMMUNITY_IMAGE_PREFIX):
+            image_config = {'community_gallery_image_id': image_id}
+        else:
+            publisher, offer, sku, version = image_id.split(':')
+            image_config = {
+                'image_publisher': publisher,
+                'image_offer': offer,
+                'image_sku': sku,
+                'image_version': version,
+            }
 
         # Setup the A10 nvidia driver.
         need_nvidia_driver_extension = (acc_dict is not None and
@@ -380,7 +417,6 @@ class Azure(clouds.Cloud):
         # Setting disk performance tier for high disk tier.
         if disk_tier == resources_utils.DiskTier.HIGH:
             resources_vars['disk_performance_tier'] = 'P50'
-
         return resources_vars
 
     def _get_feasible_launchable_resources(

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/clouds/oci.py

@@ -468,7 +468,7 @@ class OCI(clouds.Cloud):
             api_key_file = oci_cfg[
                 'key_file'] if 'key_file' in oci_cfg else 'BadConf'
             sky_cfg_file = oci_utils.oci_config.get_sky_user_config_file()
-        except ImportError:
+        except (ImportError, oci_adaptor.oci.exceptions.ConfigFileNotFound):
             return {}
 
         # OCI config and API key file are mandatory

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/clouds/service_catalog/azure_catalog.py

@@ -12,6 +12,21 @@ from sky.clouds.service_catalog import common
 from sky.utils import resources_utils
 from sky.utils import ux_utils
 
+# This list should match the list of regions in
+# skypilot image generation Packer script's replication_regions
+# sky/clouds/service_catalog/images/skypilot-azure-cpu-ubuntu.pkr.hcl
+COMMUNITY_IMAGE_AVAILABLE_REGIONS = {
+    'centralus',
+    'eastus',
+    'eastus2',
+    'northcentralus',
+    'southcentralus',
+    'westcentralus',
+    'westus',
+    'westus2',
+    'westus3',
+}
+
 # The frequency of pulling the latest catalog from the cloud provider.
 # Though the catalog update is manual in our skypilot-catalog repo, we
 # still want to pull the latest catalog periodically to make sure the

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/clouds/service_catalog/kubernetes_catalog.py

@@ -120,8 +120,14 @@ def list_accelerators_realtime(
 
             # Generate the GPU quantities for the accelerators
             if accelerator_name and accelerator_count > 0:
-                for count in range(1, accelerator_count + 1):
+                count = 1
+                while count <= accelerator_count:
                     accelerators_qtys.add((accelerator_name, count))
+                    count *= 2
+                # Add the accelerator count if it's not already in the set
+                # (e.g., if there's 12 GPUs, we should have qtys 1, 2, 4, 8, 12)
+                if accelerator_count not in accelerators_qtys:
+                    accelerators_qtys.add((accelerator_name, accelerator_count))
 
             for pod in pods:
                 # Get all the pods running on the node

skypilot_nightly-1.0.0.dev20241025/sky/clouds/utils/azure_utils.py

@@ -0,0 +1,91 @@
+"""Utilies for Azure"""
+
+import typing
+
+from sky import exceptions
+from sky.adaptors import azure
+from sky.utils import ux_utils
+
+if typing.TYPE_CHECKING:
+    from azure.mgmt import compute as azure_compute
+    from azure.mgmt.compute import models as azure_compute_models
+
+
+def validate_image_id(image_id: str):
+    """Check if the image ID has a valid format.
+
+    Raises:
+        ValueError: If the image ID is invalid.
+    """
+    image_id_colon_splitted = image_id.split(':')
+    image_id_slash_splitted = image_id.split('/')
+    if len(image_id_slash_splitted) != 5 and len(image_id_colon_splitted) != 4:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(
+                f'Invalid image id for Azure: {image_id}. Expected format: \n'
+                '* Marketplace image ID: <publisher>:<offer>:<sku>:<version>\n'
+                '* Community image ID: '
+                '/CommunityGalleries/<gallery-name>/Images/<image-name>')
+    if len(image_id_slash_splitted) == 5:
+        _, gallery_type, _, image_type, _ = image_id.split('/')
+        if gallery_type != 'CommunityGalleries' or image_type != 'Images':
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    f'Invalid community image id for Azure: {image_id}.\n'
+                    'Expected format: '
+                    '/CommunityGalleries/<gallery-name>/Images/<image-name>')
+
+
+def get_community_image(
+        compute_client: 'azure_compute.ComputeManagementClient', image_id: str,
+        region: str) -> 'azure_compute_models.CommunityGalleryImage':
+    """Get community image from cloud.
+
+    Args:
+        image_id: /CommunityGalleries/<gallery-name>/Images/<image-name>
+    Raises:
+        ResourcesUnavailableError
+    """
+    try:
+        _, _, gallery_name, _, image_name = image_id.split('/')
+        return compute_client.community_gallery_images.get(
+            location=region,
+            public_gallery_name=gallery_name,
+            gallery_image_name=image_name)
+    except azure.exceptions().AzureError as e:
+        raise exceptions.ResourcesUnavailableError(
+            f'Community image {image_id} does not exist in region {region}.'
+        ) from e
+
+
+def get_community_image_size(
+        compute_client: 'azure_compute.ComputeManagementClient',
+        gallery_name: str, image_name: str, region: str) -> float:
+    """Get the size of the community image from cloud.
+
+    Args:
+        image_id: /CommunityGalleries/<gallery-name>/Images/<image-name>
+    Raises:
+        ResourcesUnavailableError
+    """
+    try:
+        image_versions = compute_client.community_gallery_image_versions.list(
+            location=region,
+            public_gallery_name=gallery_name,
+            gallery_image_name=image_name,
+        )
+        image_versions = list(image_versions)
+        if not image_versions:
+            raise exceptions.ResourcesUnavailableError(
+                f'No versions available for Azure community image {image_name}')
+        latest_version = image_versions[-1].name
+
+        image_details = compute_client.community_gallery_image_versions.get(
+            location=region,
+            public_gallery_name=gallery_name,
+            gallery_image_name=image_name,
+            gallery_image_version_name=latest_version)
+        return image_details.storage_profile.os_disk_image.disk_size_gb
+    except azure.exceptions().AzureError as e:
+        raise exceptions.ResourcesUnavailableError(
+            f'Failed to get community image size: {e}.') from e

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/exceptions.py

@@ -1,7 +1,7 @@
 """Exceptions."""
 import enum
 import typing
-from typing import List, Optional
+from typing import List, Optional, Sequence
 
 if typing.TYPE_CHECKING:
     from sky import status_lib
@@ -61,12 +61,12 @@ class ProvisionPrechecksError(Exception):
     the error will be raised.
 
     Args:
-        reasons: (List[Exception]) The reasons why the prechecks failed.
+        reasons: (Sequence[Exception]) The reasons why the prechecks failed.
     """
 
-    def __init__(self, reasons: List[Exception]) -> None:
+    def __init__(self, reasons: Sequence[Exception]) -> None:
         super().__init__()
-        self.reasons = list(reasons)
+        self.reasons = reasons
 
 
 class ManagedJobReachedMaxRetriesError(Exception):

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/jobs/recovery_strategy.py

@@ -24,6 +24,7 @@ from sky.utils import common_utils
 from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
+    from sky import resources
     from sky import task as task_lib
 
 logger = sky_logging.init_logger(__name__)
@@ -327,8 +328,7 @@ class StrategyExecutor:
                         'Failure happened before provisioning. Failover '
                         f'reasons: {reasons_str}')
                     if raise_on_failure:
-                        raise exceptions.ProvisionPrechecksError(
-                            reasons=reasons)
+                        raise exceptions.ProvisionPrechecksError(reasons)
                     return None
                 logger.info('Failed to launch a cluster with error: '
                             f'{common_utils.format_exception(e)})')
@@ -382,7 +382,7 @@ class FailoverStrategyExecutor(StrategyExecutor, name='FAILOVER',
         # first retry in the same cloud/region. (Inside recover() we may not
         # rely on cluster handle, as it can be None if the cluster is
         # preempted.)
-        self._launched_resources: Optional['sky.resources.Resources'] = None
+        self._launched_resources: Optional['resources.Resources'] = None
 
     def _launch(self,
                 max_retry: Optional[int] = 3,

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/provision/azure/azure-config-template.json

@@ -13,6 +13,12 @@
             "metadata": {
                 "description": "Subnet parameters."
             }
+        },
+        "nsgName": {
+            "type": "string",
+            "metadata": {
+                "description": "Name of the Network Security Group associated with the SkyPilot cluster."
+            }
         }
     },
     "variables": {
@@ -20,7 +26,7 @@
         "location": "[resourceGroup().location]",
         "msiName": "[concat('sky-', parameters('clusterId'), '-msi')]",
         "roleAssignmentName": "[concat('sky-', parameters('clusterId'), '-ra')]",
-        "nsgName": "[concat('sky-', parameters('clusterId'), '-nsg')]",
+        "nsgName": "[parameters('nsgName')]",
         "nsg": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]",
         "vnetName": "[concat('sky-', parameters('clusterId'), '-vnet')]",
         "subnetName": "[concat('sky-', parameters('clusterId'), '-subnet')]"

{skypilot_nightly-1.0.0.dev20241024 → skypilot_nightly-1.0.0.dev20241025}/sky/provision/azure/config.py

@@ -8,7 +8,7 @@ import json
 from pathlib import Path
 import random
 import time
-from typing import Any, Callable
+from typing import Any, Callable, Tuple
 
 from sky import exceptions
 from sky import sky_logging
@@ -22,6 +22,7 @@ UNIQUE_ID_LEN = 4
 _DEPLOYMENT_NAME = 'skypilot-config'
 _LEGACY_DEPLOYMENT_NAME = 'ray-config'
 _RESOURCE_GROUP_WAIT_FOR_DELETION_TIMEOUT = 480  # 8 minutes
+_CLUSTER_ID = '{cluster_name_on_cloud}-{unique_id}'
 
 
 def get_azure_sdk_function(client: Any, function_name: str) -> Callable:
@@ -41,6 +42,19 @@ def get_azure_sdk_function(client: Any, function_name: str) -> Callable:
     return func
 
 
+def get_cluster_id_and_nsg_name(resource_group: str,
+                                cluster_name_on_cloud: str) -> Tuple[str, str]:
+    hasher = hashlib.md5(resource_group.encode('utf-8'))
+    unique_id = hasher.hexdigest()[:UNIQUE_ID_LEN]
+    # We use the cluster name + resource group hash as the
+    # unique ID for the cluster, as we need to make sure that
+    # the deployments have unique names during failover.
+    cluster_id = _CLUSTER_ID.format(cluster_name_on_cloud=cluster_name_on_cloud,
+                                    unique_id=unique_id)
+    nsg_name = f'sky-{cluster_id}-nsg'
+    return cluster_id, nsg_name
+
+
 @common.log_function_start_end
 def bootstrap_instances(
         region: str, cluster_name_on_cloud: str,
@@ -117,12 +131,13 @@ def bootstrap_instances(
 
     logger.info(f'Using cluster name: {cluster_name_on_cloud}')
 
-    hasher = hashlib.md5(provider_config['resource_group'].encode('utf-8'))
-    unique_id = hasher.hexdigest()[:UNIQUE_ID_LEN]
+    cluster_id, nsg_name = get_cluster_id_and_nsg_name(
+        resource_group=provider_config['resource_group'],
+        cluster_name_on_cloud=cluster_name_on_cloud)
     subnet_mask = provider_config.get('subnet_mask')
     if subnet_mask is None:
         # choose a random subnet, skipping most common value of 0
-        random.seed(unique_id)
+        random.seed(cluster_id)
         subnet_mask = f'10.{random.randint(1, 254)}.0.0/16'
     logger.info(f'Using subnet mask: {subnet_mask}')
 
@@ -135,10 +150,10 @@ def bootstrap_instances(
                     'value': subnet_mask
                 },
                 'clusterId': {
-                    # We use the cluster name + resource group hash as the
-                    # unique ID for the cluster, as we need to make sure that
-                    # the deployments have unique names during failover.
-                    'value': f'{cluster_name_on_cloud}-{unique_id}'
+                    'value': cluster_id
+                },
+                'nsgName': {
+                    'value': nsg_name
                 },
             },
         }