skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/provision/gcp/instance.py

@@ -4,15 +4,17 @@ import copy
 from multiprocessing import pool
 import re
 import time
-from typing import Any, Callable, Dict, Iterable, List, Optional, Type
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
 
 from sky import sky_logging
 from sky.adaptors import gcp
 from sky.provision import common
 from sky.provision import constants as provision_constants
+from sky.provision.gcp import config as gcp_config
 from sky.provision.gcp import constants
 from sky.provision.gcp import instance_utils
 from sky.utils import common_utils
+from sky.utils import resources_utils
 from sky.utils import status_lib
 
 logger = sky_logging.init_logger(__name__)
@@ -56,11 +58,14 @@ def _filter_instances(
 # for terminated instances, if they have already been fully deleted.
 @common_utils.retry
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     zone = provider_config['availability_zone']
     project_id = provider_config['project_id']
@@ -82,7 +87,8 @@ def query_instances(
     )
 
     raw_statuses = {}
-    statuses = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst_id, instance in instances.items():
         raw_status = instance[handler.STATUS_FIELD]
         raw_statuses[inst_id] = raw_status
@@ -96,7 +102,7 @@ def query_instances(
             status = None
         if non_terminated_only and status is None:
             continue
-        statuses[inst_id] = status
+        statuses[inst_id] = (status, None)
 
     # GCP does not clean up preempted TPU VMs. We remove it ourselves.
     if handler == instance_utils.GCPTPUVMInstance:
@@ -355,9 +361,10 @@ def _run_instances(region: str, cluster_name_on_cloud: str,
                                   created_instance_ids=created_instance_ids)
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     try:
         return _run_instances(region, cluster_name_on_cloud, config)
     except gcp.http_error_exception() as e:
@@ -530,9 +537,11 @@ def terminate_instances(
     use_mig = provider_config.get('use_managed_instance_group', False)
     if use_mig:
         # Deleting the MIG will also delete the instances.
-        instance_utils.GCPManagedInstanceGroup.delete_mig(
-            project_id, zone, cluster_name_on_cloud)
-        return
+        mig_exists_and_deleted = (
+            instance_utils.GCPManagedInstanceGroup.delete_mig(
+                project_id, zone, cluster_name_on_cloud))
+        if mig_exists_and_deleted:
+            return
 
     label_filters = {
         provision_constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud
@@ -570,6 +579,25 @@ def terminate_instances(
     # time (same as what we did in ray's node_provider).
 
 
+def cleanup_custom_multi_network(
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    failover: bool = False,
+) -> None:
+    """See sky/provision/__init__.py"""
+    assert provider_config is not None, cluster_name_on_cloud
+    project_id = provider_config['project_id']
+    region = provider_config['region']
+    enable_gpu_direct = provider_config.get('enable_gpu_direct', False)
+    network_tier = provider_config.get('network_tier', 'standard')
+
+    if (enable_gpu_direct or
+            network_tier == resources_utils.NetworkTier.BEST.value):
+        gcp_config.delete_gpu_direct_vpcs_and_subnets(cluster_name_on_cloud,
+                                                      project_id, region,
+                                                      failover)
+
+
 def open_ports(
     cluster_name_on_cloud: str,
     ports: List[str],

sky/provision/gcp/instance_utils.py

@@ -826,6 +826,16 @@ class GCPComputeInstance(GCPInstance):
         # https://cloud.google.com/compute/docs/reference/rest/v1/instances/bulkInsert # pylint: disable=line-too-long
         if config.get('sourceMachineImage') is not None:
             return False
+        # bulkInsert does not support attaching existing
+        # disks to the instances with READ_WRITE mode.
+        if config.get('disks') is not None:
+            for disk in config['disks']:
+                if disk.get('source') is not None and disk.get(
+                        'mode', 'READ_WRITE') == 'READ_WRITE':
+                    return False
+                if disk.get('initializeParams') is not None and disk.get(
+                        'initializeParams', {}).get('diskName') is not None:
+                    return False
         return True
 
     @classmethod
@@ -1125,12 +1135,14 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
             if re.search(mig_utils.IT_RESOURCE_NOT_FOUND_PATTERN,
                          str(e)) is None:
                 raise
-            logger.warning(
+            logger.debug(
                 f'Instance template {instance_template_name!r} does not exist. '
                 'Skip deletion.')
 
     @classmethod
-    def delete_mig(cls, project_id: str, zone: str, cluster_name: str) -> None:
+    def delete_mig(cls, project_id: str, zone: str, cluster_name: str) -> bool:
+        """Returns whether the MIG is deleted successfully."""
+        mig_exists_and_deleted = True
         mig_name = mig_utils.get_managed_instance_group_name(cluster_name)
         # Get all resize request of the MIG and cancel them.
         mig_utils.cancel_all_resize_request_for_mig(project_id, zone, mig_name)
@@ -1144,8 +1156,9 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
             if re.search(mig_utils.MIG_RESOURCE_NOT_FOUND_PATTERN,
                          str(e)) is None:
                 raise
-            logger.warning(f'MIG {mig_name!r} does not exist. Skip '
-                           'deletion.')
+            logger.debug(f'MIG {mig_name!r} does not exist. Skip '
+                         'deletion.')
+            mig_exists_and_deleted = False
 
         # In the autostop case, the following deletion of instance template
         # will not be executed as the instance that runs the deletion will be
@@ -1156,6 +1169,7 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
         cls._delete_instance_template(
             project_id, zone,
             mig_utils.get_instance_template_name(cluster_name))
+        return mig_exists_and_deleted
 
     @classmethod
     def _add_labels_and_find_head(

sky/provision/gcp/volume_utils.py

@@ -0,0 +1,247 @@
+"""Utilities for GCP volumes."""
+from typing import Any, Dict, List, Optional
+
+from sky import clouds
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import gcp
+from sky.provision.gcp import constants
+from sky.utils import resources_utils
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+
+def get_data_disk_tier_mapping(
+    instance_type: Optional[str],) -> Dict[resources_utils.DiskTier, str]:
+    # Define the default mapping from disk tiers to disk types.
+    # Refer to https://cloud.google.com/compute/docs/disks/hyperdisks
+    # and https://cloud.google.com/compute/docs/disks/persistent-disks
+    tier2name = {
+        resources_utils.DiskTier.ULTRA: 'pd-extreme',
+        resources_utils.DiskTier.HIGH: 'pd-ssd',
+        resources_utils.DiskTier.MEDIUM: 'pd-balanced',
+        resources_utils.DiskTier.LOW: 'pd-standard',
+    }
+
+    if instance_type is None:
+        return tier2name
+
+    # Remap series-specific disk types.
+    series = instance_type.split('-')[0]
+
+    if series in ['a4', 'x4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+    elif series in ['m4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 112:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['c4', 'c4a', 'c4d']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['a3']:
+        if (instance_type.startswith('a3-ultragpu') or
+                instance_type.startswith('a3-megagpu') or
+                instance_type.startswith('a3-edgegpu')):
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+            tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+            tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+            tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        elif instance_type.startswith('a3-highgpu'):
+            tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+            if instance_type.startswith('a3-highgpu-8g'):
+                tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+                tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+                tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+            elif instance_type.startswith('a3-highgpu-4g'):
+                tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+            else:
+                tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+    elif series in ['c3d']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 60:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['c3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 88:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['n4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+    elif series in ['n2d', 'n1', 't2d', 't2a', 'e2', 'c2', 'c2d', 'a2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+    elif series in ['z3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['h3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['m3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['m2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+    elif series in ['m1']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 80:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['g2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['n2']:
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+        elif num_cpus >= 80:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+
+    return tier2name
+
+
+def validate_instance_volumes(
+    instance_type: Optional[str],
+    volumes: Optional[List[Dict[str, Any]]],
+) -> None:
+    if not volumes:
+        return
+    if instance_type is None:
+        logger.warning('Instance type is not specified,'
+                       ' skipping instance volume validation')
+        return
+    instance_volume_count = 0
+    for volume in volumes:
+        if volume['storage_type'] == resources_utils.StorageType.INSTANCE:
+            instance_volume_count += 1
+    if (instance_type in constants.SSD_AUTO_ATTACH_MACHINE_TYPES and
+            instance_volume_count >
+            constants.SSD_AUTO_ATTACH_MACHINE_TYPES[instance_type]):
+        raise exceptions.ResourcesUnavailableError(
+            f'The instance type {instance_type} supports'
+            f' {constants.SSD_AUTO_ATTACH_MACHINE_TYPES[instance_type]}'
+            f'  instance storage, but {instance_volume_count} are specified')
+    # TODO(hailong):
+    # check the instance storage count for the other instance types,
+    # refer to https://cloud.google.com/compute/docs/disks/local-ssd
+
+
+def translate_attach_mode(attach_mode: resources_utils.DiskAttachMode) -> str:
+    if attach_mode == resources_utils.DiskAttachMode.READ_ONLY:
+        return 'READ_ONLY'
+    return 'READ_WRITE'
+
+
+def check_volume_name_exist_in_region(
+        project_id: str, region: clouds.Region, use_mig: bool,
+        volume_name: str) -> Optional[Dict[str, Any]]:
+    """Check if the volume name exists and return the volume info."""
+    logger.debug(f'Checking volume {volume_name} in region {region}')
+    try:
+        compute = gcp.build('compute',
+                            'v1',
+                            credentials=None,
+                            cache_discovery=False)
+    except gcp.credential_error_exception():
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('Not able to build compute client') from None
+
+    # Get all the zones in the region
+    all_zones = compute.zones().list(project=project_id).execute()
+    region_zones = []
+    if 'items' in all_zones:
+        for zone in all_zones['items']:
+            if zone['region'].split('/')[-1] == region.name:
+                region_zones.append(zone['name'])
+    volume_info = None
+    for zone in region_zones:
+        try:
+            volume_info = compute.disks().get(project=project_id,
+                                              zone=zone,
+                                              disk=volume_name).execute()
+            if volume_info is not None:
+                if use_mig:
+                    # With MIG, instance template will be used, in this case,
+                    # the `selfLink` for zonal disk needs to be the volume name
+                    # Refer to https://cloud.google.com/compute/docs/
+                    # reference/rest/v1/instances/insert
+                    volume_info['selfLink'] = volume_name
+                volume_info['available_zones'] = [zone]
+                return volume_info
+        except gcp.http_error_exception() as e:
+            if e.resp.status == 403:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError('Not able to access the volume '
+                                     f'{volume_name!r}') from None
+            if e.resp.status == 404:
+                continue  # Try next zone
+            raise
+
+    # If not found in any zone, check region disk
+    try:
+        volume_info = compute.regionDisks().get(project=project_id,
+                                                region=region.name,
+                                                disk=volume_name).execute()
+        # 'replicaZones':
+        #  ['https://xxx/compute/v1/projects/sky-dev-465/zones/us-central1-a',
+        # 'https://xxx/compute/v1/projects/sky-dev-465/zones/us-central1-c']
+        if volume_info is not None and 'replicaZones' in volume_info:
+            replica_zones = [
+                zone.split('/')[-1] for zone in volume_info['replicaZones']
+            ]
+            volume_info['available_zones'] = replica_zones
+        return volume_info
+    except gcp.http_error_exception() as e:
+        if e.resp.status == 403:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('Not able to access the volume '
+                                 f'{volume_name!r}') from None
+        if e.resp.status == 404:
+            logger.warning(
+                f'Volume {volume_name} is not found in region {region}.'
+                f' It will be created.')
+            return volume_info
+        raise
+
+
+def check_volume_zone_match(volume_name: str,
+                            zones: Optional[List[clouds.Zone]],
+                            available_zones: List[str]):
+    if zones is None:
+        return None
+    for zone in zones:
+        if zone.name in available_zones:
+            return None
+    with ux_utils.print_exception_no_traceback():
+        # Return a ResourcesUnavailableError to trigger failover
+        raise exceptions.ResourcesUnavailableError(
+            f'Volume {volume_name} not available in zones {zones}') from None

sky/provision/hyperbolic/__init__.py

@@ -0,0 +1,12 @@
+"""Hyperbolic provisioner for SkyPilot."""
+
+from sky.provision.hyperbolic.config import bootstrap_instances
+from sky.provision.hyperbolic.instance import cleanup_custom_multi_network
+from sky.provision.hyperbolic.instance import cleanup_ports
+from sky.provision.hyperbolic.instance import get_cluster_info
+from sky.provision.hyperbolic.instance import open_ports
+from sky.provision.hyperbolic.instance import query_instances
+from sky.provision.hyperbolic.instance import run_instances
+from sky.provision.hyperbolic.instance import stop_instances
+from sky.provision.hyperbolic.instance import terminate_instances
+from sky.provision.hyperbolic.instance import wait_instances

sky/provision/hyperbolic/config.py

@@ -0,0 +1,10 @@
+"""Hyperbolic Cloud configuration bootstrapping"""
+
+from sky.provision import common
+
+
+def bootstrap_instances(
+        region: str, cluster_name: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    del region, cluster_name  # unused
+    return config