skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/gcp/constants.py

@@ -1,4 +1,5 @@
 """Constants used by the GCP provisioner."""
+import textwrap
 
 VERSION = 'v1'
 # Using v2 according to
@@ -41,6 +42,223 @@ HAS_TPU_PROVIDER_FIELD = '_has_tpus'
 # with ServiceAccounts.
 
 SKYPILOT_VPC_NAME = 'skypilot-vpc'
+SKYPILOT_GPU_DIRECT_VPC_NUM = 5
+SKYPILOT_GPU_DIRECT_VPC_CIDR_PREFIX = '10.129'
+GPU_DIRECT_TCPX_INSTANCE_TYPES = [
+    'a3-edgegpu-8g',
+    'a3-highgpu-8g',
+]
+
+COMPACT_GROUP_PLACEMENT_POLICY = 'compact'
+COLLOCATED_COLLOCATION = 'COLLOCATED'
+
+# From https://cloud.google.com/compute/docs/gpus/gpudirect
+# A specific image is used to ensure that the the GPU is configured with TCPX support.
+GCP_GPU_DIRECT_IMAGE_ID = 'docker:us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx'
+GPU_DIRECT_TCPX_USER_DATA = textwrap.dedent("""
+    # Install GPU Direct TCPX
+    cos-extensions install gpu -- --version=latest;
+    sudo mount --bind /var/lib/nvidia /var/lib/nvidia;
+    sudo mount -o remount,exec /var/lib/nvidia;
+    docker ps -a | grep -q receive-datapath-manager || \
+    docker run \
+    --detach \
+    --pull=always \
+    --name receive-datapath-manager \
+    --privileged \
+    --cap-add=NET_ADMIN --network=host \
+    --volume /var/lib/nvidia/lib64:/usr/local/nvidia/lib64 \
+    --device /dev/nvidia0:/dev/nvidia0 --device /dev/nvidia1:/dev/nvidia1 \
+    --device /dev/nvidia2:/dev/nvidia2 --device /dev/nvidia3:/dev/nvidia3 \
+    --device /dev/nvidia4:/dev/nvidia4 --device /dev/nvidia5:/dev/nvidia5 \
+    --device /dev/nvidia6:/dev/nvidia6 --device /dev/nvidia7:/dev/nvidia7 \
+    --device /dev/nvidia-uvm:/dev/nvidia-uvm --device /dev/nvidiactl:/dev/nvidiactl \
+    --env LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \
+    --volume /run/tcpx:/run/tcpx \
+    --entrypoint /tcpgpudmarxd/build/app/tcpgpudmarxd \
+    us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/tcpgpudmarxd \
+    --gpu_nic_preset a3vm --gpu_shmem_type fd --uds_path "/run/tcpx" --setup_param "--verbose 128 2 0";
+    sudo iptables -I INPUT -p tcp -m tcp -j ACCEPT;
+    docker run --rm -v /var/lib:/var/lib us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx install --install-nccl;
+    sudo mount --bind /var/lib/tcpx /var/lib/tcpx;
+    sudo mount -o remount,exec /var/lib/tcpx;
+    echo "GPU Direct TCPX installed"
+    """)
+
+# Some NCCL options are from the following link.
+# https://docs.nvidia.com/dgx-cloud/run-ai/latest/appendix-gcp.html
+GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
+    '--cap-add=IPC_LOCK',
+    '--userns=host',
+    '--volume /run/tcpx:/run/tcpx',
+    '--volume /var/lib/nvidia/lib64:/usr/local/nvidia/lib64',
+    '--volume /var/lib/tcpx/lib64:/usr/local/tcpx/lib64',
+    '--volume /var/lib/nvidia/bin:/usr/local/nvidia/bin',
+    '--shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864',
+    '--device /dev/nvidia0:/dev/nvidia0',
+    '--device /dev/nvidia1:/dev/nvidia1',
+    '--device /dev/nvidia2:/dev/nvidia2',
+    '--device /dev/nvidia3:/dev/nvidia3',
+    '--device /dev/nvidia4:/dev/nvidia4',
+    '--device /dev/nvidia5:/dev/nvidia5',
+    '--device /dev/nvidia6:/dev/nvidia6',
+    '--device /dev/nvidia7:/dev/nvidia7',
+    '--device /dev/nvidia-uvm:/dev/nvidia-uvm',
+    '--device /dev/nvidiactl:/dev/nvidiactl',
+    '--env LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/tcpx/lib64',
+    '--env NCCL_GPUDIRECTTCPX_SOCKET_IFNAME=eth1,eth2,eth3,eth4',
+    '--env NCCL_GPUDIRECTTCPX_CTRL_DEV=eth0',
+    '--env NCCL_GPUDIRECTTCPX_TX_BINDINGS="eth1:8-21,112-125;eth2:8-21,112-125;eth3:60-73,164-177;eth4:60-73,164-177"',
+    '--env NCCL_GPUDIRECTTCPX_RX_BINDINGS="eth1:22-35,126-139;eth2:22-35,126-139;eth3:74-87,178-191;eth4:74-87,178-191"',
+    '--env NCCL_GPUDIRECTTCPX_PROGRAM_FLOW_STEERING_WAIT_MICROS=50000',
+    '--env NCCL_GPUDIRECTTCPX_UNIX_CLIENT_PREFIX="/run/tcpx"',
+    '--env NCCL_GPUDIRECTTCPX_FORCE_ACK=0',
+    '--env NCCL_SOCKET_IFNAME=eth0',
+]
+
+PD_EXTREME_IOPS = 20000
+DEFAULT_DISK_SIZE = 100
+NETWORK_STORAGE_TYPE = 'PERSISTENT'
+INSTANCE_STORAGE_TYPE = 'SCRATCH'
+INSTANCE_STORAGE_DISK_TYPE = 'local-ssd'
+INSTANCE_STORAGE_INTERFACE_TYPE = 'NVME'
+INSTANCE_STORAGE_DEVICE_NAME_PREFIX = '/dev/disk/by-id/google-local-nvme-ssd-'
+DEVICE_NAME_PREFIX = '/dev/disk/by-id/google-'
+
+BASH_SCRIPT_START = textwrap.dedent("""#!/bin/bash
+set -e
+set -x
+""")
+DISK_MOUNT_USER_DATA_TEMPLATE = textwrap.dedent("""
+    # Define arrays for devices and mount points
+    declare -A device_mounts=(
+        {device_mounts}
+    )
+
+    # Function to format and mount a single device
+    format_and_mount() {{
+        local device_name="$1"
+        local mount_point="$2"
+
+        if [ ! -e "$device_name" ]; then
+            echo "Error: Device $device_name does not exist."
+            return 1
+        fi
+
+        # Check if filesystem is already formatted (ext4)
+        if ! sudo blkid "$device_name" | grep -q 'TYPE="ext4"'; then
+            if [[ "$device_name" == "/dev/disk/by-id/google-local-nvme-ssd"* ]]; then
+                echo "Formatting local SSD $device_name..."
+                if ! sudo mkfs.ext4 -F "$device_name"; then
+                    echo "Error: Failed to format $device_name"
+                    return 1
+                fi
+            else
+                echo "Formatting persistent disk $device_name..."
+                if ! sudo mkfs.ext4 -m 0 -E lazy_itable_init=0,lazy_journal_init=0,discard "$device_name"; then
+                    echo "Error: Failed to format $device_name"
+                    return 1
+                fi
+            fi
+        else
+            echo "$device_name is already formatted."
+        fi
+
+        # Check if already mounted
+        if ! grep -q "$mount_point" /proc/mounts; then
+            echo "Mounting $device_name to $mount_point..."
+            if ! sudo mkdir -p "$mount_point"; then
+                echo "Error: Failed to create mount point $mount_point"
+                return 1
+            fi
+
+            if ! sudo mount "$device_name" "$mount_point"; then
+                echo "Error: Failed to mount $device_name to $mount_point"
+                return 1
+            fi
+
+            # Add to fstab if not already present
+            if ! grep -q " $mount_point " /etc/fstab; then
+                echo "Adding mount entry to /etc/fstab..."
+                echo "UUID=`sudo blkid -s UUID -o value $device_name` $mount_point ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
+            else
+                echo "Mount entry already exists in /etc/fstab"
+            fi
+        else
+            echo "$device_name is already mounted at $mount_point"
+        fi
+    }}
+
+    # Main execution
+    echo "Starting device mounting process..."
+
+    # Process each device-mount pair
+    for device in "${{!device_mounts[@]}}"; do
+        mount_point="${{device_mounts[$device]}}"
+        echo "Processing device: $device -> $mount_point"
+        if ! format_and_mount "$device" "$mount_point"; then
+            echo "Failed to process device $device"
+            # Continue with other devices even if one fails
+            continue
+        fi
+    done
+
+    echo "Device mounting process completed."
+""")
+
+# The local SSDs will be attached automatically to the following
+# machine types with the following number of disks.
+# Refer to https://cloud.google.com/compute/docs/disks/local-ssd#lssd_disks_fixed
+SSD_AUTO_ATTACH_MACHINE_TYPES = {
+    'c4a-standard-4-lssd': 1,
+    'c4a-highmem-4-lssd': 1,
+    'c4a-standard-8-lssd': 2,
+    'c4a-highmem-8-lssd': 2,
+    'c4a-standard-16-lssd': 4,
+    'c4a-highmem-16-lssd': 4,
+    'c4a-standard-32-lssd': 6,
+    'c4a-highmem-32-lssd': 6,
+    'c4a-standard-48-lssd': 10,
+    'c4a-highmem-48-lssd': 10,
+    'c4a-standard-64-lssd': 14,
+    'c4a-highmem-64-lssd': 14,
+    'c4a-standard-72-lssd': 16,
+    'c4a-highmem-72-lssd': 16,
+    'c3-standard-4-lssd': 1,
+    'c3-standard-8-lssd': 2,
+    'c3-standard-22-lssd': 4,
+    'c3-standard-44-lssd': 8,
+    'c3-standard-88-lssd': 16,
+    'c3-standard-176-lssd': 32,
+    'c3d-standard-8-lssd': 1,
+    'c3d-highmem-8-lssd': 1,
+    'c3d-standard-16-lssd': 1,
+    'c3d-highmem-16-lssd': 1,
+    'c3d-standard-30-lssd': 2,
+    'c3d-highmem-30-lssd': 2,
+    'c3d-standard-60-lssd': 4,
+    'c3d-highmem-60-lssd': 4,
+    'c3d-standard-90-lssd': 8,
+    'c3d-highmem-90-lssd': 8,
+    'c3d-standard-180-lssd': 16,
+    'c3d-highmem-180-lssd': 16,
+    'c3d-standard-360-lssd': 32,
+    'c3d-highmem-360-lssd': 32,
+    'a4-highgpu-8g': 32,
+    'a3-ultragpu-8g': 32,
+    'a3-megagpu-8g': 16,
+    'a3-highgpu-1g': 2,
+    'a3-highgpu-2g': 4,
+    'a3-highgpu-4g': 8,
+    'a3-highgpu-8g': 16,
+    'a3-edgegpu-8g': 16,
+    'a2-ultragpu-1g': 1,
+    'a2-ultragpu-2g': 2,
+    'a2-ultragpu-4g': 4,
+    'a2-ultragpu-8g': 8,
+    'z3-highmem-88': 12,
+    'z3-highmem-176': 12,
+}
 
 # Below parameters are from the default VPC on GCP.
 # https://cloud.google.com/vpc/docs/firewalls#more_rules_default_vpc

sky/provision/gcp/instance.py

@@ -4,15 +4,17 @@ import copy
 from multiprocessing import pool
 import re
 import time
-from typing import Any, Callable, Dict, Iterable, List, Optional, Type
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
 
 from sky import sky_logging
 from sky.adaptors import gcp
 from sky.provision import common
 from sky.provision import constants as provision_constants
+from sky.provision.gcp import config as gcp_config
 from sky.provision.gcp import constants
 from sky.provision.gcp import instance_utils
 from sky.utils import common_utils
+from sky.utils import resources_utils
 from sky.utils import status_lib
 
 logger = sky_logging.init_logger(__name__)
@@ -56,11 +58,14 @@ def _filter_instances(
 # for terminated instances, if they have already been fully deleted.
 @common_utils.retry
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     zone = provider_config['availability_zone']
     project_id = provider_config['project_id']
@@ -82,7 +87,8 @@ def query_instances(
     )
 
     raw_statuses = {}
-    statuses = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst_id, instance in instances.items():
         raw_status = instance[handler.STATUS_FIELD]
         raw_statuses[inst_id] = raw_status
@@ -96,7 +102,7 @@ def query_instances(
             status = None
         if non_terminated_only and status is None:
             continue
-        statuses[inst_id] = status
+        statuses[inst_id] = (status, None)
 
     # GCP does not clean up preempted TPU VMs. We remove it ourselves.
     if handler == instance_utils.GCPTPUVMInstance:
@@ -355,9 +361,10 @@ def _run_instances(region: str, cluster_name_on_cloud: str,
                                   created_instance_ids=created_instance_ids)
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     try:
         return _run_instances(region, cluster_name_on_cloud, config)
     except gcp.http_error_exception() as e:
@@ -530,9 +537,11 @@ def terminate_instances(
     use_mig = provider_config.get('use_managed_instance_group', False)
     if use_mig:
         # Deleting the MIG will also delete the instances.
-        instance_utils.GCPManagedInstanceGroup.delete_mig(
-            project_id, zone, cluster_name_on_cloud)
-        return
+        mig_exists_and_deleted = (
+            instance_utils.GCPManagedInstanceGroup.delete_mig(
+                project_id, zone, cluster_name_on_cloud))
+        if mig_exists_and_deleted:
+            return
 
     label_filters = {
         provision_constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud
@@ -570,6 +579,25 @@ def terminate_instances(
     # time (same as what we did in ray's node_provider).
 
 
+def cleanup_custom_multi_network(
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    failover: bool = False,
+) -> None:
+    """See sky/provision/__init__.py"""
+    assert provider_config is not None, cluster_name_on_cloud
+    project_id = provider_config['project_id']
+    region = provider_config['region']
+    enable_gpu_direct = provider_config.get('enable_gpu_direct', False)
+    network_tier = provider_config.get('network_tier', 'standard')
+
+    if (enable_gpu_direct or
+            network_tier == resources_utils.NetworkTier.BEST.value):
+        gcp_config.delete_gpu_direct_vpcs_and_subnets(cluster_name_on_cloud,
+                                                      project_id, region,
+                                                      failover)
+
+
 def open_ports(
     cluster_name_on_cloud: str,
     ports: List[str],

sky/provision/gcp/instance_utils.py

@@ -826,6 +826,16 @@ class GCPComputeInstance(GCPInstance):
         # https://cloud.google.com/compute/docs/reference/rest/v1/instances/bulkInsert # pylint: disable=line-too-long
         if config.get('sourceMachineImage') is not None:
             return False
+        # bulkInsert does not support attaching existing
+        # disks to the instances with READ_WRITE mode.
+        if config.get('disks') is not None:
+            for disk in config['disks']:
+                if disk.get('source') is not None and disk.get(
+                        'mode', 'READ_WRITE') == 'READ_WRITE':
+                    return False
+                if disk.get('initializeParams') is not None and disk.get(
+                        'initializeParams', {}).get('diskName') is not None:
+                    return False
         return True
 
     @classmethod
@@ -1125,12 +1135,14 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
             if re.search(mig_utils.IT_RESOURCE_NOT_FOUND_PATTERN,
                          str(e)) is None:
                 raise
-            logger.warning(
+            logger.debug(
                 f'Instance template {instance_template_name!r} does not exist. '
                 'Skip deletion.')
 
     @classmethod
-    def delete_mig(cls, project_id: str, zone: str, cluster_name: str) -> None:
+    def delete_mig(cls, project_id: str, zone: str, cluster_name: str) -> bool:
+        """Returns whether the MIG is deleted successfully."""
+        mig_exists_and_deleted = True
         mig_name = mig_utils.get_managed_instance_group_name(cluster_name)
         # Get all resize request of the MIG and cancel them.
         mig_utils.cancel_all_resize_request_for_mig(project_id, zone, mig_name)
@@ -1144,8 +1156,9 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
             if re.search(mig_utils.MIG_RESOURCE_NOT_FOUND_PATTERN,
                          str(e)) is None:
                 raise
-            logger.warning(f'MIG {mig_name!r} does not exist. Skip '
-                           'deletion.')
+            logger.debug(f'MIG {mig_name!r} does not exist. Skip '
+                         'deletion.')
+            mig_exists_and_deleted = False
 
         # In the autostop case, the following deletion of instance template
         # will not be executed as the instance that runs the deletion will be
@@ -1156,6 +1169,7 @@ class GCPManagedInstanceGroup(GCPComputeInstance):
         cls._delete_instance_template(
             project_id, zone,
             mig_utils.get_instance_template_name(cluster_name))
+        return mig_exists_and_deleted
 
     @classmethod
     def _add_labels_and_find_head(

sky/provision/gcp/volume_utils.py

@@ -0,0 +1,247 @@
+"""Utilities for GCP volumes."""
+from typing import Any, Dict, List, Optional
+
+from sky import clouds
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import gcp
+from sky.provision.gcp import constants
+from sky.utils import resources_utils
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+
+def get_data_disk_tier_mapping(
+    instance_type: Optional[str],) -> Dict[resources_utils.DiskTier, str]:
+    # Define the default mapping from disk tiers to disk types.
+    # Refer to https://cloud.google.com/compute/docs/disks/hyperdisks
+    # and https://cloud.google.com/compute/docs/disks/persistent-disks
+    tier2name = {
+        resources_utils.DiskTier.ULTRA: 'pd-extreme',
+        resources_utils.DiskTier.HIGH: 'pd-ssd',
+        resources_utils.DiskTier.MEDIUM: 'pd-balanced',
+        resources_utils.DiskTier.LOW: 'pd-standard',
+    }
+
+    if instance_type is None:
+        return tier2name
+
+    # Remap series-specific disk types.
+    series = instance_type.split('-')[0]
+
+    if series in ['a4', 'x4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+    elif series in ['m4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 112:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['c4', 'c4a', 'c4d']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['a3']:
+        if (instance_type.startswith('a3-ultragpu') or
+                instance_type.startswith('a3-megagpu') or
+                instance_type.startswith('a3-edgegpu')):
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+            tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+            tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+            tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+        elif instance_type.startswith('a3-highgpu'):
+            tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+            if instance_type.startswith('a3-highgpu-8g'):
+                tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+                tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+                tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+            elif instance_type.startswith('a3-highgpu-4g'):
+                tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+            else:
+                tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+    elif series in ['c3d']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 60:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['c3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 88:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['n4']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'hyperdisk-balanced'
+    elif series in ['n2d', 'n1', 't2d', 't2a', 'e2', 'c2', 'c2d', 'a2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+    elif series in ['z3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['h3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['m3']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        tier2name[resources_utils.DiskTier.MEDIUM] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['m2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+    elif series in ['m1']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+        tier2name[resources_utils.DiskTier.HIGH] = 'hyperdisk-balanced'
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 80:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-balanced'
+    elif series in ['g2']:
+        tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+        tier2name[resources_utils.DiskTier.LOW] = 'pd-balanced'
+    elif series in ['n2']:
+        num_cpus = int(instance_type.split('-')[2])  # type: ignore
+        if num_cpus < 64:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'pd-ssd'
+        elif num_cpus >= 80:
+            tier2name[resources_utils.DiskTier.ULTRA] = 'hyperdisk-extreme'
+
+    return tier2name
+
+
+def validate_instance_volumes(
+    instance_type: Optional[str],
+    volumes: Optional[List[Dict[str, Any]]],
+) -> None:
+    if not volumes:
+        return
+    if instance_type is None:
+        logger.warning('Instance type is not specified,'
+                       ' skipping instance volume validation')
+        return
+    instance_volume_count = 0
+    for volume in volumes:
+        if volume['storage_type'] == resources_utils.StorageType.INSTANCE:
+            instance_volume_count += 1
+    if (instance_type in constants.SSD_AUTO_ATTACH_MACHINE_TYPES and
+            instance_volume_count >
+            constants.SSD_AUTO_ATTACH_MACHINE_TYPES[instance_type]):
+        raise exceptions.ResourcesUnavailableError(
+            f'The instance type {instance_type} supports'
+            f' {constants.SSD_AUTO_ATTACH_MACHINE_TYPES[instance_type]}'
+            f'  instance storage, but {instance_volume_count} are specified')
+    # TODO(hailong):
+    # check the instance storage count for the other instance types,
+    # refer to https://cloud.google.com/compute/docs/disks/local-ssd
+
+
+def translate_attach_mode(attach_mode: resources_utils.DiskAttachMode) -> str:
+    if attach_mode == resources_utils.DiskAttachMode.READ_ONLY:
+        return 'READ_ONLY'
+    return 'READ_WRITE'
+
+
+def check_volume_name_exist_in_region(
+        project_id: str, region: clouds.Region, use_mig: bool,
+        volume_name: str) -> Optional[Dict[str, Any]]:
+    """Check if the volume name exists and return the volume info."""
+    logger.debug(f'Checking volume {volume_name} in region {region}')
+    try:
+        compute = gcp.build('compute',
+                            'v1',
+                            credentials=None,
+                            cache_discovery=False)
+    except gcp.credential_error_exception():
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('Not able to build compute client') from None
+
+    # Get all the zones in the region
+    all_zones = compute.zones().list(project=project_id).execute()
+    region_zones = []
+    if 'items' in all_zones:
+        for zone in all_zones['items']:
+            if zone['region'].split('/')[-1] == region.name:
+                region_zones.append(zone['name'])
+    volume_info = None
+    for zone in region_zones:
+        try:
+            volume_info = compute.disks().get(project=project_id,
+                                              zone=zone,
+                                              disk=volume_name).execute()
+            if volume_info is not None:
+                if use_mig:
+                    # With MIG, instance template will be used, in this case,
+                    # the `selfLink` for zonal disk needs to be the volume name
+                    # Refer to https://cloud.google.com/compute/docs/
+                    # reference/rest/v1/instances/insert
+                    volume_info['selfLink'] = volume_name
+                volume_info['available_zones'] = [zone]
+                return volume_info
+        except gcp.http_error_exception() as e:
+            if e.resp.status == 403:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError('Not able to access the volume '
+                                     f'{volume_name!r}') from None
+            if e.resp.status == 404:
+                continue  # Try next zone
+            raise
+
+    # If not found in any zone, check region disk
+    try:
+        volume_info = compute.regionDisks().get(project=project_id,
+                                                region=region.name,
+                                                disk=volume_name).execute()
+        # 'replicaZones':
+        #  ['https://xxx/compute/v1/projects/sky-dev-465/zones/us-central1-a',
+        # 'https://xxx/compute/v1/projects/sky-dev-465/zones/us-central1-c']
+        if volume_info is not None and 'replicaZones' in volume_info:
+            replica_zones = [
+                zone.split('/')[-1] for zone in volume_info['replicaZones']
+            ]
+            volume_info['available_zones'] = replica_zones
+        return volume_info
+    except gcp.http_error_exception() as e:
+        if e.resp.status == 403:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('Not able to access the volume '
+                                 f'{volume_name!r}') from None
+        if e.resp.status == 404:
+            logger.warning(
+                f'Volume {volume_name} is not found in region {region}.'
+                f' It will be created.')
+            return volume_info
+        raise
+
+
+def check_volume_zone_match(volume_name: str,
+                            zones: Optional[List[clouds.Zone]],
+                            available_zones: List[str]):
+    if zones is None:
+        return None
+    for zone in zones:
+        if zone.name in available_zones:
+            return None
+    with ux_utils.print_exception_no_traceback():
+        # Return a ResourcesUnavailableError to trigger failover
+        raise exceptions.ResourcesUnavailableError(
+            f'Volume {volume_name} not available in zones {zones}') from None

sky/provision/hyperbolic/__init__.py

@@ -0,0 +1,12 @@
+"""Hyperbolic provisioner for SkyPilot."""
+
+from sky.provision.hyperbolic.config import bootstrap_instances
+from sky.provision.hyperbolic.instance import cleanup_custom_multi_network
+from sky.provision.hyperbolic.instance import cleanup_ports
+from sky.provision.hyperbolic.instance import get_cluster_info
+from sky.provision.hyperbolic.instance import open_ports
+from sky.provision.hyperbolic.instance import query_instances
+from sky.provision.hyperbolic.instance import run_instances
+from sky.provision.hyperbolic.instance import stop_instances
+from sky.provision.hyperbolic.instance import terminate_instances
+from sky.provision.hyperbolic.instance import wait_instances

sky/provision/hyperbolic/config.py

@@ -0,0 +1,10 @@
+"""Hyperbolic Cloud configuration bootstrapping"""
+
+from sky.provision import common
+
+
+def bootstrap_instances(
+        region: str, cluster_name: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    del region, cluster_name  # unused
+    return config