skypilot-nightly 1.0.0.dev20240912__py3-none-any.whl → 1.0.0.dev20240914__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '49b27fc889efa20ca688909b624f01da214a7be8'
+_SKYPILOT_COMMIT_SHA = 'c1464e14358f7eecb1c3b7e1de69b17e107a06fe'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240912'
+__version__ = '1.0.0.dev20240914'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/clouds/gcp.py

@@ -424,6 +424,21 @@ class GCP(clouds.Cloud):
         # issue when first booted.
         image_id = 'skypilot:cpu-debian-11'
 
+        def _failover_disk_tier() -> Optional[resources_utils.DiskTier]:
+            if (r.disk_tier is not None and
+                    r.disk_tier != resources_utils.DiskTier.BEST):
+                return r.disk_tier
+            # Failover disk tier from ultra to low.
+            all_tiers = list(reversed(resources_utils.DiskTier))
+            start_index = all_tiers.index(GCP._translate_disk_tier(r.disk_tier))
+            while start_index < len(all_tiers):
+                disk_tier = all_tiers[start_index]
+                ok, _ = GCP.check_disk_tier(r.instance_type, disk_tier)
+                if ok:
+                    return disk_tier
+                start_index += 1
+            assert False, 'Low disk tier should always be supported on GCP.'
+
         r = resources
         # Find GPU spec, if any.
         resources_vars = {
@@ -437,7 +452,7 @@ class GCP(clouds.Cloud):
             'custom_resources': None,
             'use_spot': r.use_spot,
             'gcp_project_id': self.get_project_id(dryrun),
-            **GCP._get_disk_specs(r.disk_tier),
+            **GCP._get_disk_specs(_failover_disk_tier()),
         }
         accelerators = r.accelerators
         if accelerators is not None:
@@ -532,6 +547,10 @@ class GCP(clouds.Cloud):
     ) -> 'resources_utils.FeasibleResources':
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
+            ok, _ = GCP.check_disk_tier(resources.instance_type,
+                                        resources.disk_tier)
+            if not ok:
+                return resources_utils.FeasibleResources([], [], None)
             return resources_utils.FeasibleResources([resources], [], None)
 
         if resources.accelerators is None:
@@ -544,15 +563,17 @@ class GCP(clouds.Cloud):
                 # TODO: Add hints to all return values in this method to help
                 #  users understand why the resources are not launchable.
                 return resources_utils.FeasibleResources([], [], None)
-            else:
-                r = resources.copy(
-                    cloud=GCP(),
-                    instance_type=host_vm_type,
-                    accelerators=None,
-                    cpus=None,
-                    memory=None,
-                )
-                return resources_utils.FeasibleResources([r], [], None)
+            ok, _ = GCP.check_disk_tier(host_vm_type, resources.disk_tier)
+            if not ok:
+                return resources_utils.FeasibleResources([], [], None)
+            r = resources.copy(
+                cloud=GCP(),
+                instance_type=host_vm_type,
+                accelerators=None,
+                cpus=None,
+                memory=None,
+            )
+            return resources_utils.FeasibleResources([r], [], None)
 
         # Find instance candidates to meet user's requirements
         assert len(resources.accelerators.items()
@@ -615,6 +636,10 @@ class GCP(clouds.Cloud):
         else:
             host_vm_type = instance_list[0]
 
+        ok, _ = GCP.check_disk_tier(host_vm_type, resources.disk_tier)
+        if not ok:
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
         acc_dict = {acc: acc_count}
         r = resources.copy(
             cloud=GCP(),
@@ -916,6 +941,36 @@ class GCP(clouds.Cloud):
             resources.instance_type, resources.accelerators, resources.zone,
             'gcp')
 
+    @classmethod
+    def check_disk_tier(
+            cls, instance_type: Optional[str],
+            disk_tier: Optional[resources_utils.DiskTier]) -> Tuple[bool, str]:
+        if disk_tier != resources_utils.DiskTier.ULTRA or instance_type is None:
+            return True, ''
+        # Ultra disk tier (pd-extreme) only support m2, m3 and part of n2
+        # instance types, so we failover to lower tiers for other instance
+        # types. Reference:
+        # https://cloud.google.com/compute/docs/disks/extreme-persistent-disk#machine_shape_support  # pylint: disable=line-too-long
+        series = instance_type.split('-')[0]
+        if series in ['m2', 'm3', 'n2']:
+            if series == 'n2':
+                num_cpus = int(instance_type.split('-')[2])
+                if num_cpus < 64:
+                    return False, ('n2 series with less than 64 vCPUs are '
+                                   'not supported with pd-extreme.')
+            return True, ''
+        return False, (f'{series} series is not supported with pd-extreme. '
+                       'Only m2, m3 series and n2 series with 64 or more vCPUs '
+                       'are supported.')
+
+    @classmethod
+    def check_disk_tier_enabled(cls, instance_type: Optional[str],
+                                disk_tier: resources_utils.DiskTier) -> None:
+        ok, msg = cls.check_disk_tier(instance_type, disk_tier)
+        if not ok:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.NotSupportedError(msg)
+
     @classmethod
     def _get_disk_type(cls,
                        disk_tier: Optional[resources_utils.DiskTier]) -> str:

sky/clouds/service_catalog/__init__.py

@@ -337,10 +337,13 @@ def get_common_gpus() -> List[str]:
 def get_tpus() -> List[str]:
     """Returns a list of TPU names."""
     # TODO(wei-lin): refactor below hard-coded list.
+    # There are many TPU configurations available, we show the three smallest
+    # and the largest configuration for the latest gen TPUs.
     return [
-        'tpu-v2-8', 'tpu-v2-32', 'tpu-v2-128', 'tpu-v2-256', 'tpu-v2-512',
-        'tpu-v3-8', 'tpu-v3-32', 'tpu-v3-64', 'tpu-v3-128', 'tpu-v3-256',
-        'tpu-v3-512', 'tpu-v3-1024', 'tpu-v3-2048'
+        'tpu-v2-512', 'tpu-v3-2048', 'tpu-v4-8', 'tpu-v4-16', 'tpu-v4-32',
+        'tpu-v4-3968', 'tpu-v5litepod-1', 'tpu-v5litepod-4', 'tpu-v5litepod-8',
+        'tpu-v5litepod-256', 'tpu-v5p-8', 'tpu-v5p-32', 'tpu-v5p-128',
+        'tpu-v5p-12288'
     ]
 
 

sky/clouds/service_catalog/data_fetchers/fetch_gcp.py

@@ -10,6 +10,7 @@ import io
 import multiprocessing
 import os
 import textwrap
+import time
 import typing
 from typing import Any, Callable, Dict, List, Optional, Set
 
@@ -19,6 +20,7 @@ import numpy as np
 
 from sky.adaptors import common as adaptors_common
 from sky.adaptors import gcp
+from sky.utils import common_utils
 
 if typing.TYPE_CHECKING:
     import pandas as pd
@@ -38,6 +40,9 @@ TPU_SERVICE_ID = 'E000-3F24-B8AA'
 # The number of digits to round the price to.
 PRICE_ROUNDING = 5
 
+# The number of retries for the TPU API.
+TPU_RETRY_CNT = 3
+
 # This zone is only for TPU v4, and does not appear in the skus yet.
 TPU_V4_ZONES = ['us-central2-b']
 # TPU v3 pods are available in us-east1-d, but hidden in the skus.
@@ -518,6 +523,34 @@ def get_gpu_df(skus: List[Dict[str, Any]],
     return df
 
 
+def _get_tpu_response_for_zone(zone: str) -> list:
+    parent = f'projects/{project_id}/locations/{zone}'
+    # Sometimes the response is empty ({}) even for enabled zones. Here we
+    # retry the request for a few times.
+    backoff = common_utils.Backoff(initial_backoff=1)
+    for _ in range(TPU_RETRY_CNT):
+        tpus_request = (
+            tpu_client.projects().locations().acceleratorTypes().list(
+                parent=parent))
+        try:
+            tpus_response = tpus_request.execute()
+            if 'acceleratorTypes' in tpus_response:
+                return tpus_response['acceleratorTypes']
+        except gcp.http_error_exception() as error:
+            if error.resp.status == 403:
+                print('  TPU API is not enabled or you don\'t have TPU access '
+                      f'to zone: {zone!r}.')
+            else:
+                print(f'  An error occurred: {error}')
+            # If error happens, fail early.
+            return []
+        time_to_sleep = backoff.current_backoff()
+        print(f'  Retry zone {zone!r} in {time_to_sleep} seconds...')
+        time.sleep(time_to_sleep)
+    print(f'ERROR: Failed to fetch TPUs for zone {zone!r}.')
+    return []
+
+
 def _get_tpu_for_zone(zone: str) -> 'pd.DataFrame':
     # Use hardcoded TPU V5 data as it is invisible in some zones.
     missing_tpus_df = pd.DataFrame(columns=[
@@ -526,19 +559,8 @@ def _get_tpu_for_zone(zone: str) -> 'pd.DataFrame':
     if zone in TPU_V5_MISSING_ZONES_DF:
         missing_tpus_df = TPU_V5_MISSING_ZONES_DF[zone]
     tpus = []
-    parent = f'projects/{project_id}/locations/{zone}'
-    tpus_request = tpu_client.projects().locations().acceleratorTypes().list(
-        parent=parent)
-    try:
-        tpus_response = tpus_request.execute()
-        for tpu in tpus_response['acceleratorTypes']:
-            tpus.append(tpu)
-    except gcp.http_error_exception() as error:
-        if error.resp.status == 403:
-            print('  TPU API is not enabled or you don\'t have TPU access '
-                  f'to zone: {zone!r}.')
-        else:
-            print(f'  An error occurred: {error}')
+    for tpu in _get_tpu_response_for_zone(zone):
+        tpus.append(tpu)
     new_tpus = []
     for tpu in tpus:
         tpu_name = tpu['type']

sky/clouds/service_catalog/gcp_catalog.py

@@ -9,6 +9,7 @@ from typing import Dict, List, Optional, Tuple
 from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
+from sky.clouds import GCP
 from sky.clouds.service_catalog import common
 from sky.utils import resources_utils
 from sky.utils import ux_utils
@@ -243,7 +244,6 @@ def get_default_instance_type(
         cpus: Optional[str] = None,
         memory: Optional[str] = None,
         disk_tier: Optional[resources_utils.DiskTier] = None) -> Optional[str]:
-    del disk_tier  # unused
     if cpus is None and memory is None:
         cpus = f'{_DEFAULT_NUM_VCPUS}+'
     if memory is None:
@@ -254,6 +254,12 @@ def get_default_instance_type(
         f'{family}-' for family in _DEFAULT_INSTANCE_FAMILY)
     df = _df[_df['InstanceType'].notna()]
     df = df[df['InstanceType'].str.startswith(instance_type_prefix)]
+
+    def _filter_disk_type(instance_type: str) -> bool:
+        valid, _ = GCP.check_disk_tier(instance_type, disk_tier)
+        return valid
+
+    df = df.loc[df['InstanceType'].apply(_filter_disk_type)]
     return common.get_instance_type_for_cpus_mem_impl(df, cpus,
                                                       memory_gb_or_ratio)
 

{skypilot_nightly-1.0.0.dev20240912.dist-info → skypilot_nightly-1.0.0.dev20240914.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240912
+Version: 1.0.0.dev20240914
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240912.dist-info → skypilot_nightly-1.0.0.dev20240914.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=-nmd-U8Khf-VwOCNfdHElLaHZL6aTC9_nIfaAOQUHYA,5588
+sky/__init__.py,sha256=EglV87JjMuUsTAewhIicWlTbUHe6hR3fYA0Yx6Mq0r4,5588
 sky/authentication.py,sha256=yvpdkXS9htf-X83DPCiSG3mQ41y0zV1BQ0YgOMgTYBU,20612
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
 sky/cli.py,sha256=2cOw3lXzRA-uLlEH-eK7N_1VmUpf1LR4Ztu-ZaKu3Is,201673
@@ -45,7 +45,7 @@ sky/clouds/cloud.py,sha256=PPk-Cbf1YbJT8bswcQLtPBtko02OWrRGJKkLzDpytTI,34858
 sky/clouds/cloud_registry.py,sha256=4yQMv-iBSgyN5aNL4Qxbn0JVE-dkVoEUIgj7S1z9S_Q,955
 sky/clouds/cudo.py,sha256=H4VyMo5wWGAv2MXZ3xsbWjlZA_cZYnt4ecNlTOOao8Y,13147
 sky/clouds/fluidstack.py,sha256=iOmoOx52yTrHKMzwBDaxFJCfNo79M61d5tj-Np24Lyc,12436
-sky/clouds/gcp.py,sha256=mgXRVBArN1rj4hFCNg7bIarv2UfqlHDzyjYg7JnzsKQ,51165
+sky/clouds/gcp.py,sha256=CrSsaSXd83tM78foKH9viBfW1cQsjve3aUQbshsqvDg,54033
 sky/clouds/ibm.py,sha256=M8QdjeSFlwssfoY2aOodxG4q5R3eT9K-4lTPDHYvEYI,21476
 sky/clouds/kubernetes.py,sha256=L_4ES8i4oR27NIalTW3CVgjMUdYScm1lS-GQCueN_js,22198
 sky/clouds/lambda_cloud.py,sha256=2Al3qCSl-I4iTi7pPPNXcbaLyVfCUgTl__vYBunLB6k,12439
@@ -54,7 +54,7 @@ sky/clouds/paperspace.py,sha256=lmUZPYAblaqiBmGQwCunccMiTF_dVA1o3vqY9Q_Nc28,1092
 sky/clouds/runpod.py,sha256=lstUC6f4JDhtcH9NfwkbpCJMmfmvMigoanhPXPbTYds,11540
 sky/clouds/scp.py,sha256=2KLTuNSMdBzK8CLwSesv7efOuiLidIMoyNG4AOt5Sqw,15870
 sky/clouds/vsphere.py,sha256=7eZFYIDtY5sX_ATr8h7kwwkY9t8Z-EYMJ9HCjoRBoxI,12309
-sky/clouds/service_catalog/__init__.py,sha256=8SJi4HBq7osm9fgo2y9KBEtI-prnMxhghAwYXZgIAZ8,14554
+sky/clouds/service_catalog/__init__.py,sha256=e0K-c64jQV9d6zly5OnIXMsYaZXs_Ko9osAbDaRlOOw,14743
 sky/clouds/service_catalog/aws_catalog.py,sha256=OURG9JeSGQ_zNO-E2KwDgUJeQuJEvMa7arzhf4Kza6M,12549
 sky/clouds/service_catalog/azure_catalog.py,sha256=VJi3yfhZy9Sc6UfcLAc8xIoTlUlUr090TODkCZyyHFw,7311
 sky/clouds/service_catalog/common.py,sha256=LED9Yzh1nsurvjCtqXGcsbUI8N-YFx05kazBgLF-HIQ,27256
@@ -62,7 +62,7 @@ sky/clouds/service_catalog/config.py,sha256=ylzqewdEBjDg4awvFek6ldYmFrnvD2bVGLZu
 sky/clouds/service_catalog/constants.py,sha256=ai2yOlsVqBnEpbxaEHXt61COsHBLwOfw6GZXntEPj7k,411
 sky/clouds/service_catalog/cudo_catalog.py,sha256=WPLIQ9NVvuKAS48KYujV6dQsyeppIcUCVXpapy13bbI,4335
 sky/clouds/service_catalog/fluidstack_catalog.py,sha256=c8MMTldG-q97MJ0zJymudQiOVQC_rxS7vqrZgLrgbQA,5038
-sky/clouds/service_catalog/gcp_catalog.py,sha256=-jcBQZWiPdaIi52A04Ium1eKdbNqmdEKOM0xUHtNiqc,24120
+sky/clouds/service_catalog/gcp_catalog.py,sha256=MHWq_-jqm68oNpK1i8AlJIGBkSKT-P6xX7DkpvqvpHU,24323
 sky/clouds/service_catalog/ibm_catalog.py,sha256=0dzjmXABFECzaAuIa0E6pVINhVK6-G6U52Mj-L45gK8,4472
 sky/clouds/service_catalog/kubernetes_catalog.py,sha256=AYuKWEfrOQP4g0yNscS_vq9zdPSYT0adCuV1VI5Po6k,7892
 sky/clouds/service_catalog/lambda_catalog.py,sha256=BAhUGqHj8aVe1zUhEQNO7bQUhcd9jAespGvPyQubTJY,5281
@@ -77,7 +77,7 @@ sky/clouds/service_catalog/data_fetchers/fetch_aws.py,sha256=Fcm9_IkmTBXM9gp5Voh
 sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=jsSVqbSbBIw_IYmO-y2u4co20AJ-JF713KFjUKdO_VA,12272
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=35nO_VaDOgp5W13kt_lIANSk_CNf7gBiZGJ5fGyZu6o,6808
-sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=8-b9sAMspMtNIgG0HpQr_rW45RNrqZrdVbDdbr3l7PQ,28251
+sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=2VebGjKkCckQZgOW3MBlfjFvA56U7eaIAh2q-6NYcL0,29070
 sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=B7H14so38zayuJGgUrD1PJYJKiVZHGnwH6JJop3F7o0,4918
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=SF_gTU74qg6L-DSWneCAbqP0lwZXaaDi5otiMIJbrw0,21462
 sky/clouds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -270,9 +270,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=Ma-N9a271fTfdgP5-8XIQL7KPf8IPUo-uY004PCdUFo,747
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240912.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240912.dist-info/METADATA,sha256=pbozO5GU_wBUbUHrc4vZ8vCqaHvJWDcVrN7qvNYobUs,19011
-skypilot_nightly-1.0.0.dev20240912.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-skypilot_nightly-1.0.0.dev20240912.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240912.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240912.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240914.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240914.dist-info/METADATA,sha256=173qFqRG_A3Vs_yjnG4I29fBIHpjiZrZyLQcloyuZ2U,19011
+skypilot_nightly-1.0.0.dev20240914.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+skypilot_nightly-1.0.0.dev20240914.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240914.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240914.dist-info/RECORD,,