skypilot-nightly 1.0.0.dev20241212__py3-none-any.whl → 1.0.0.dev20241214__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '1ed40e3174646ba835423b9308d8d6489f83b6bc'
+_SKYPILOT_COMMIT_SHA = '346646949beead6d93f793851e959da4b56246b0'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241212'
+__version__ = '1.0.0.dev20241214'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/clouds/service_catalog/constants.py

@@ -1,6 +1,6 @@
 """Constants used for service catalog."""
 HOSTED_CATALOG_DIR_URL = 'https://raw.githubusercontent.com/skypilot-org/skypilot-catalog/master/catalogs'  # pylint: disable=line-too-long
-CATALOG_SCHEMA_VERSION = 'v5'
+CATALOG_SCHEMA_VERSION = 'v6'
 CATALOG_DIR = '~/.sky/catalogs'
 ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vsphere', 'cudo', 'fluidstack',

sky/clouds/service_catalog/data_fetchers/fetch_gcp.py

@@ -476,9 +476,6 @@ def _get_gpus_for_zone(zone: str) -> 'pd.DataFrame':
             gpu_name = gpu_name.upper()
             if 'H100-80GB' in gpu_name:
                 gpu_name = 'H100'
-                if count != 8:
-                    # H100 only has 8 cards.
-                    continue
             if 'H100-MEGA-80GB' in gpu_name:
                 gpu_name = 'H100-MEGA'
                 if count != 8:

sky/clouds/service_catalog/gcp_catalog.py

@@ -97,6 +97,9 @@ _ACC_INSTANCE_TYPE_DICTS = {
         8: ['g2-standard-96'],
     },
     'H100': {
+        1: ['a3-highgpu-1g'],
+        2: ['a3-highgpu-2g'],
+        4: ['a3-highgpu-4g'],
         8: ['a3-highgpu-8g'],
     },
     'H100-MEGA': {
@@ -289,7 +292,9 @@ def get_instance_type_for_accelerator(
 
     if acc_name in _ACC_INSTANCE_TYPE_DICTS:
         df = _df[_df['InstanceType'].notna()]
-        instance_types = _ACC_INSTANCE_TYPE_DICTS[acc_name][acc_count]
+        instance_types = _ACC_INSTANCE_TYPE_DICTS[acc_name].get(acc_count, None)
+        if instance_types is None:
+            return None, []
         df = df[df['InstanceType'].isin(instance_types)]
 
         # Check the cpus and memory specified by the user.

sky/provision/kubernetes/instance.py

@@ -879,27 +879,62 @@ def _terminate_node(namespace: str, context: Optional[str],
                     pod_name: str) -> None:
     """Terminate a pod."""
     logger.debug('terminate_instances: calling delete_namespaced_pod')
-    try:
-        kubernetes.core_api(context).delete_namespaced_service(
-            pod_name, namespace, _request_timeout=config_lib.DELETION_TIMEOUT)
-        kubernetes.core_api(context).delete_namespaced_service(
-            f'{pod_name}-ssh',
-            namespace,
-            _request_timeout=config_lib.DELETION_TIMEOUT)
-    except kubernetes.api_exception():
-        pass
+
+    def _delete_k8s_resource_with_retry(delete_func: Callable,
+                                        resource_type: str,
+                                        resource_name: str) -> None:
+        """Helper to delete Kubernetes resources with 404 handling and retries.
+
+        Args:
+            delete_func: Function to call to delete the resource
+            resource_type: Type of resource being deleted (e.g. 'service'),
+                used in logging
+            resource_name: Name of the resource being deleted, used in logging
+        """
+        max_retries = 3
+        retry_delay = 5  # seconds
+
+        for attempt in range(max_retries):
+            try:
+                delete_func()
+                return
+            except kubernetes.api_exception() as e:
+                if e.status == 404:
+                    logger.warning(
+                        f'terminate_instances: Tried to delete {resource_type} '
+                        f'{resource_name}, but the {resource_type} was not '
+                        'found (404).')
+                    return
+                elif attempt < max_retries - 1:
+                    logger.warning(f'terminate_instances: Failed to delete '
+                                   f'{resource_type} {resource_name} (attempt '
+                                   f'{attempt + 1}/{max_retries}). Error: {e}. '
+                                   f'Retrying in {retry_delay} seconds...')
+                    time.sleep(retry_delay)
+                else:
+                    raise
+
+    # Delete services for the pod
+    for service_name in [pod_name, f'{pod_name}-ssh']:
+        _delete_k8s_resource_with_retry(
+            delete_func=lambda name=service_name: kubernetes.core_api(
+                context).delete_namespaced_service(name=name,
+                                                   namespace=namespace,
+                                                   _request_timeout=config_lib.
+                                                   DELETION_TIMEOUT),
+            resource_type='service',
+            resource_name=service_name)
+
     # Note - delete pod after all other resources are deleted.
     # This is to ensure there are no leftover resources if this down is run
     # from within the pod, e.g., for autodown.
-    try:
-        kubernetes.core_api(context).delete_namespaced_pod(
-            pod_name, namespace, _request_timeout=config_lib.DELETION_TIMEOUT)
-    except kubernetes.api_exception() as e:
-        if e.status == 404:
-            logger.warning('terminate_instances: Tried to delete pod '
-                           f'{pod_name}, but the pod was not found (404).')
-        else:
-            raise
+    _delete_k8s_resource_with_retry(
+        delete_func=lambda: kubernetes.core_api(context).delete_namespaced_pod(
+            name=pod_name,
+            namespace=namespace,
+            _request_timeout=config_lib.DELETION_TIMEOUT),
+        resource_type='pod',
+        resource_name=pod_name)
 
 
 def terminate_instances(

sky/sky_logging.py

@@ -15,6 +15,7 @@ _show_logging_prefix = (env_options.Options.SHOW_DEBUG_INFO.get() or
                         not env_options.Options.MINIMIZE_LOGGING.get())
 _FORMAT = '%(levelname).1s %(asctime)s %(filename)s:%(lineno)d] %(message)s'
 _DATE_FORMAT = '%m-%d %H:%M:%S'
+_SENSITIVE_LOGGER = ['sky.provisioner', 'sky.optimizer']
 
 
 class NewLineFormatter(logging.Formatter):
@@ -75,6 +76,23 @@ def _setup_logger():
     # Setting this will avoid the message
     # being propagated to the parent logger.
     _root_logger.propagate = False
+    if env_options.Options.SUPPRESS_SENSITIVE_LOG.get():
+        # If the sensitive log is enabled, we reinitialize a new handler
+        # and force set the level to INFO to suppress the debug logs
+        # for certain loggers.
+        for logger_name in _SENSITIVE_LOGGER:
+            logger = logging.getLogger(logger_name)
+            handler_to_logger = RichSafeStreamHandler(sys.stdout)
+            handler_to_logger.flush = sys.stdout.flush  # type: ignore
+            logger.addHandler(handler_to_logger)
+            logger.setLevel(logging.INFO)
+            if _show_logging_prefix:
+                handler_to_logger.setFormatter(FORMATTER)
+            else:
+                handler_to_logger.setFormatter(NO_PREFIX_FORMATTER)
+            # Do not propagate to the parent logger to avoid parent
+            # logger printing the logs.
+            logger.propagate = False
 
 
 def reload_logger():

sky/utils/env_options.py

@@ -11,6 +11,7 @@ class Options(enum.Enum):
     SHOW_DEBUG_INFO = ('SKYPILOT_DEBUG', False)
     DISABLE_LOGGING = ('SKYPILOT_DISABLE_USAGE_COLLECTION', False)
     MINIMIZE_LOGGING = ('SKYPILOT_MINIMIZE_LOGGING', True)
+    SUPPRESS_SENSITIVE_LOG = ('SKYPILOT_SUPPRESS_SENSITIVE_LOG', False)
     # Internal: this is used to skip the cloud user identity check, which is
     # used to protect cluster operations in a multi-identity scenario.
     # Currently, this is only used in the job and serve controller, as there

{skypilot_nightly-1.0.0.dev20241212.dist-info → skypilot_nightly-1.0.0.dev20241214.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241212
+Version: 1.0.0.dev20241214
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241212.dist-info → skypilot_nightly-1.0.0.dev20241214.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=BSUiTgI24KPNL66tZ50gHcfp58wm0j6iLb_aP5dUT2g,5944
+sky/__init__.py,sha256=952AxZEJvtF5tit7QrsMn94iTibGwYuSM1gkl_VxM64,5944
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=kACHmiZgWgRpYd1wx1ofbXRMErfMcFmWrkw4a9NxYrY,20988
 sky/check.py,sha256=D3Y3saIFAYVvPxuBHnVgJEO0fUVDxgjwuMBaO-D778k,9472
@@ -11,7 +11,7 @@ sky/execution.py,sha256=dpbk1kGRkGHT0FCJKGvjqeV3qIGEN2K20NDZbVrcAvI,28483
 sky/global_user_state.py,sha256=m2LJsXkh8eAvvz0ADnSP6idfYWZTA_Xi3uxwR3DrJxo,30241
 sky/optimizer.py,sha256=GjvKQIBtY3NlULzau_9tfa7V2KYVJRrmNrjKVIWCPIQ,59753
 sky/resources.py,sha256=4T-zQK0OSLC1z5-sLFluEQJ_Y8CAJX3jb4ZSPedwy1s,70352
-sky/sky_logging.py,sha256=oLmTmwkuucIto3LHXLJfMcyRpYSkmZAZa5XzQPA5IHk,4434
+sky/sky_logging.py,sha256=SSHibLotMOQKYX_XKj4Va2PTt6dKGZNCWvGm3FuM-Sc,5373
 sky/skypilot_config.py,sha256=E3g65cX3P3dT9b5N0GgFBG6yB0FXwIGpisKoozmJmWU,9094
 sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
 sky/task.py,sha256=KDsTBIxYpkCOPHv3_ei5H3LDMiGHvDeS9_2HeL6yyLA,49766
@@ -60,10 +60,10 @@ sky/clouds/service_catalog/aws_catalog.py,sha256=j33lNC5GXWK6CiGWZORCnumGlRODmCA
 sky/clouds/service_catalog/azure_catalog.py,sha256=5Q51x_WEKvQ2YSgJvZHRH3URlbwIstYuwpjaWW_wJlw,8149
 sky/clouds/service_catalog/common.py,sha256=fCGabkqCMmzNaeT4_--V4K5GsGdf8v_pzYHGYNRDot4,27709
 sky/clouds/service_catalog/config.py,sha256=ylzqewdEBjDg4awvFek6ldYmFrnvD2bVGLZuLPvEVYA,1793
-sky/clouds/service_catalog/constants.py,sha256=ai2yOlsVqBnEpbxaEHXt61COsHBLwOfw6GZXntEPj7k,411
+sky/clouds/service_catalog/constants.py,sha256=u_SjIr3gJI6P-xpRgNAzZG2Z4M5UfE1PL7z4atfQncE,411
 sky/clouds/service_catalog/cudo_catalog.py,sha256=V_takvL6dWTGQaTLCEvjKIotCDPnMujiNUZ87kZKGVI,4673
 sky/clouds/service_catalog/fluidstack_catalog.py,sha256=21-cvrYEYTIi7n3ZNF2e7_0QX-PF4BkhlVJUWQOvKrY,5059
-sky/clouds/service_catalog/gcp_catalog.py,sha256=v_5fsB3dB9oD8U7lBKnCe5ii6AUWEOiQjNarMnU_qLA,24379
+sky/clouds/service_catalog/gcp_catalog.py,sha256=jJEfWjZ4ItsE657LjIf9mruJVZERFegCD5Qtu20AFNc,24542
 sky/clouds/service_catalog/ibm_catalog.py,sha256=1iK0KvbI82U7sySb7chr-qm_16x3tTnZ6nIo7o76ouc,4493
 sky/clouds/service_catalog/kubernetes_catalog.py,sha256=D0DvhVlK6Z6HJcZHPOWqRNAbXgFaQOKUnS_xkmqzukA,12550
 sky/clouds/service_catalog/lambda_catalog.py,sha256=2R-ccu63BbdvO6X80MtxiniA-jLewXb6I0Ye1rYD9fY,5302
@@ -78,7 +78,7 @@ sky/clouds/service_catalog/data_fetchers/fetch_aws.py,sha256=ro2zazdkDF6z9bE7QFy
 sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=K5jyfCAR5d-Hg78tDhmqpz0DQl79ndCW1ZRhBDLcBdM,12796
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=yKuAFbjBRNz_e2RNNDT_aHHAuKQ86Ac7GKgIie5O6Pg,7273
-sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=VrTTkMF5AjiplfDmvPBW-otR3oXGU3-oFouVMfIua4Q,33447
+sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=4ltAlVWFDcLMRoUQHT8NhkYy6ujwYOueYCeUVp37Rz0,33342
 sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=MN54h0CAGPHQAeF2eTmuESq3b0-d1kDARRUM6OkivCk,4962
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=SF_gTU74qg6L-DSWneCAbqP0lwZXaaDi5otiMIJbrw0,21462
 sky/clouds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -137,7 +137,7 @@ sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHa
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
 sky/provision/kubernetes/config.py,sha256=WEKcFXXhe89bLGAvoMiBvTDxdxkpTIA6ezrj2vmzldc,29072
-sky/provision/kubernetes/instance.py,sha256=2zd_Z09amOsi0vPZjQYMJCkCWbN2YecMLL9HkmUuPrM,48414
+sky/provision/kubernetes/instance.py,sha256=ryJHnFvE2WH2zC4sBC7DjH0yt1ScTC7Sqx8xJXzKI-c,50024
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
 sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMshsvsYmnMUcq8k,11399
 sky/provision/kubernetes/utils.py,sha256=cnhmVcy8ri8iKMr404iugxBR2gQIXZiJVCxXwi3vglc,102225
@@ -256,7 +256,7 @@ sky/utils/control_master_utils.py,sha256=90hnxiAUP20gbJ9e3MERh7rb04ZO_I3LsljNjR2
 sky/utils/controller_utils.py,sha256=t6PkhrGt7fRUzRqdSox0a-825Cy9b-xACk64exr5HRc,40698
 sky/utils/dag_utils.py,sha256=pVX3lGDDcYTcGoH_1jEWzl9767Y4mwlIEYIzoyHO6gM,6105
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
-sky/utils/env_options.py,sha256=3oAaUPxowL6vI2XmxXrH56V7Myj9IJWsL-MXFmRFVdI,1294
+sky/utils/env_options.py,sha256=E5iwRFBUY2Iq6e0y0c1Mv5OSQ4MRNdk0-p38xUyVerc,1366
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
 sky/utils/log_utils.py,sha256=oZYF45uC7GFjAqO-Je-aiX6zhtq91TP-KKaIbQNF-jY,14024
 sky/utils/resources_utils.py,sha256=Xqi7gxPYw2y5wl5okUI5zx5LEij0hJF_V3Zi8q7TXYg,7890
@@ -279,9 +279,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7Z
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241212.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241212.dist-info/METADATA,sha256=JusT-4ZZvxLeBa4Wrt9o6g8PReUHuWS1kSym2uX3hDI,20319
-skypilot_nightly-1.0.0.dev20241212.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-skypilot_nightly-1.0.0.dev20241212.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241212.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241212.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241214.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241214.dist-info/METADATA,sha256=BDAqiSBeCr0fJIBAs7J2UTyWswclFpu5N3Crv5LO1hk,20319
+skypilot_nightly-1.0.0.dev20241214.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+skypilot_nightly-1.0.0.dev20241214.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241214.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241214.dist-info/RECORD,,