skypilot-nightly 1.0.0.dev20250324__py3-none-any.whl → 1.0.0.dev20250326__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '633e16611f2f858dc27c9eae2f410811e0bc714c'
+_SKYPILOT_COMMIT_SHA = '4f8721ab403668198ffd61297874c4c365e642b7'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250324'
+__version__ = '1.0.0.dev20250326'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -3632,8 +3632,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             # should be higher priority than the cluster requests, and we should
             # release the lock from other requests.
             exclude_request_to_kill = 'sky.down' if terminate else 'sky.stop'
-            requests_lib.kill_cluster_requests(handle.cluster_name,
-                                               exclude_request_to_kill)
+            try:
+                # TODO(zhwu): we should get rid of this when it is being called
+                # internally without involving an API server, e.g., when a
+                # controller is trying to terminate a cluster.
+                requests_lib.kill_cluster_requests(handle.cluster_name,
+                                                   exclude_request_to_kill)
+            except Exception as e:  # pylint: disable=broad-except
+                # We allow the failure to kill other launch requests, because
+                # it is not critical to the cluster teardown.
+                logger.warning(
+                    'Failed to kill other launch requests for the '
+                    f'cluster {handle.cluster_name}: '
+                    f'{common_utils.format_exception(e, use_bracket=True)}')
             try:
                 with filelock.FileLock(
                         lock_path,
@@ -4030,8 +4041,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # the cluster is terminated/stopped. Otherwise, it will be quite
         # confusing to see the cluster restarted immediately after it is
         # terminated/stopped, when there is a pending launch request.
-        requests_lib.kill_cluster_requests(handle.cluster_name,
-                                           exclude_request_to_kill)
+        try:
+            # TODO(zhwu): we should get rid of this when it is being called
+            # internally without involving an API server, e.g., when a
+            # controller is trying to terminate a cluster.
+            requests_lib.kill_cluster_requests(handle.cluster_name,
+                                               exclude_request_to_kill)
+        except Exception as e:  # pylint: disable=broad-except
+            # We allow the failure to kill other launch requests, because
+            # it is not critical to the cluster teardown.
+            logger.warning(
+                'Failed to kill other launch requests for the '
+                f'cluster {handle.cluster_name}: '
+                f'{common_utils.format_exception(e, use_bracket=True)}')
         cluster_status_fetched = False
         if refresh_cluster_status:
             try:
@@ -4358,7 +4380,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # If cluster_yaml is None, the cluster should ensured to be terminated,
         # so we don't need to do the double check.
         if handle.cluster_yaml is not None:
-            _detect_abnormal_non_terminated_nodes(handle)
+            try:
+                _detect_abnormal_non_terminated_nodes(handle)
+            except exceptions.ClusterStatusFetchingError as e:
+                if purge:
+                    msg = common_utils.format_exception(e, use_bracket=True)
+                    logger.warning(
+                        'Failed abnormal non-terminated nodes cleanup. '
+                        'Skipping and cleaning up as purge is set. '
+                        f'Details: {msg}')
+                    logger.debug(f'Full exception details: {msg}',
+                                 exc_info=True)
+                else:
+                    raise
 
         if not terminate or remove_from_db:
             global_user_state.remove_cluster(handle.cluster_name,

sky/provision/gcp/constants.py

@@ -182,13 +182,7 @@ VM_MINIMAL_PERMISSIONS = [
 
 STORAGE_MINIMAL_PERMISSIONS = [
     'storage.buckets.create',
-    'storage.buckets.get',
     'storage.buckets.delete',
-    'storage.objects.create',
-    'storage.objects.update',
-    'storage.objects.delete',
-    'storage.objects.get',
-    'storage.objects.list',
 ]
 
 # Permissions implied by GCP built-in roles. We hardcode these here, as we

sky/server/requests/requests.py

@@ -396,7 +396,6 @@ pathlib.Path(_DB_PATH).parents[0].mkdir(parents=True, exist_ok=True)
 
 
 def create_table(cursor, conn):
-    del conn
     # Enable WAL mode to avoid locking issues.
     # See: issue #1441 and PR #1509
     # https://github.com/microsoft/WSL/issues/2395
@@ -428,6 +427,9 @@ def create_table(cursor, conn):
         {COL_USER_ID} TEXT,
         {COL_STATUS_MSG} TEXT)""")
 
+    db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_STATUS_MSG,
+                                 'TEXT')
+
 
 _DB = None
 

sky/utils/rich_utils.py

@@ -144,15 +144,28 @@ class _RevertibleStatus:
         return _statuses[self.status_type]
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        global _status_nesting_level
-        _status_nesting_level -= 1
-        if _status_nesting_level <= 0:
-            _status_nesting_level = 0
-            if _statuses[self.status_type] is not None:
-                _statuses[self.status_type].__exit__(exc_type, exc_val, exc_tb)
-                _statuses[self.status_type] = None
-        else:
-            _statuses[self.status_type].update(self.previous_message)
+        # We use the same lock with the `safe_logger` to avoid the following 2
+        # raice conditions. We refer loggers in another thread as "thread
+        # logger" hereafter.
+        # 1. When a thread logger stopped the status in `safe_logger`, and
+        #    here we exit the status and set it to None. Then the thread logger
+        #    will raise an error when it tries to restart the status.
+        # 2. When a thread logger stopped the status in `safe_logger`, and
+        #    here we exit the status and entered a new one. Then the thread
+        #    logger will raise an error when it tries to restart the old status,
+        #    since only one LiveStatus can be started at the same time.
+        # Please refer to #4995 for more information.
+        with _logging_lock:
+            global _status_nesting_level
+            _status_nesting_level -= 1
+            if _status_nesting_level <= 0:
+                _status_nesting_level = 0
+                if _statuses[self.status_type] is not None:
+                    _statuses[self.status_type].__exit__(
+                        exc_type, exc_val, exc_tb)
+                    _statuses[self.status_type] = None
+            else:
+                _statuses[self.status_type].update(self.previous_message)
 
     def update(self, *args, **kwargs):
         _statuses[self.status_type].update(*args, **kwargs)

{skypilot_nightly-1.0.0.dev20250324.dist-info → skypilot_nightly-1.0.0.dev20250326.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250324
+Version: 1.0.0.dev20250326
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -193,6 +193,10 @@ Dynamic: summary
     <img alt="Downloads" src="https://img.shields.io/pypi/dm/skypilot">
   </a>
 
+  <a href="https://buildkite.com/skypilot-1/full-smoke-tests-run">
+    <img alt="Smoke Tests" src="https://badge.buildkite.com/d3aa9d2370e4a9ac4fb5e210381f955082a63a9a46673b197a.svg?theme=github&branch=master">
+  </a>
+
 </p>
 
 <h3 align="center">

{skypilot_nightly-1.0.0.dev20250324.dist-info → skypilot_nightly-1.0.0.dev20250326.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=FZLm0eOUqmRoDKokxdSzkA-KffEhUH6hS9QYWsjROyY,6428
+sky/__init__.py,sha256=ZQgnwZFx1OI8QGgvePoEaTVl6x9kc5Opq9Vsu0qdHWQ,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=QWybUjX-O6TXzCDSu2vtSgyeQb-OY9B6gDpT-jwYK9I,22700
 sky/check.py,sha256=iMGuM7yjUPRgDHc13Pf1_LSybBqIexq-6aXfnVcaI54,15898
@@ -34,7 +34,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
 sky/backends/backend_utils.py,sha256=ndY4IPs1F9QovyiKAnB1FNYGWm52_ylwf_K7wY50cv0,134922
-sky/backends/cloud_vm_ray_backend.py,sha256=zoLk0j6o84fZpGOupRv5mOpDKRf7coPrMMeHCLHdE8w,246263
+sky/backends/cloud_vm_ray_backend.py,sha256=8uMkhOzJqeHDkh1212RUe8HrT5fLYTdxe-beTXgygR8,248154
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=meypuMaygSXXjGdXfq6dhWl-OrpAybg9KVRoup4D0wU,9098
@@ -155,7 +155,7 @@ sky/provision/fluidstack/fluidstack_utils.py,sha256=NdhQcwhIPGT21g7lQR-t6j-1zTA_
 sky/provision/fluidstack/instance.py,sha256=TCGLojd5mEuEaUQ1BnmRvXMOSSBjltyf7dhPG3OLdgQ,13787
 sky/provision/gcp/__init__.py,sha256=zlgjR2JoaGD7sStGStMRu9bJ62f-8NKEIyb-bFHBlzM,528
 sky/provision/gcp/config.py,sha256=kU357o4tCTuQ2e0Gind5q_tC0kFRW2tch-NQ2DnwN9Q,33319
-sky/provision/gcp/constants.py,sha256=KbnAupsKCp_mNNOas62-yiCJcqYoaer7iMljFq-CPfk,7739
+sky/provision/gcp/constants.py,sha256=G4Q5zUantKFEnkt9DG1-in3q_KtD7JObRFTw1kEFDWA,7567
 sky/provision/gcp/instance.py,sha256=47jDHLbIAI5M1MZIQTCiKGfwc9QzPOyjApkShqBRczE,25035
 sky/provision/gcp/instance_utils.py,sha256=T0AVT8lMn128snPp3MvqmhXOihlZSC8-c1QpgYT4_FA,71377
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
@@ -239,7 +239,7 @@ sky/server/requests/event_loop.py,sha256=OhpPbuce65bbjpGRlcJa78AVnYSm08SzFKt70yp
 sky/server/requests/executor.py,sha256=BNJqkTQ3swYeRO5YVW-dTmobL2CYnDDf_m-kY7__n40,21684
 sky/server/requests/payloads.py,sha256=6egfR6QSpd1WZP7K0joCQOrA0K6n7iTQbKL02T_RCOM,16494
 sky/server/requests/preconditions.py,sha256=ipxIb_3JXG6S3-ymcOdqQNb7VDvoPqADxu9ZK7-nQWc,7179
-sky/server/requests/requests.py,sha256=Sys2rg22rIXn7SrHfKzDVuTjBdRlm5oZk58u1UmS6JA,21231
+sky/server/requests/requests.py,sha256=9ovdQE-zv_Mvc6IbGATHVyQlOxSKjg_OankZbgDVGeE,21338
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/queues/mp_queue.py,sha256=_7AFas__0b1L8e7Bwy4lu0VYU18R85YwMlDHPhQCfh0,2998
 sky/server/requests/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -326,7 +326,7 @@ sky/utils/message_utils.py,sha256=zi2Z7PEX6Xq_zvho-aEZe_J7UvpKOLdVDdGAcipRQPU,26
 sky/utils/registry.py,sha256=sH_VBupeczMHJIQMXPFv9jNMqE_ZF1ytOUcDFGdHjxA,4132
 sky/utils/resources_utils.py,sha256=URp6OS9B9nc9tIB5ibZCgGK4XSABmI4kRG0wOM6qgvs,7774
 sky/utils/rich_console_utils.py,sha256=wPvAlshaFHuMZSjiDnaK3OSBppZLBjAn-lj7AvxNBQk,553
-sky/utils/rich_utils.py,sha256=7QknWkr5uRUawabEYM41Lf5CBHLdPGpTAdOfmy40_S4,12126
+sky/utils/rich_utils.py,sha256=PK5nVUFRMvg7ngYoIxZU6Ldcyiy3VfXriOLgNFkv53g,12980
 sky/utils/schemas.py,sha256=KJCHrn1nMZ3XqzddWuu_nFQoRQw01cZh9qh19OrRtps,30145
 sky/utils/status_lib.py,sha256=zn_MSuRYQdNKF8pnFOGQ54X_s_R7dyqWS6Q3a9zENw8,1512
 sky/utils/subprocess_utils.py,sha256=yM2WumV49gSKuZs0v6E3R8XKl5Q9b6veIzi6us5ORU8,15927
@@ -348,9 +348,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=NtfbovAECN2A_R8x4FiNPaBVPWeuwaOVuuGsnVOSk2g,10231
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250324.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250324.dist-info/METADATA,sha256=wJClvsEDKwrcbhu1y2kadg0B6iGbfknPR8dJAaJjKpA,18438
-skypilot_nightly-1.0.0.dev20250324.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-skypilot_nightly-1.0.0.dev20250324.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250324.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250324.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250326.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250326.dist-info/METADATA,sha256=V5UlwHyFmeBcPPaUBEzuhCvdyoXArUCqy4zdQrckbMg,18657
+skypilot_nightly-1.0.0.dev20250326.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250326.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250326.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250326.dist-info/RECORD,,

{skypilot_nightly-1.0.0.dev20250324.dist-info → skypilot_nightly-1.0.0.dev20250326.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any