skypilot-nightly 1.0.0.dev20250324__tar.gz → 1.0.0.dev20250326__tar.gz

{skypilot_nightly-1.0.0.dev20250324/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250326}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250324
+Version: 1.0.0.dev20250326
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -193,6 +193,10 @@ Dynamic: summary
     <img alt="Downloads" src="https://img.shields.io/pypi/dm/skypilot">
   </a>
 
+  <a href="https://buildkite.com/skypilot-1/full-smoke-tests-run">
+    <img alt="Smoke Tests" src="https://badge.buildkite.com/d3aa9d2370e4a9ac4fb5e210381f955082a63a9a46673b197a.svg?theme=github&branch=master">
+  </a>
+
 </p>
 
 <h3 align="center">

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/README.md

@@ -22,6 +22,10 @@
     <img alt="Downloads" src="https://img.shields.io/pypi/dm/skypilot">
   </a>
 
+  <a href="https://buildkite.com/skypilot-1/full-smoke-tests-run">
+    <img alt="Smoke Tests" src="https://badge.buildkite.com/d3aa9d2370e4a9ac4fb5e210381f955082a63a9a46673b197a.svg?theme=github&branch=master">
+  </a>
+
 </p>
 
 <h3 align="center">

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '633e16611f2f858dc27c9eae2f410811e0bc714c'
+_SKYPILOT_COMMIT_SHA = '4f8721ab403668198ffd61297874c4c365e642b7'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250324'
+__version__ = '1.0.0.dev20250326'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/sky/backends/cloud_vm_ray_backend.py

@@ -3632,8 +3632,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             # should be higher priority than the cluster requests, and we should
             # release the lock from other requests.
             exclude_request_to_kill = 'sky.down' if terminate else 'sky.stop'
-            requests_lib.kill_cluster_requests(handle.cluster_name,
-                                               exclude_request_to_kill)
+            try:
+                # TODO(zhwu): we should get rid of this when it is being called
+                # internally without involving an API server, e.g., when a
+                # controller is trying to terminate a cluster.
+                requests_lib.kill_cluster_requests(handle.cluster_name,
+                                                   exclude_request_to_kill)
+            except Exception as e:  # pylint: disable=broad-except
+                # We allow the failure to kill other launch requests, because
+                # it is not critical to the cluster teardown.
+                logger.warning(
+                    'Failed to kill other launch requests for the '
+                    f'cluster {handle.cluster_name}: '
+                    f'{common_utils.format_exception(e, use_bracket=True)}')
             try:
                 with filelock.FileLock(
                         lock_path,
@@ -4030,8 +4041,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # the cluster is terminated/stopped. Otherwise, it will be quite
         # confusing to see the cluster restarted immediately after it is
         # terminated/stopped, when there is a pending launch request.
-        requests_lib.kill_cluster_requests(handle.cluster_name,
-                                           exclude_request_to_kill)
+        try:
+            # TODO(zhwu): we should get rid of this when it is being called
+            # internally without involving an API server, e.g., when a
+            # controller is trying to terminate a cluster.
+            requests_lib.kill_cluster_requests(handle.cluster_name,
+                                               exclude_request_to_kill)
+        except Exception as e:  # pylint: disable=broad-except
+            # We allow the failure to kill other launch requests, because
+            # it is not critical to the cluster teardown.
+            logger.warning(
+                'Failed to kill other launch requests for the '
+                f'cluster {handle.cluster_name}: '
+                f'{common_utils.format_exception(e, use_bracket=True)}')
         cluster_status_fetched = False
         if refresh_cluster_status:
             try:
@@ -4358,7 +4380,19 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # If cluster_yaml is None, the cluster should ensured to be terminated,
         # so we don't need to do the double check.
         if handle.cluster_yaml is not None:
-            _detect_abnormal_non_terminated_nodes(handle)
+            try:
+                _detect_abnormal_non_terminated_nodes(handle)
+            except exceptions.ClusterStatusFetchingError as e:
+                if purge:
+                    msg = common_utils.format_exception(e, use_bracket=True)
+                    logger.warning(
+                        'Failed abnormal non-terminated nodes cleanup. '
+                        'Skipping and cleaning up as purge is set. '
+                        f'Details: {msg}')
+                    logger.debug(f'Full exception details: {msg}',
+                                 exc_info=True)
+                else:
+                    raise
 
         if not terminate or remove_from_db:
             global_user_state.remove_cluster(handle.cluster_name,

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/sky/provision/gcp/constants.py

@@ -182,13 +182,7 @@ VM_MINIMAL_PERMISSIONS = [
 
 STORAGE_MINIMAL_PERMISSIONS = [
     'storage.buckets.create',
-    'storage.buckets.get',
     'storage.buckets.delete',
-    'storage.objects.create',
-    'storage.objects.update',
-    'storage.objects.delete',
-    'storage.objects.get',
-    'storage.objects.list',
 ]
 
 # Permissions implied by GCP built-in roles. We hardcode these here, as we

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/sky/server/requests/requests.py

@@ -396,7 +396,6 @@ pathlib.Path(_DB_PATH).parents[0].mkdir(parents=True, exist_ok=True)
 
 
 def create_table(cursor, conn):
-    del conn
     # Enable WAL mode to avoid locking issues.
     # See: issue #1441 and PR #1509
     # https://github.com/microsoft/WSL/issues/2395
@@ -428,6 +427,9 @@ def create_table(cursor, conn):
         {COL_USER_ID} TEXT,
         {COL_STATUS_MSG} TEXT)""")
 
+    db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_STATUS_MSG,
+                                 'TEXT')
+
 
 _DB = None
 

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326}/sky/utils/rich_utils.py

@@ -144,15 +144,28 @@ class _RevertibleStatus:
         return _statuses[self.status_type]
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        global _status_nesting_level
-        _status_nesting_level -= 1
-        if _status_nesting_level <= 0:
-            _status_nesting_level = 0
-            if _statuses[self.status_type] is not None:
-                _statuses[self.status_type].__exit__(exc_type, exc_val, exc_tb)
-                _statuses[self.status_type] = None
-        else:
-            _statuses[self.status_type].update(self.previous_message)
+        # We use the same lock with the `safe_logger` to avoid the following 2
+        # raice conditions. We refer loggers in another thread as "thread
+        # logger" hereafter.
+        # 1. When a thread logger stopped the status in `safe_logger`, and
+        #    here we exit the status and set it to None. Then the thread logger
+        #    will raise an error when it tries to restart the status.
+        # 2. When a thread logger stopped the status in `safe_logger`, and
+        #    here we exit the status and entered a new one. Then the thread
+        #    logger will raise an error when it tries to restart the old status,
+        #    since only one LiveStatus can be started at the same time.
+        # Please refer to #4995 for more information.
+        with _logging_lock:
+            global _status_nesting_level
+            _status_nesting_level -= 1
+            if _status_nesting_level <= 0:
+                _status_nesting_level = 0
+                if _statuses[self.status_type] is not None:
+                    _statuses[self.status_type].__exit__(
+                        exc_type, exc_val, exc_tb)
+                    _statuses[self.status_type] = None
+            else:
+                _statuses[self.status_type].update(self.previous_message)
 
     def update(self, *args, **kwargs):
         _statuses[self.status_type].update(*args, **kwargs)

{skypilot_nightly-1.0.0.dev20250324 → skypilot_nightly-1.0.0.dev20250326/skypilot_nightly.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250324
+Version: 1.0.0.dev20250326
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -193,6 +193,10 @@ Dynamic: summary
     <img alt="Downloads" src="https://img.shields.io/pypi/dm/skypilot">
   </a>
 
+  <a href="https://buildkite.com/skypilot-1/full-smoke-tests-run">
+    <img alt="Smoke Tests" src="https://badge.buildkite.com/d3aa9d2370e4a9ac4fb5e210381f955082a63a9a46673b197a.svg?theme=github&branch=master">
+  </a>
+
 </p>
 
 <h3 align="center">