skypilot-nightly 1.0.0.dev20251027__py3-none-any.whl → 1.0.0.dev20251029__py3-none-any.whl

sky/data/storage.py

@@ -23,6 +23,7 @@ from sky import skypilot_config
 from sky.adaptors import aws
 from sky.adaptors import azure
 from sky.adaptors import cloudflare
+from sky.adaptors import coreweave
 from sky.adaptors import gcp
 from sky.adaptors import ibm
 from sky.adaptors import nebius
@@ -62,6 +63,7 @@ STORE_ENABLED_CLOUDS: List[str] = [
     str(clouds.OCI()),
     str(clouds.Nebius()),
     cloudflare.NAME,
+    coreweave.NAME,
 ]
 
 # Maximum number of concurrent rsync upload processes
@@ -93,6 +95,12 @@ def get_cached_enabled_storage_cloud_names_or_refresh(
     r2_is_enabled, _ = cloudflare.check_storage_credentials()
     if r2_is_enabled:
         enabled_clouds.append(cloudflare.NAME)
+
+    # Similarly, handle CoreWeave storage credentials
+    coreweave_is_enabled, _ = coreweave.check_storage_credentials()
+    if coreweave_is_enabled:
+        enabled_clouds.append(coreweave.NAME)
+
     if raise_if_no_cloud_access and not enabled_clouds:
         raise exceptions.NoCloudAccessError(
             'No cloud access available for storage. '
@@ -126,6 +134,7 @@ class StoreType(enum.Enum):
     IBM = 'IBM'
     OCI = 'OCI'
     NEBIUS = 'NEBIUS'
+    COREWEAVE = 'COREWEAVE'
     VOLUME = 'VOLUME'
 
     @classmethod
@@ -883,7 +892,7 @@ class Storage(object):
                             f'{source} in the file_mounts section of your YAML')
                 is_local_source = True
             elif split_path.scheme in [
-                    's3', 'gs', 'https', 'r2', 'cos', 'oci', 'nebius'
+                    's3', 'gs', 'https', 'r2', 'cos', 'oci', 'nebius', 'cw'
             ]:
                 is_local_source = False
                 # Storage mounting does not support mounting specific files from
@@ -908,7 +917,8 @@ class Storage(object):
                 with ux_utils.print_exception_no_traceback():
                     raise exceptions.StorageSourceError(
                         f'Supported paths: local, s3://, gs://, https://, '
-                        f'r2://, cos://, oci://, nebius://. Got: {source}')
+                        f'r2://, cos://, oci://, nebius://, cw://. '
+                        f'Got: {source}')
         return source, is_local_source
 
     def _validate_storage_spec(self, name: Optional[str]) -> None:
@@ -923,7 +933,16 @@ class Storage(object):
             """
             prefix = name.split('://')[0]
             prefix = prefix.lower()
-            if prefix in ['s3', 'gs', 'https', 'r2', 'cos', 'oci', 'nebius']:
+            if prefix in [
+                    's3',
+                    'gs',
+                    'https',
+                    'r2',
+                    'cos',
+                    'oci',
+                    'nebius',
+                    'cw',
+            ]:
                 with ux_utils.print_exception_no_traceback():
                     raise exceptions.StorageNameError(
                         'Prefix detected: `name` cannot start with '
@@ -1062,6 +1081,12 @@ class Storage(object):
                         source=self.source,
                         sync_on_reconstruction=self.sync_on_reconstruction,
                         _bucket_sub_path=self._bucket_sub_path)
+                elif s_type == StoreType.COREWEAVE:
+                    store = CoreWeaveStore.from_metadata(
+                        s_metadata,
+                        source=self.source,
+                        sync_on_reconstruction=self.sync_on_reconstruction,
+                        _bucket_sub_path=self._bucket_sub_path)
                 else:
                     with ux_utils.print_exception_no_traceback():
                         raise ValueError(f'Unknown store type: {s_type}')
@@ -1417,6 +1442,7 @@ class S3CompatibleConfig:
     aws_profile: Optional[str] = None
     get_endpoint_url: Optional[Callable[[], str]] = None
     credentials_file: Optional[str] = None
+    config_file: Optional[str] = None
     extra_cli_args: Optional[List[str]] = None
 
     # Provider-specific settings
@@ -1437,8 +1463,8 @@ class S3CompatibleStore(AbstractStore):
     """Base class for S3-compatible object storage providers.
 
     This class provides a unified interface for all S3-compatible storage
-    providers (AWS S3, Cloudflare R2, Nebius, MinIO, etc.) by leveraging
-    a configuration-driven approach that eliminates code duplication.
+    providers (AWS S3, Cloudflare R2, Nebius, MinIO, CoreWeave, etc.) by
+    leveraging a configuration-driven approach that eliminates code duplication
 
     ## Adding a New S3-Compatible Store
 
@@ -1864,6 +1890,9 @@ class S3CompatibleStore(AbstractStore):
             if self.config.credentials_file:
                 cmd = 'AWS_SHARED_CREDENTIALS_FILE=' + \
                 f'{self.config.credentials_file} {cmd}'
+            if self.config.config_file:
+                cmd = 'AWS_CONFIG_FILE=' + \
+                f'{self.config.config_file} {cmd}'
 
             return cmd
 
@@ -1909,6 +1938,9 @@ class S3CompatibleStore(AbstractStore):
             if self.config.credentials_file:
                 cmd = 'AWS_SHARED_CREDENTIALS_FILE=' + \
                 f'{self.config.credentials_file} {cmd}'
+            if self.config.config_file:
+                cmd = 'AWS_CONFIG_FILE=' + \
+                f'{self.config.config_file} {cmd}'
 
             return cmd
 
@@ -1962,6 +1994,9 @@ class S3CompatibleStore(AbstractStore):
                 if self.config.credentials_file:
                     command = (f'AWS_SHARED_CREDENTIALS_FILE='
                                f'{self.config.credentials_file} {command}')
+                if self.config.config_file:
+                    command = 'AWS_CONFIG_FILE=' + \
+                    f'{self.config.config_file} {command}'
                 with ux_utils.print_exception_no_traceback():
                     raise exceptions.StorageBucketGetError(
                         _BUCKET_FAIL_TO_CONNECT_MESSAGE.format(name=self.name) +
@@ -2034,7 +2069,9 @@ class S3CompatibleStore(AbstractStore):
             remove_command = (f'AWS_SHARED_CREDENTIALS_FILE='
                               f'{self.config.credentials_file} '
                               f'{remove_command}')
-
+        if self.config.config_file:
+            remove_command = 'AWS_CONFIG_FILE=' + \
+            f'{self.config.config_file} {remove_command}'
         return self._execute_remove_command(
             remove_command, bucket_name,
             f'Deleting {self.config.store_type} bucket {bucket_name}',
@@ -2047,8 +2084,9 @@ class S3CompatibleStore(AbstractStore):
         try:
             with rich_utils.safe_status(
                     ux_utils.spinner_message(hint_operating)):
-                subprocess.check_output(command.split(' '),
-                                        stderr=subprocess.STDOUT)
+                subprocess.check_output(command,
+                                        stderr=subprocess.STDOUT,
+                                        shell=True)
         except subprocess.CalledProcessError as e:
             if 'NoSuchBucket' in e.output.decode('utf-8'):
                 logger.debug(
@@ -2091,7 +2129,9 @@ class S3CompatibleStore(AbstractStore):
             remove_command = (f'AWS_SHARED_CREDENTIALS_FILE='
                               f'{self.config.credentials_file} '
                               f'{remove_command}')
-
+        if self.config.config_file:
+            remove_command = 'AWS_CONFIG_FILE=' + \
+            f'{self.config.config_file} {remove_command}'
         return self._execute_remove_command(
             remove_command, bucket_name,
             (f'Removing objects from {self.config.store_type} bucket '
@@ -2168,6 +2208,10 @@ class GcsStore(AbstractStore):
             elif self.source.startswith('oci://'):
                 raise NotImplementedError(
                     'Moving data from OCI to GCS is currently not supported.')
+            elif self.source.startswith('cw://'):
+                raise NotImplementedError(
+                    'Moving data from CoreWeave Object Storage to GCS is'
+                    ' currently not supported.')
         # Validate name
         self.name = self.validate_name(self.name)
         # Check if the storage is enabled
@@ -2783,6 +2827,10 @@ class AzureBlobStore(AbstractStore):
             elif self.source.startswith('oci://'):
                 raise NotImplementedError(
                     'Moving data from OCI to AZureBlob is not supported.')
+            elif self.source.startswith('cw://'):
+                raise NotImplementedError(
+                    'Moving data from CoreWeave Object Storage to AzureBlob is'
+                    ' currently not supported.')
         # Validate name
         self.name = self.validate_name(self.name)
 
@@ -3154,6 +3202,8 @@ class AzureBlobStore(AbstractStore):
                     raise NotImplementedError(error_message.format('OCI'))
                 elif self.source.startswith('nebius://'):
                     raise NotImplementedError(error_message.format('NEBIUS'))
+                elif self.source.startswith('cw://'):
+                    raise NotImplementedError(error_message.format('CoreWeave'))
                 else:
                     self.batch_az_blob_sync([self.source])
         except exceptions.StorageUploadError:
@@ -3572,6 +3622,10 @@ class IBMCosStore(AbstractStore):
                 assert self.name == data_utils.split_cos_path(self.source)[0], (
                     'COS Bucket is specified as path, the name should be '
                     'the same as COS bucket.')
+            elif self.source.startswith('cw://'):
+                raise NotImplementedError(
+                    'Moving data from CoreWeave Object Storage to COS is '
+                    'currently not supported.')
         # Validate name
         self.name = IBMCosStore.validate_name(self.name)
 
@@ -3670,6 +3724,9 @@ class IBMCosStore(AbstractStore):
                 elif self.source.startswith('r2://'):
                     raise Exception('IBM COS currently not supporting'
                                     'data transfers between COS and r2')
+                elif self.source.startswith('cw://'):
+                    raise Exception('IBM COS currently not supporting'
+                                    'data transfers between COS and CoreWeave')
                 else:
                     self.batch_ibm_rsync([self.source])
 
@@ -4595,3 +4652,103 @@ class NebiusStore(S3CompatibleStore):
             rclone_config, rclone_profile_name, self.bucket.name, mount_path)
         return mounting_utils.get_mounting_command(mount_path, install_cmd,
                                                    mount_cached_cmd)
+
+
+@register_s3_compatible_store
+class CoreWeaveStore(S3CompatibleStore):
+    """CoreWeaveStore inherits from S3CompatibleStore and represents the backend
+    for CoreWeave Object Storage buckets.
+    """
+
+    @classmethod
+    def get_config(cls) -> S3CompatibleConfig:
+        """Return the configuration for CoreWeave Object Storage."""
+        return S3CompatibleConfig(
+            store_type='COREWEAVE',
+            url_prefix='cw://',
+            client_factory=lambda region: data_utils.create_coreweave_client(),
+            resource_factory=lambda name: coreweave.resource('s3').Bucket(name),
+            split_path=data_utils.split_coreweave_path,
+            verify_bucket=data_utils.verify_coreweave_bucket,
+            aws_profile=coreweave.COREWEAVE_PROFILE_NAME,
+            get_endpoint_url=coreweave.get_endpoint,
+            credentials_file=coreweave.COREWEAVE_CREDENTIALS_PATH,
+            config_file=coreweave.COREWEAVE_CONFIG_PATH,
+            cloud_name=coreweave.NAME,
+            default_region=coreweave.DEFAULT_REGION,
+            mount_cmd_factory=cls._get_coreweave_mount_cmd,
+        )
+
+    def _get_bucket(self) -> Tuple[StorageHandle, bool]:
+        """Get or create bucket using CoreWeave's S3 API"""
+        bucket = self.config.resource_factory(self.name)
+
+        # Use our custom bucket verification instead of head_bucket
+        if data_utils.verify_coreweave_bucket(self.name):
+            self._validate_existing_bucket()
+            return bucket, False
+
+        # TODO(hailong): Enable the bucket creation for CoreWeave
+        # Disable this to avoid waiting too long until the following
+        # issue is resolved:
+        # https://github.com/skypilot-org/skypilot/issues/7736
+        raise exceptions.StorageBucketGetError(
+            f'Bucket {self.name!r} does not exist. CoreWeave buckets can take'
+            ' a long time to become accessible after creation, so SkyPilot'
+            ' does not create them automatically. Please create the bucket'
+            ' manually in CoreWeave and wait for it to be accessible before'
+            ' using it.')
+
+        # # Check if this is a source with URL prefix (existing bucket case)
+        # if isinstance(self.source, str) and self.source.startswith(
+        #         self.config.url_prefix):
+        #     with ux_utils.print_exception_no_traceback():
+        #         raise exceptions.StorageBucketGetError(
+        #             'Attempted to use a non-existent bucket as a source: '
+        #             f'{self.source}.')
+
+        # # If bucket cannot be found, create it if needed
+        # if self.sync_on_reconstruction:
+        #     bucket = self._create_bucket(self.name)
+        #     return bucket, True
+        # else:
+        #     raise exceptions.StorageExternalDeletionError(
+        #         'Attempted to fetch a non-existent bucket: '
+        #         f'{self.name}')
+
+    @classmethod
+    def _get_coreweave_mount_cmd(cls, bucket_name: str, mount_path: str,
+                                 bucket_sub_path: Optional[str]) -> str:
+        """Factory method for CoreWeave mount command."""
+        endpoint_url = coreweave.get_endpoint()
+        return mounting_utils.get_coreweave_mount_cmd(
+            coreweave.COREWEAVE_CREDENTIALS_PATH,
+            coreweave.COREWEAVE_PROFILE_NAME, bucket_name, endpoint_url,
+            mount_path, bucket_sub_path)
+
+    def mount_cached_command(self, mount_path: str) -> str:
+        """CoreWeave-specific cached mount implementation using rclone."""
+        install_cmd = mounting_utils.get_rclone_install_cmd()
+        rclone_profile_name = (
+            data_utils.Rclone.RcloneStores.COREWEAVE.get_profile_name(
+                self.name))
+        rclone_config = data_utils.Rclone.RcloneStores.COREWEAVE.get_config(
+            rclone_profile_name=rclone_profile_name)
+        mount_cached_cmd = mounting_utils.get_mount_cached_cmd(
+            rclone_config, rclone_profile_name, self.bucket.name, mount_path)
+        return mounting_utils.get_mounting_command(mount_path, install_cmd,
+                                                   mount_cached_cmd)
+
+    def _create_bucket(self, bucket_name: str) -> StorageHandle:
+        """Create bucket using S3 API with timing handling for CoreWeave."""
+        result = super()._create_bucket(bucket_name)
+        # Ensure bucket is created
+        # The newly created bucket ever takes about 18min to be accessible,
+        # here we just retry for 36 times (5s * 36 = 180s) to avoid waiting
+        # too long
+        # TODO(hailong): Update the logic here when the following
+        # issue is resolved:
+        # https://github.com/skypilot-org/skypilot/issues/7736
+        data_utils.verify_coreweave_bucket(bucket_name, retry=36)
+
+        return result

sky/global_user_state.py

@@ -1605,7 +1605,6 @@ def get_cluster_from_name(
         cluster_table.c.owner,
         cluster_table.c.metadata,
         cluster_table.c.cluster_hash,
-        cluster_table.c.storage_mounts_metadata,
         cluster_table.c.cluster_ever_up,
         cluster_table.c.status_updated_at,
         cluster_table.c.user_hash,
@@ -1642,8 +1641,6 @@ def get_cluster_from_name(
         'owner': _load_owner(row.owner),
         'metadata': json.loads(row.metadata),
         'cluster_hash': row.cluster_hash,
-        'storage_mounts_metadata': _load_storage_mounts_metadata(
-            row.storage_mounts_metadata),
         'cluster_ever_up': bool(row.cluster_ever_up),
         'status_updated_at': row.status_updated_at,
         'workspace': row.workspace,
@@ -1704,27 +1701,27 @@ def get_clusters(
         cluster_table.c.name,
         cluster_table.c.launched_at,
         cluster_table.c.handle,
-        cluster_table.c.last_use,
         cluster_table.c.status,
         cluster_table.c.autostop,
         cluster_table.c.to_down,
-        cluster_table.c.owner,
-        cluster_table.c.metadata,
         cluster_table.c.cluster_hash,
-        cluster_table.c.storage_mounts_metadata,
         cluster_table.c.cluster_ever_up,
-        cluster_table.c.status_updated_at,
         cluster_table.c.user_hash,
-        cluster_table.c.config_hash,
         cluster_table.c.workspace,
-        cluster_table.c.is_managed,
         user_table.c.name.label('user_name'),
     ]
     if not summary_response:
         query_fields.extend([
             cluster_table.c.last_creation_yaml,
             cluster_table.c.last_creation_command,
+            cluster_table.c.config_hash,
+            cluster_table.c.owner,
+            cluster_table.c.metadata,
+            cluster_table.c.last_use,
+            cluster_table.c.status_updated_at,
         ])
+    if not exclude_managed_clusters:
+        query_fields.append(cluster_table.c.is_managed)
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
         query = session.query(*query_fields).outerjoin(
             user_table, cluster_table.c.user_hash == user_table.c.id)
@@ -1771,30 +1768,29 @@ def get_clusters(
             'name': row.name,
             'launched_at': row.launched_at,
             'handle': pickle.loads(row.handle),
-            'last_use': row.last_use,
             'status': status_lib.ClusterStatus[row.status],
             'autostop': row.autostop,
             'to_down': bool(row.to_down),
-            'owner': _load_owner(row.owner),
-            'metadata': json.loads(row.metadata),
             'cluster_hash': row.cluster_hash,
-            'storage_mounts_metadata': _load_storage_mounts_metadata(
-                row.storage_mounts_metadata),
             'cluster_ever_up': bool(row.cluster_ever_up),
-            'status_updated_at': row.status_updated_at,
             'user_hash': (row.user_hash
                           if row.user_hash is not None else current_user_hash),
             'user_name': (row.user_name
                           if row.user_name is not None else current_user_name),
             'workspace': row.workspace,
-            'is_managed': bool(row.is_managed),
-            'config_hash': row.config_hash,
+            'is_managed': False
+                          if exclude_managed_clusters else bool(row.is_managed),
         }
         if not summary_response:
             record['last_creation_yaml'] = row.last_creation_yaml
             record['last_creation_command'] = row.last_creation_command
             record['last_event'] = last_cluster_event_dict.get(
                 row.cluster_hash, None)
+            record['config_hash'] = row.config_hash
+            record['owner'] = _load_owner(row.owner)
+            record['metadata'] = json.loads(row.metadata)
+            record['last_use'] = row.last_use
+            record['status_updated_at'] = row.status_updated_at
 
         records.append(record)
     return records

sky/jobs/server/server.py

@@ -206,8 +206,8 @@ async def pool_tail_logs(
         request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
-    request_task = api_requests.get_request(request.state.request_id,
-                                            fields=['request_id'])
+    request_task = await api_requests.get_request_async(
+        request.state.request_id, fields=['request_id'])
 
     return stream_utils.stream_response_for_long_request(
         request_id=request_task.request_id,

sky/jobs/utils.py

@@ -1522,12 +1522,11 @@ def get_managed_job_queue(
             handle = cluster_name_to_handle.get(
                 cluster_name, None) if cluster_name is not None else None
             if isinstance(handle, backends.CloudVmRayResourceHandle):
-                resources_str = resources_utils.get_readable_resources_repr(
-                    handle, simplify=True)
-                resources_str_full = (
-                    resources_utils.get_readable_resources_repr(handle,
-                                                                simplify=False))
-                job['cluster_resources'] = resources_str
+                resources_str_simple, resources_str_full = (
+                    resources_utils.get_readable_resources_repr(
+                        handle, simplified_only=False))
+                assert resources_str_full is not None
+                job['cluster_resources'] = resources_str_simple
                 job['cluster_resources_full'] = resources_str_full
                 job['cloud'] = str(handle.launched_resources.cloud)
                 job['region'] = handle.launched_resources.region

sky/optimizer.py

@@ -1019,7 +1019,7 @@ class Optimizer:
                             if res.instance_type is not None
                         ])
                         candidate_str = resources_utils.format_resource(
-                            best_resources, simplify=True)
+                            best_resources, simplified_only=True)[0]
 
                         logger.info(
                             f'{colorama.Style.DIM}🔍 Multiple {cloud} instances '

sky/provision/kubernetes/instance.py

@@ -33,6 +33,7 @@ from sky.utils.db import db_utils
 POLL_INTERVAL = 2
 _TIMEOUT_FOR_POD_TERMINATION = 60  # 1 minutes
 _MAX_RETRIES = 3
+_MAX_MISSING_PODS_RETRIES = 5
 _NUM_THREADS = subprocess_utils.get_parallel_threads('kubernetes')
 
 # Pattern to extract SSH user from command output, handling MOTD contamination
@@ -489,17 +490,17 @@ def _wait_for_pods_to_schedule(namespace, context, new_nodes, timeout: int,
 
 
 @timeline.event
-def _wait_for_pods_to_run(namespace, context, new_nodes):
+def _wait_for_pods_to_run(namespace, context, cluster_name, new_pods):
     """Wait for pods and their containers to be ready.
 
     Pods may be pulling images or may be in the process of container
     creation.
     """
-    if not new_nodes:
+    if not new_pods:
         return
 
     # Create a set of pod names we're waiting for
-    expected_pod_names = {node.metadata.name for node in new_nodes}
+    expected_pod_names = {pod.metadata.name for pod in new_pods}
 
     def _check_init_containers(pod):
         # Check if any of the init containers failed
@@ -526,28 +527,62 @@ def _wait_for_pods_to_run(namespace, context, new_nodes):
                     'Failed to create init container for pod '
                     f'{pod.metadata.name}. Error details: {msg}.')
 
+    missing_pods_retry = 0
     while True:
         # Get all pods in a single API call
-        cluster_name = new_nodes[0].metadata.labels[
+        cluster_name_on_cloud = new_pods[0].metadata.labels[
             k8s_constants.TAG_SKYPILOT_CLUSTER_NAME]
         all_pods = kubernetes.core_api(context).list_namespaced_pod(
             namespace,
             label_selector=
-            f'{k8s_constants.TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}').items
+            f'{k8s_constants.TAG_SKYPILOT_CLUSTER_NAME}={cluster_name_on_cloud}'
+        ).items
 
         # Get the set of found pod names and check if we have all expected pods
         found_pod_names = {pod.metadata.name for pod in all_pods}
-        missing_pods = expected_pod_names - found_pod_names
-        if missing_pods:
+        missing_pod_names = expected_pod_names - found_pod_names
+        if missing_pod_names:
+            # In _wait_for_pods_to_schedule, we already wait for all pods to go
+            # from pending to scheduled. So if a pod is missing here, it means
+            # something unusual must have happened, and so should be treated as
+            # an exception.
+            # It is also only in _wait_for_pods_to_schedule that
+            # provision_timeout is used.
+            # TODO(kevin): Should we take provision_timeout into account here,
+            # instead of hardcoding the number of retries?
+            if missing_pods_retry >= _MAX_MISSING_PODS_RETRIES:
+                for pod_name in missing_pod_names:
+                    reason = _get_pod_missing_reason(context, namespace,
+                                                     cluster_name, pod_name)
+                    logger.warning(f'Pod {pod_name} missing: {reason}')
+                raise config_lib.KubernetesError(
+                    f'Failed to get all pods after {missing_pods_retry} '
+                    f'retries. Some pods may have been terminated or failed '
+                    f'unexpectedly. Run `sky logs --provision {cluster_name}` '
+                    'for more details.')
             logger.info('Retrying running pods check: '
-                        f'Missing pods: {missing_pods}')
+                        f'Missing pods: {missing_pod_names}')
             time.sleep(0.5)
+            missing_pods_retry += 1
             continue
 
         all_pods_running = True
         for pod in all_pods:
             if pod.metadata.name not in expected_pod_names:
                 continue
+
+            # Check if pod is terminated/preempted/failed.
+            if (pod.metadata.deletion_timestamp is not None or
+                    pod.status.phase == 'Failed'):
+                # Get the reason and write to cluster events before
+                # the pod gets completely deleted from the API.
+                reason = _get_pod_termination_reason(pod, cluster_name)
+                logger.warning(f'Pod {pod.metadata.name} terminated: {reason}')
+                raise config_lib.KubernetesError(
+                    f'Pod {pod.metadata.name} has terminated or failed '
+                    f'unexpectedly. Run `sky logs --provision {cluster_name}` '
+                    'for more details.')
+
             # Continue if pod and all the containers within the
             # pod are successfully created and running.
             if pod.status.phase == 'Running' and all(
@@ -1169,7 +1204,7 @@ def _create_pods(region: str, cluster_name: str, cluster_name_on_cloud: str,
     # fail early if there is an error
     logger.debug(f'run_instances: waiting for pods to be running (pulling '
                  f'images): {[pod.metadata.name for pod in pods]}')
-    _wait_for_pods_to_run(namespace, context, pods)
+    _wait_for_pods_to_run(namespace, context, cluster_name, pods)
     logger.debug(f'run_instances: all pods are scheduled and running: '
                  f'{[pod.metadata.name for pod in pods]}')
 
@@ -1428,9 +1463,45 @@ def get_cluster_info(
 
 
 def _get_pod_termination_reason(pod: Any, cluster_name: str) -> str:
-    """Get pod termination reason and write to cluster events."""
-    reasons = []
+    """Get pod termination reason and write to cluster events.
+
+    Checks both pod conditions (for preemption/disruption) and
+    container statuses (for exit codes/errors).
+    """
     latest_timestamp = pod.status.start_time or datetime.datetime.min
+    ready_state = 'Unknown'
+    termination_reason = 'Terminated unexpectedly'
+    container_reasons = []
+
+    # Check pod status conditions for high level overview.
+    # No need to sort, as each condition.type will only appear once.
+    for condition in pod.status.conditions:
+        reason = condition.reason or 'Unknown reason'
+        message = condition.message or ''
+
+        # Get last known readiness state.
+        if condition.type == 'Ready':
+            ready_state = f'{reason} ({message})' if message else reason
+        # Kueue preemption, as defined in:
+        # https://pkg.go.dev/sigs.k8s.io/kueue/pkg/controller/jobs/pod#pkg-constants
+        elif condition.type == 'TerminationTarget':
+            termination_reason = f'Preempted by Kueue: {reason}'
+            if message:
+                termination_reason += f' ({message})'
+        # Generic disruption.
+        elif condition.type == 'DisruptionTarget':
+            termination_reason = f'Disrupted: {reason}'
+            if message:
+                termination_reason += f' ({message})'
+
+        if condition.last_transition_time is not None:
+            latest_timestamp = max(latest_timestamp,
+                                   condition.last_transition_time)
+
+    pod_reason = (f'{termination_reason}.\n'
+                  f'Last known state: {ready_state}.')
+
+    # Check container statuses for exit codes/errors
     if pod.status and pod.status.container_statuses:
         for container_status in pod.status.container_statuses:
             terminated = container_status.state.terminated
@@ -1445,18 +1516,15 @@ def _get_pod_termination_reason(pod: Any, cluster_name: str) -> str:
                 if reason is None:
                     # just in-case reason is None, have default for debugging
                     reason = f'exit({exit_code})'
-                reasons.append(reason)
-                if terminated.finished_at > latest_timestamp:
-                    latest_timestamp = terminated.finished_at
+                container_reasons.append(reason)
+                latest_timestamp = max(latest_timestamp, terminated.finished_at)
 
             # TODO (kyuds): later, if needed, query `last_state` too.
 
-    if not reasons:
-        return ''
-
     # Normally we will have a single container per pod for skypilot
     # but doing this just in-case there are multiple containers.
-    pod_reason = ' | '.join(reasons)
+    if container_reasons:
+        pod_reason += f'\nContainer errors: {" | ".join(container_reasons)}'
 
     global_user_state.add_cluster_event(
         cluster_name,
@@ -1658,9 +1726,10 @@ def query_instances(
                                     Optional[str]]] = {}
     for pod in pods:
         phase = pod.status.phase
+        is_terminating = pod.metadata.deletion_timestamp is not None
         pod_status = status_map[phase]
         reason = None
-        if phase in ('Failed', 'Unknown'):
+        if phase in ('Failed', 'Unknown') or is_terminating:
             reason = _get_pod_termination_reason(pod, cluster_name)
             logger.debug(f'Pod Status ({phase}) Reason(s): {reason}')
         if non_terminated_only and pod_status is None:

sky/provision/kubernetes/volume.py

@@ -75,7 +75,6 @@ def delete_volume(config: models.VolumeConfig) -> models.VolumeConfig:
     """Deletes a volume."""
     context, namespace = _get_context_namespace(config)
     pvc_name = config.name_on_cloud
-    logger.info(f'Deleting PVC {pvc_name}')
     kubernetes_utils.delete_k8s_resource_with_retry(
         delete_func=lambda pvc_name=pvc_name: kubernetes.core_api(
             context).delete_namespaced_persistent_volume_claim(
@@ -84,6 +83,7 @@ def delete_volume(config: models.VolumeConfig) -> models.VolumeConfig:
                 _request_timeout=config_lib.DELETION_TIMEOUT),
         resource_type='pvc',
         resource_name=pvc_name)
+    logger.info(f'Deleted PVC {pvc_name} in namespace {namespace}')
     return config
 
 
@@ -242,9 +242,9 @@ def create_persistent_volume_claim(namespace: str, context: Optional[str],
     except kubernetes.api_exception() as e:
         if e.status != 404:  # Not found
             raise
-    logger.info(f'Creating PVC {pvc_name}')
     kubernetes.core_api(context).create_namespaced_persistent_volume_claim(
         namespace=namespace, body=pvc_spec)
+    logger.info(f'Created PVC {pvc_name} in namespace {namespace}')
 
 
 def _get_pvc_spec(namespace: str,