skypilot-nightly 1.0.0.dev20250402__py3-none-any.whl → 1.0.0.dev20250404__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '515e5df7c14cfd425f606de20d4b96d6a897ee7d'
+_SKYPILOT_COMMIT_SHA = 'dfd12c30b89ae96f5596418fed58547c159ff40a'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250402'
+__version__ = '1.0.0.dev20250404'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/check.py

@@ -289,6 +289,12 @@ def _print_checked_cloud(
         cloud_tuple: The cloud to print the capabilities for.
         cloud_capabilities: The capabilities for the cloud.
     """
+
+    def _yellow_color(str_to_format: str) -> str:
+        return (f'{colorama.Fore.LIGHTYELLOW_EX}'
+                f'{str_to_format}'
+                f'{colorama.Style.RESET_ALL}')
+
     cloud_repr, cloud = cloud_tuple
     # Print the capabilities for the cloud.
     # consider cloud enabled if any capability is enabled.
@@ -319,7 +325,7 @@ def _print_checked_cloud(
     if activated_account is not None:
         echo(f'    Activated account: {activated_account}')
     for reason, caps in hints_to_capabilities.items():
-        echo(f'    Hint [{", ".join(caps)}]: {reason}')
+        echo(f'    Hint [{", ".join(caps)}]: {_yellow_color(reason)}')
     for reason, caps in reasons_to_capabilities.items():
         echo(f'    Reason [{", ".join(caps)}]: {reason}')
 

sky/clouds/kubernetes.py

@@ -679,7 +679,8 @@ class Kubernetes(clouds.Cloud):
         success = False
         for context in existing_allowed_contexts:
             try:
-                check_result = kubernetes_utils.check_credentials(context)
+                check_result = kubernetes_utils.check_credentials(
+                    context, run_optional_checks=True)
                 if check_result[0]:
                     success = True
                     if check_result[1] is not None:

sky/clouds/service_catalog/__init__.py

@@ -328,9 +328,9 @@ def get_common_gpus() -> List[str]:
         'A100',
         'A100-80GB',
         'H100',
+        'H200',
         'L4',
         'L40S',
-        'P100',
         'T4',
         'V100',
         'V100-32GB',

sky/data/storage.py

@@ -273,6 +273,8 @@ MOUNTABLE_STORAGE_MODES = [
     StorageMode.MOUNT_CACHED,
 ]
 
+DEFAULT_STORAGE_MODE = StorageMode.MOUNT
+
 
 class AbstractStore:
     """AbstractStore abstracts away the different storage types exposed by
@@ -591,7 +593,7 @@ class Storage(object):
         source: Optional[SourceType] = None,
         stores: Optional[List[StoreType]] = None,
         persistent: Optional[bool] = True,
-        mode: StorageMode = StorageMode.MOUNT,
+        mode: StorageMode = DEFAULT_STORAGE_MODE,
         sync_on_reconstruction: bool = True,
         # pylint: disable=invalid-name
         _is_sky_managed: Optional[bool] = None,
@@ -855,7 +857,7 @@ class Storage(object):
                 is_local_source = False
                 # Storage mounting does not support mounting specific files from
                 # cloud store - ensure path points to only a directory
-                if mode == StorageMode.MOUNT:
+                if mode in MOUNTABLE_STORAGE_MODES:
                     if (split_path.scheme != 'https' and
                         ((split_path.scheme != 'cos' and
                           split_path.path.strip('/') != '') or
@@ -1284,8 +1286,7 @@ class Storage(object):
             # Make mode case insensitive, if specified
             mode = StorageMode(mode_str.upper())
         else:
-            # Make sure this keeps the same as the default mode in __init__
-            mode = StorageMode.MOUNT
+            mode = DEFAULT_STORAGE_MODE
         persistent = config.pop('persistent', None)
         if persistent is None:
             persistent = True

sky/data/storage_utils.py

@@ -223,7 +223,7 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
 def get_excluded_files(src_dir_path: str) -> List[str]:
     # TODO: this could return a huge list of files,
     # should think of ways to optimize.
-    """ List files and directories to be excluded."""
+    """List files and directories to be excluded."""
     expand_src_dir_path = os.path.expanduser(src_dir_path)
     skyignore_path = os.path.join(expand_src_dir_path,
                                   constants.SKY_IGNORE_FILE)
@@ -273,12 +273,22 @@ def zip_files_and_folders(items: List[str],
                     zipf.write(item)
                 elif os.path.isdir(item):
                     for root, dirs, files in os.walk(item, followlinks=False):
+                        # Modify dirs in-place to control os.walk()'s traversal
+                        # behavior. This filters out excluded directories BEFORE
+                        # os.walk() visits the files and sub-directories under
+                        # them, preventing traversal into any excluded directory
+                        # and its contents.
+                        # Note: dirs[:] = ... is required for in-place
+                        # modification.
+                        dirs[:] = [
+                            d for d in dirs
+                            if os.path.join(root, d) not in excluded_files
+                        ]
+
                         # Store directory entries (important for empty
                         # directories)
                         for dir_name in dirs:
                             dir_path = os.path.join(root, dir_name)
-                            if dir_path in excluded_files:
-                                continue
                             # If it's a symlink, store it as a symlink
                             if os.path.islink(dir_path):
                                 _store_symlink(zipf, dir_path, is_dir=True)

sky/jobs/controller.py

@@ -227,13 +227,13 @@ class JobsController:
                 self._backend, cluster_name)
 
             if job_status == job_lib.JobStatus.SUCCEEDED:
-                end_time = managed_job_utils.try_to_get_job_end_time(
+                success_end_time = managed_job_utils.try_to_get_job_end_time(
                     self._backend, cluster_name)
                 # The job is done. Set the job to SUCCEEDED first before start
                 # downloading and streaming the logs to make it more responsive.
                 managed_job_state.set_succeeded(self._job_id,
                                                 task_id,
-                                                end_time=end_time,
+                                                end_time=success_end_time,
                                                 callback_func=callback_func)
                 logger.info(
                     f'Managed job {self._job_id} (task: {task_id}) SUCCEEDED. '
@@ -299,23 +299,40 @@ class JobsController:
                 if job_status is not None and not job_status.is_terminal():
                     # The multi-node job is still running, continue monitoring.
                     continue
-                elif job_status in job_lib.JobStatus.user_code_failure_states():
+                elif (job_status
+                      in job_lib.JobStatus.user_code_failure_states() or
+                      job_status == job_lib.JobStatus.FAILED_DRIVER):
                     # The user code has probably crashed, fail immediately.
                     end_time = managed_job_utils.try_to_get_job_end_time(
                         self._backend, cluster_name)
                     logger.info(
-                        'The user job failed. Please check the logs below.\n'
+                        f'The user job failed ({job_status}). Please check the '
+                        'logs below.\n'
                         f'== Logs of the user job (ID: {self._job_id}) ==\n')
 
                     self._download_log_and_stream(task_id, handle)
+
+                    failure_reason = (
+                        'To see the details, run: '
+                        f'sky jobs logs --controller {self._job_id}')
+
                     managed_job_status = (
                         managed_job_state.ManagedJobStatus.FAILED)
                     if job_status == job_lib.JobStatus.FAILED_SETUP:
                         managed_job_status = (
                             managed_job_state.ManagedJobStatus.FAILED_SETUP)
-                    failure_reason = (
-                        'To see the details, run: '
-                        f'sky jobs logs --controller {self._job_id}')
+                    elif job_status == job_lib.JobStatus.FAILED_DRIVER:
+                        # FAILED_DRIVER is kind of an internal error, so we mark
+                        # this as FAILED_CONTROLLER, even though the failure is
+                        # not strictly within the controller.
+                        managed_job_status = (
+                            managed_job_state.ManagedJobStatus.FAILED_CONTROLLER
+                        )
+                        failure_reason = (
+                            'The job driver on the remote cluster failed. This '
+                            'can be caused by the job taking too much memory '
+                            'or other resources. Try adding more memory, CPU, '
+                            f'or disk in your job definition. {failure_reason}')
                     should_restart_on_failure = (
                         self._strategy_executor.should_restart_on_failure())
                     if should_restart_on_failure:
@@ -337,6 +354,21 @@ class JobsController:
                             end_time=end_time,
                             callback_func=callback_func)
                         return False
+                elif job_status is not None:
+                    # Either the job is cancelled (should not happen) or in some
+                    # unknown new state that we do not handle.
+                    logger.error(f'Unknown job status: {job_status}')
+                    failure_reason = (
+                        f'Unknown job status {job_status}. To see the details, '
+                        f'run: sky jobs logs --controller {self._job_id}')
+                    managed_job_state.set_failed(
+                        self._job_id,
+                        task_id,
+                        failure_type=managed_job_state.ManagedJobStatus.
+                        FAILED_CONTROLLER,
+                        failure_reason=failure_reason,
+                        callback_func=callback_func)
+                    return False
                 else:
                     # Although the cluster is healthy, we fail to access the
                     # job status. Try to recover the job (will not restart the

sky/jobs/server/server.py

@@ -160,7 +160,7 @@ async def dashboard(request: fastapi.Request,
                 async with httpx.AsyncClient() as client:
                     response = await client.request('GET',
                                                     dashboard_url,
-                                                    timeout=1)
+                                                    timeout=5)
                 break  # Connection successful, proceed with the request
             except Exception as e:  # pylint: disable=broad-except
                 # We catch all exceptions to gracefully handle unknown

sky/provision/kubernetes/utils.py

@@ -272,7 +272,8 @@ def get_gke_accelerator_name(accelerator: str) -> str:
     if accelerator == 'H100':
         # H100 is named as H100-80GB in GKE.
         accelerator = 'H100-80GB'
-    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB'):
+    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB',
+                       'B200'):
         # A100-80GB, L4, H100-80GB and H100-MEGA-80GB
         # have a different name pattern.
         return 'nvidia-{}'.format(accelerator.lower())
@@ -1096,7 +1097,9 @@ def get_accelerator_label_key_value(
         ResourcesUnavailableError: Can be raised from the following conditions:
             - The cluster does not have GPU/TPU resources
                 (nvidia.com/gpu, google.com/tpu)
-            - The cluster does not have GPU/TPU labels setup correctly
+            - The cluster has GPU/TPU resources, but no node in the cluster has
+              an accelerator label.
+            - The cluster has a node with an invalid accelerator label value.
             - The cluster doesn't have any nodes with acc_type GPU/TPU
     """
     # Check if the cluster has GPU resources
@@ -1291,7 +1294,8 @@ def get_external_ip(network_mode: Optional[
 
 
 def check_credentials(context: Optional[str],
-                      timeout: int = kubernetes.API_TIMEOUT) -> \
+                      timeout: int = kubernetes.API_TIMEOUT,
+                      run_optional_checks: bool = False) -> \
         Tuple[bool, Optional[str]]:
     """Check if the credentials in kubeconfig file are valid
 
@@ -1333,6 +1337,9 @@ def check_credentials(context: Optional[str],
                        f'{common_utils.format_exception(e, use_bracket=True)}')
 
     # If we reach here, the credentials are valid and Kubernetes cluster is up.
+    if not run_optional_checks:
+        return True, None
+
     # We now do softer checks to check if exec based auth is used and to
     # see if the cluster is GPU-enabled.
 
@@ -1344,16 +1351,36 @@ def check_credentials(context: Optional[str],
     # `sky launch --gpus <gpu>` and the optimizer does not list Kubernetes as a
     # provider if their cluster GPUs are not setup correctly.
     gpu_msg = ''
-    try:
-        get_accelerator_label_key_value(context,
-                                        acc_type='',
-                                        acc_count=0,
-                                        check_mode=True)
-    except exceptions.ResourcesUnavailableError as e:
-        # If GPUs are not available, we return cluster as enabled (since it can
-        # be a CPU-only cluster) but we also return the exception message which
-        # serves as a hint for how to enable GPU access.
-        gpu_msg = str(e)
+    unlabeled_nodes = get_unlabeled_accelerator_nodes(context)
+    if unlabeled_nodes:
+        gpu_msg = (f'Cluster has {len(unlabeled_nodes)} nodes with '
+                   f'accelerators that are not labeled. '
+                   f'To label the nodes, run '
+                   f'`python -m sky.utils.kubernetes.gpu_labeler '
+                   f'--context {context}`')
+    else:
+        try:
+            # This function raises a ResourcesUnavailableError in three cases:
+            # 1. If no node in cluster has GPU/TPU resource in its capacity.
+            #    (e.g. google.com/tpu, nvidia.com/gpu)
+            # 2. If at least one node in cluster has GPU/TPU resource in its
+            #    capacity, but no node in the cluster has an accelerator label.
+            # 3. If an accelerator label on a node is invalid.
+            # Exception 2 is a special case of a cluster having at least one
+            # unlabelled node, which is caught in
+            # `get_unlabeled_accelerator_nodes`.
+            # Therefore, if `get_unlabeled_accelerator_nodes` detects unlabelled
+            # nodes, we skip this check.
+            get_accelerator_label_key_value(context,
+                                            acc_type='',
+                                            acc_count=0,
+                                            check_mode=True)
+        except exceptions.ResourcesUnavailableError as e:
+            # If GPUs are not available, we return cluster as enabled
+            # (since it can be a CPU-only cluster) but we also return the
+            # exception message which serves as a hint for how to enable
+            # GPU access.
+            gpu_msg = str(e)
     if exec_msg and gpu_msg:
         return True, f'{gpu_msg}\n    Additionally, {exec_msg}'
     elif gpu_msg:
@@ -2457,6 +2484,43 @@ def dict_to_k8s_object(object_dict: Dict[str, Any], object_type: 'str') -> Any:
     return kubernetes.api_client().deserialize(fake_kube_response, object_type)
 
 
+def get_unlabeled_accelerator_nodes(context: Optional[str] = None) -> List[Any]:
+    """Gets a list of unlabeled GPU nodes in the cluster.
+
+    This function returns a list of nodes that have GPU resources but no label
+    that indicates the accelerator type.
+
+    Args:
+        context: The context to check.
+
+    Returns:
+        List[Any]: List of unlabeled nodes with accelerators.
+    """
+    nodes = get_kubernetes_nodes(context=context)
+    nodes_with_accelerator = []
+    for node in nodes:
+        if get_gpu_resource_key() in node.status.capacity:
+            nodes_with_accelerator.append(node)
+
+    label_formatter, _ = detect_gpu_label_formatter(context)
+    if not label_formatter:
+        return nodes_with_accelerator
+    else:
+        label_keys = label_formatter.get_label_keys()
+
+    unlabeled_nodes = []
+    for node in nodes_with_accelerator:
+        labeled = False
+        for label_key in label_keys:
+            if label_key in node.metadata.labels:
+                labeled = True
+                break
+        if not labeled:
+            unlabeled_nodes.append(node)
+
+    return unlabeled_nodes
+
+
 def get_kubernetes_node_info(
         context: Optional[str] = None) -> Dict[str, models.KubernetesNodeInfo]:
     """Gets the resource information for all the nodes in the cluster.

sky/task.py

@@ -745,7 +745,7 @@ class Task:
 
         # Evaluate if the task requires FUSE and set the requires_fuse flag
         for _, storage_obj in self.storage_mounts.items():
-            if storage_obj.mode == storage_lib.StorageMode.MOUNT:
+            if storage_obj.mode in storage_lib.MOUNTABLE_STORAGE_MODES:
                 for r in self.resources:
                     r.requires_fuse = True
                 break
@@ -924,7 +924,7 @@ class Task:
                         'Storage mount destination path cannot be cloud storage'
                     )
 
-            if storage_obj.mode == storage_lib.StorageMode.MOUNT:
+            if storage_obj.mode in storage_lib.MOUNTABLE_STORAGE_MODES:
                 # If any storage is using MOUNT mode, we need to enable FUSE in
                 # the resources.
                 for r in self.resources:

sky/utils/controller_utils.py

@@ -916,7 +916,7 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
                 name=bucket_name,
                 source=local_fm_path,
                 persistent=False,
-                mode=storage_lib.StorageMode.MOUNT,
+                mode=storage_lib.DEFAULT_STORAGE_MODE,
                 stores=stores,
                 _is_sky_managed=not bucket_wth_prefix,
                 _bucket_sub_path=file_mounts_tmp_subpath)

sky/utils/kubernetes/gpu_labeler.py

@@ -51,6 +51,18 @@ def label(context: Optional[str] = None):
         print(reason)
         return
 
+    unlabeled_gpu_nodes = kubernetes_utils.get_unlabeled_accelerator_nodes()
+
+    if not unlabeled_gpu_nodes:
+        print('No unlabeled GPU nodes found in the cluster. If you have '
+              'unlabeled GPU nodes, please ensure that they have the resource '
+              f'`{kubernetes_utils.get_gpu_resource_key()}: <number of GPUs>` '
+              'in their capacity.')
+        return
+
+    print(f'Found {len(unlabeled_gpu_nodes)} '
+          'unlabeled GPU nodes in the cluster')
+
     sky_dir = os.path.dirname(sky.__file__)
     manifest_dir = os.path.join(sky_dir, 'utils/kubernetes')
 
@@ -77,17 +89,6 @@ def label(context: Optional[str] = None):
         with open(job_manifest_path, 'r', encoding='utf-8') as file:
             job_manifest = yaml.safe_load(file)
 
-        # Iterate over nodes
-        nodes = kubernetes_utils.get_kubernetes_nodes(context=context)
-
-        # Get the list of nodes with GPUs
-        gpu_nodes = []
-        for node in nodes:
-            if kubernetes_utils.get_gpu_resource_key() in node.status.capacity:
-                gpu_nodes.append(node)
-
-        print(f'Found {len(gpu_nodes)} GPU node(s) in the cluster')
-
         # Check if the 'nvidia' RuntimeClass exists
         try:
             nvidia_exists = kubernetes_utils.check_nvidia_runtime_class(
@@ -108,7 +109,7 @@ def label(context: Optional[str] = None):
         else:
             print('Using default RuntimeClass for GPU labeling.')
 
-        for node in gpu_nodes:
+        for node in unlabeled_gpu_nodes:
             node_name = node.metadata.name
 
             # Modify the job manifest for the current node
@@ -122,21 +123,16 @@ def label(context: Optional[str] = None):
             # Create the job for this node`
             batch_v1.create_namespaced_job(namespace, job_manifest)
             print(f'Created GPU labeler job for node {node_name}')
-    if not gpu_nodes:
-        print('No GPU nodes found in the cluster. If you have GPU nodes, '
-              'please ensure that they have the label '
-              f'`{kubernetes_utils.get_gpu_resource_key()}: <number of GPUs>`')
-    else:
-        context_str = f' --context {context}' if context else ''
-        print(
-            f'GPU labeling started - this may take 10 min or more to complete.'
-            '\nTo check the status of GPU labeling jobs, run '
-            f'`kubectl get jobs -n kube-system '
-            f'-l job=sky-gpu-labeler{context_str}`'
-            '\nYou can check if nodes have been labeled by running '
-            f'`kubectl describe nodes{context_str}` '
-            'and looking for labels of the format '
-            '`skypilot.co/accelerator: <gpu_name>`. ')
+
+    context_str = f' --context {context}' if context else ''
+    print(f'GPU labeling started - this may take 10 min or more to complete.'
+          '\nTo check the status of GPU labeling jobs, run '
+          f'`kubectl get jobs -n kube-system '
+          f'-l job=sky-gpu-labeler{context_str}`'
+          '\nYou can check if nodes have been labeled by running '
+          f'`kubectl describe nodes{context_str}` '
+          'and looking for labels of the format '
+          '`skypilot.co/accelerator: <gpu_name>`. ')
 
 
 def main():

{skypilot_nightly-1.0.0.dev20250402.dist-info → skypilot_nightly-1.0.0.dev20250404.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250402
+Version: 1.0.0.dev20250404
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250402.dist-info → skypilot_nightly-1.0.0.dev20250404.dist-info}/RECORD

@@ -1,7 +1,7 @@
-sky/__init__.py,sha256=RCyN48gsf7Sx7Y7utA4xGJ1RIMT8wt5rMyEh62L6l-I,6428
+sky/__init__.py,sha256=DAFp3ES0ncxRm0D0gyRcS1gr1FFsn4gkUD6kWcFNHlg,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=ND011K_-Ud1dVZF37A9KrwYir_ihJXcHc7iDWmuBc8Q,22872
-sky/check.py,sha256=oktScSPsHIyO7ZrVHy3QaybB6-s_D6eMEjmICAiUtDo,15902
+sky/check.py,sha256=PPNQnaaZBA9_aogJpN4gnG4XWnTqkd74c-rBYDkDRDY,16101
 sky/cli.py,sha256=Zcio2ak6zX_5_N_lshDUqCvoV6NEOmGS6Tp6AgS9VAk,222446
 sky/cloud_stores.py,sha256=cmKdSoB4bmwrd-Z1NCZBFb6IIJt0jKVxkGPoX86280s,26606
 sky/core.py,sha256=G3n6z0dyvoU4FJVGnnTu3kFdu_EtQC1l57er5voRAX0,47926
@@ -14,7 +14,7 @@ sky/optimizer.py,sha256=uutziDwxyq-f5n31YXgCF9fmqx5vndIU8g_avvGWuGc,58532
 sky/resources.py,sha256=2qc5U09MFDaJjI1dHcThcRodpMGY7HyXzQn8eC4lvbE,72402
 sky/sky_logging.py,sha256=pID2RINjH62n7SZpv70DuN8BSFYdCfTJ2ScGQpVmugg,5725
 sky/skypilot_config.py,sha256=CdaIbPL_7ECG5laOARca4p9df_6NLhT-bO8WnalxZAY,8839
-sky/task.py,sha256=zKKdD918VNGVFI3hgN2NT3N2zNUfTp2SdluSN83Xlf4,55697
+sky/task.py,sha256=kepCHV_evVg9EKnXaKW9Stg1bQAZlaF4UQocwlR3M3g,55709
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=iH55Cm6eXWwufAG0Dgk7LTQcADawNa3ELrBH1m6yuSY,7617
 sky/adaptors/azure.py,sha256=r8xkjRgZZQfsSExNdguFa6c5fCLcUhZrFV8zs62VExo,21986
@@ -55,7 +55,7 @@ sky/clouds/do.py,sha256=P38l4otp2AuDReUH9Ii621ht9s-NIyb7-R37jbtjHk8,11580
 sky/clouds/fluidstack.py,sha256=jIqW1MLe55MVME1PATZm8e6_FsiTnJawW7OdytPW0aM,12666
 sky/clouds/gcp.py,sha256=sUJ9LXUnMxYm6OYZ5P-z1dJHxgVILuC3OW3eFSTNCv8,56919
 sky/clouds/ibm.py,sha256=XtuPN8QgrwJdb1qb_b-7KwAE2tf_N9wh9eEfi2tcg-s,22013
-sky/clouds/kubernetes.py,sha256=ivk6GlAbhj3oDaFfcIzwz2x7qRBrzShglV4FfzikqS0,36169
+sky/clouds/kubernetes.py,sha256=SNRilcl_JYvaMDoYw0jnMtfaSi7FKQycCiJVmX6-4f4,36216
 sky/clouds/lambda_cloud.py,sha256=rR2YrZ6flEbKKpQAm60eKNjiMDYvH2hqzaCo3Hx4Ffw,12916
 sky/clouds/nebius.py,sha256=4D7C2NQYI-BNhXWNOyAXNAZj7-5nN33VQW1sxfSGt9w,14662
 sky/clouds/oci.py,sha256=YO4kjSsHBmAVH4z1TuVP72zfmC0BXte4E0xIyZir9N4,27622
@@ -64,7 +64,7 @@ sky/clouds/runpod.py,sha256=P486CMN-Mt3R-7jMVd3XIGjLWYy0X5B74dK_IgAp2Cg,12149
 sky/clouds/scp.py,sha256=6OucFxDIOZFA1Z_QZwGqblW4zdBljhUjhamssvchu_0,15971
 sky/clouds/vast.py,sha256=1AZaM71dI837oeuMZEXN4muHsH3CKRLjBAPcPy1QqV4,11296
 sky/clouds/vsphere.py,sha256=yRLQESpuSOdfs4KaVTMSQJqf_3mr4VysGqSo1zs1ZtU,12453
-sky/clouds/service_catalog/__init__.py,sha256=Fbm8kKpVTCoIumnY-FWL0X4xMpKu_qwYAxyuVXu4IBQ,14997
+sky/clouds/service_catalog/__init__.py,sha256=WCYMAhek36Ee1HjJL9LP287a70hv2NMA_8plcZDI4Z4,14997
 sky/clouds/service_catalog/aws_catalog.py,sha256=PbYD37rU_8m-Y_5xTglW21ppxI0GecM1sdO1yXuPwHE,13518
 sky/clouds/service_catalog/azure_catalog.py,sha256=5Q51x_WEKvQ2YSgJvZHRH3URlbwIstYuwpjaWW_wJlw,8149
 sky/clouds/service_catalog/common.py,sha256=6TCUE1kg2FvUZXpV2ktrnsRchUYNTGx8rq16m5Nztek,27766
@@ -105,11 +105,11 @@ sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=-JcnVa_LT0kQejcSCnBwYtxhuuaNDPf_Q5oz62p186c,11973
 sky/data/data_utils.py,sha256=ryKUPgNBdeDmGIttqK-J7AKdfc70INTuYH5GOWm3C9g,33581
 sky/data/mounting_utils.py,sha256=ph2p8cYB28FODgxK5ibiD4B4iMD7T3or99zNQaD9HLs,20162
-sky/data/storage.py,sha256=ifNAvrzjS4jFkQHT6sWlATb_lVTQHkgvg9nFcWLq2k4,236674
-sky/data/storage_utils.py,sha256=0r0_I4pCnRYK2vf7syGtvIVyuSeI8w_gPnyxfYGEbcU,12698
+sky/data/storage.py,sha256=85LcC64yxfd5bzTijGZVyMZV41NyzUhOn0xJZieK2Dc,236652
+sky/data/storage_utils.py,sha256=fDEEErxu97XhOtwPdnNBqRukWcfRT4eTBUhrSGrAvsY,13255
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
 sky/jobs/constants.py,sha256=1XiIqdR5dEgGgepLKWkZCRT3MYSsMBR-dO7N4RTsjwg,3088
-sky/jobs/controller.py,sha256=4G1CKI7M7D1BgJLbJMeqzg0iDDv7FR4ObB1BKZFFjhk,29585
+sky/jobs/controller.py,sha256=d5qQYHadesfFgU7-dYtt2trZwyd5IzvlVJeNh5O8OiA,31386
 sky/jobs/recovery_strategy.py,sha256=RLrqq8B1likxTknPzt3_BqO26sFVpoatxzUuGfwc18A,26170
 sky/jobs/scheduler.py,sha256=luQgrCDaDP6bI7oIbaqzxg4qMJtUIVswypnOGUklGtw,13270
 sky/jobs/state.py,sha256=tDULLH6DVs4oKUIKhh0UAn3RzyVGuIUtEq5kW7K1Ojw,44585
@@ -122,7 +122,7 @@ sky/jobs/dashboard/templates/index.html,sha256=NrlTDiEHJDt7sViwWgXUSxVCyVl_IEukE
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 sky/jobs/server/core.py,sha256=IHbGMEEbEdNS_ivIStv3rYVH6XodFgufrzZb3HGF9uo,25257
 sky/jobs/server/dashboard_utils.py,sha256=2Mbx40W1pQqPEPHsSDbHeaF0j5cgyKy-_A9Owdwp_AQ,2315
-sky/jobs/server/server.py,sha256=pLobr0ITZJmhWUX-QrUwpZMgm7qy4_PHFuAnp4Xikac,8386
+sky/jobs/server/server.py,sha256=jk6IopWbn9DbzNzLxM4EcYzWCRnXPXthfNqMDFmtonc,8386
 sky/provision/__init__.py,sha256=LzOo5LjkRXwSf29dUqN14YbjzQu3liXLQcmweTeZ4dE,6457
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
@@ -166,7 +166,7 @@ sky/provision/kubernetes/constants.py,sha256=dZCUV8FOO9Gct80sdqeubKnxeW3CGl-u5mx
 sky/provision/kubernetes/instance.py,sha256=oag17OtuiqU-1RjkgW9NvEpxSGUFIYdI7M61S-YmPu8,50503
 sky/provision/kubernetes/network.py,sha256=AtcOM8wPs_-UlQJhGEQGP6Lh4HIgdx63Y0iWEhP5jyc,12673
 sky/provision/kubernetes/network_utils.py,sha256=6uck1aBkgtm-gGBitU3_hEUp8j14ZuG_4Xo70ReZYXs,11654
-sky/provision/kubernetes/utils.py,sha256=37esnQgQgeXfxwWyMqAo6SXCz0EyVvKECy2DOWISDEo,124630
+sky/provision/kubernetes/utils.py,sha256=D7xfk6RjqLAsERpJMxQ8ozu-D3pRTz_bRszKQeMsVno,127188
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -318,7 +318,7 @@ sky/utils/common.py,sha256=P4oVXFATUYgkruHX92cN12SJBtfb8DiOOYZtbN1kvP0,1927
 sky/utils/common_utils.py,sha256=s5YIo9wtFCwWLfLRW7fCjlC9BzqQKPGatWQjrEyYqpc,31680
 sky/utils/config_utils.py,sha256=VQ2E3DQ2XysD-kul-diSrxn_pXWsDMfKAev91OiJQ1Q,9041
 sky/utils/control_master_utils.py,sha256=iD4M0onjYOdZ2RuxjwMBl4KhafHXJzuHjvqlBUnu-VE,1450
-sky/utils/controller_utils.py,sha256=QUG3CVYLnN5F_Q1Jdbir0Oh9FxImXkdbMI9WanluSLA,49802
+sky/utils/controller_utils.py,sha256=mrmkerYyeu7gsCQ56cB3AjCz0r9WaN7teqXUItA47oQ,49805
 sky/utils/dag_utils.py,sha256=sAus0aL1wtuuFZSDnpO4LY-6WK4u5iJY952oWQzHo3Y,7532
 sky/utils/db_utils.py,sha256=K2-OHPg0FeHCarevMdWe0IWzm6wWumViEeYeJuGoFUE,3747
 sky/utils/env_options.py,sha256=aaD6GoYK0LaZIqjOEZ-R7eccQuiRriW3EuLWtOI5En8,1578
@@ -344,15 +344,15 @@ sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=EQMBC0VUqe3UaiYvwkNq4P6U9bk
 sky/utils/kubernetes/exec_kubeconfig_converter.py,sha256=fE1SnteoxI05EaugnWeV82hXwZTVHmbXsh1aaZAgF3c,2548
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=MBvXJio0PeujZSCXiRKE_pa6HCTiU9qBzR1WrXccVSY,10477
-sky/utils/kubernetes/gpu_labeler.py,sha256=1Mblw9pIWgmI5Kob29k5EEuPmip_JXYLD9Hztq6X4v0,7069
+sky/utils/kubernetes/gpu_labeler.py,sha256=MKsEiWitn-LOG3-OEdBZ-MXJd_L3ztdB9JrhD6D2T1w,6912
 sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=HPVgNt-wbCVPd9dpDFiA7t2mzQLpjXHJ61eiwRbEr-c,10378
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250402.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250402.dist-info/METADATA,sha256=bXDuFvGHbo-NYrZxLyYxZ8OLwzvAttIz0x_fAQwsW_g,18552
-skypilot_nightly-1.0.0.dev20250402.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-skypilot_nightly-1.0.0.dev20250402.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250402.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250402.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250404.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250404.dist-info/METADATA,sha256=HjDx6IDyOIoJ7lFpgxXPO2n8tIyTUh87PO4FAp-ZWcc,18552
+skypilot_nightly-1.0.0.dev20250404.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250404.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250404.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250404.dist-info/RECORD,,