skypilot-nightly 1.0.0.dev20250401__py3-none-any.whl → 1.0.0.dev20250403__py3-none-any.whl

sky/skylet/constants.py

@@ -51,7 +51,7 @@ SKY_RAY_CMD = (f'{SKY_PYTHON_CMD} $([ -s {SKY_RAY_PATH_FILE} ] && '
                f'cat {SKY_RAY_PATH_FILE} 2> /dev/null || which ray)')
 # Separate env for SkyPilot runtime dependencies.
 SKY_REMOTE_PYTHON_ENV_NAME = 'skypilot-runtime'
-SKY_REMOTE_PYTHON_ENV = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
+SKY_REMOTE_PYTHON_ENV: str = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
 ACTIVATE_SKY_REMOTE_PYTHON_ENV = f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate'
 # uv is used for venv and pip, much faster than python implementations.
 SKY_UV_INSTALL_DIR = '"$HOME/.local/bin"'
@@ -60,7 +60,7 @@ SKY_UV_CMD = f'UV_SYSTEM_PYTHON=false {SKY_UV_INSTALL_DIR}/uv'
 SKY_UV_INSTALL_CMD = (f'{SKY_UV_CMD} -V >/dev/null 2>&1 || '
                       'curl -LsSf https://astral.sh/uv/install.sh '
                       f'| UV_INSTALL_DIR={SKY_UV_INSTALL_DIR} sh')
-SKY_UV_PIP_CMD = f'VIRTUAL_ENV={SKY_REMOTE_PYTHON_ENV} {SKY_UV_CMD} pip'
+SKY_UV_PIP_CMD: str = (f'VIRTUAL_ENV={SKY_REMOTE_PYTHON_ENV} {SKY_UV_CMD} pip')
 # Deleting the SKY_REMOTE_PYTHON_ENV_NAME from the PATH to deactivate the
 # environment. `deactivate` command does not work when conda is used.
 DEACTIVATE_SKY_REMOTE_PYTHON_ENV = (
@@ -331,6 +331,12 @@ SKYPILOT_NODE_RANK = f'{SKYPILOT_ENV_VAR_PREFIX}NODE_RANK'
 # known after provisioning.
 SKY_SSH_USER_PLACEHOLDER = 'skypilot:ssh_user'
 
+RCLONE_CONFIG_DIR = '~/.config/rclone'
+RCLONE_CONFIG_PATH = f'{RCLONE_CONFIG_DIR}/rclone.conf'
+RCLONE_LOG_DIR = '~/.sky/rclone_log'
+RCLONE_CACHE_DIR = '~/.cache/rclone'
+RCLONE_CACHE_REFRESH_INTERVAL = 10
+
 # The keys that can be overridden in the `~/.sky/config.yaml` file. The
 # overrides are specified in task YAMLs.
 OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [

sky/task.py

@@ -745,7 +745,7 @@ class Task:
 
         # Evaluate if the task requires FUSE and set the requires_fuse flag
         for _, storage_obj in self.storage_mounts.items():
-            if storage_obj.mode == storage_lib.StorageMode.MOUNT:
+            if storage_obj.mode in storage_lib.MOUNTABLE_STORAGE_MODES:
                 for r in self.resources:
                     r.requires_fuse = True
                 break
@@ -924,7 +924,7 @@ class Task:
                         'Storage mount destination path cannot be cloud storage'
                     )
 
-            if storage_obj.mode == storage_lib.StorageMode.MOUNT:
+            if storage_obj.mode in storage_lib.MOUNTABLE_STORAGE_MODES:
                 # If any storage is using MOUNT mode, we need to enable FUSE in
                 # the resources.
                 for r in self.resources:
@@ -1128,7 +1128,7 @@ class Task:
                         assert storage.name is not None, storage
                         # extract region from rclone.conf
                         cos_region = data_utils.Rclone.get_region_from_rclone(
-                            storage.name, data_utils.Rclone.RcloneClouds.IBM)
+                            storage.name, data_utils.Rclone.RcloneStores.IBM)
                         blob_path = f'cos://{cos_region}/{storage.name}'
                     blob_path = storage.get_bucket_sub_path_prefix(blob_path)
                     self.update_file_mounts({mnt_path: blob_path})
@@ -1268,7 +1268,7 @@ class Task:
 
         # Storage mounting
         for _, storage_mount in self.storage_mounts.items():
-            if storage_mount.mode == storage_lib.StorageMode.MOUNT:
+            if storage_mount.mode in storage_lib.MOUNTABLE_STORAGE_MODES:
                 required_features.add(
                     clouds.CloudImplementationFeatures.STORAGE_MOUNTING)
                 break

sky/utils/controller_utils.py

@@ -916,7 +916,7 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
                 name=bucket_name,
                 source=local_fm_path,
                 persistent=False,
-                mode=storage_lib.StorageMode.MOUNT,
+                mode=storage_lib.DEFAULT_STORAGE_MODE,
                 stores=stores,
                 _is_sky_managed=not bucket_wth_prefix,
                 _bucket_sub_path=file_mounts_tmp_subpath)
@@ -1025,7 +1025,7 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
     # it was handled in step 6.
     updated_mount_storages = {}
     for storage_path, storage_obj in task.storage_mounts.items():
-        if (storage_obj.mode == storage_lib.StorageMode.MOUNT and
+        if (storage_obj.mode in storage_lib.MOUNTABLE_STORAGE_MODES and
                 not storage_obj.source):
             # Construct source URL with first store type and storage name
             # E.g., s3://my-storage-name
@@ -1043,7 +1043,7 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
             new_storage = storage_lib.Storage.from_yaml_config({
                 'source': source,
                 'persistent': storage_obj.persistent,
-                'mode': storage_lib.StorageMode.MOUNT.value,
+                'mode': storage_obj.mode.value,
                 # We enable force delete to allow the controller to delete
                 # the object store in case persistent is set to False.
                 '_force_delete': True

sky/utils/kubernetes/gpu_labeler.py

@@ -3,35 +3,17 @@ import argparse
 import hashlib
 import os
 import subprocess
-from typing import Tuple
+from typing import Optional, Tuple
 
-from kubernetes import client
-from kubernetes import config
 import yaml
 
 import sky
+from sky.adaptors import kubernetes
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.utils import rich_utils
 
 
-def prerequisite_check() -> Tuple[bool, str]:
-    """Checks if kubectl is installed and kubeconfig is set up"""
-    reason = ''
-    prereq_ok = False
-    try:
-        subprocess.run(['kubectl', 'get', 'pods'],
-                       check=True,
-                       capture_output=True)
-        prereq_ok = True
-    except FileNotFoundError:
-        reason = 'kubectl not found. Please install kubectl and try again.'
-    except subprocess.CalledProcessError as e:
-        output = e.output.decode('utf-8')
-        reason = 'Error running kubectl: ' + output
-    return prereq_ok, reason
-
-
-def cleanup() -> Tuple[bool, str]:
+def cleanup(context: Optional[str] = None) -> Tuple[bool, str]:
     """Deletes all Kubernetes resources created by this script
 
     Used to provide idempotency when the script is run multiple times. Also
@@ -42,7 +24,8 @@ def cleanup() -> Tuple[bool, str]:
                    'replicasets,configmaps,secrets,pv,pvc,clusterrole,'
                    'serviceaccount,clusterrolebinding -n kube-system '
                    '-l job=sky-gpu-labeler')
-
+    if context:
+        del_command += f' --context {context}'
     success = False
     reason = ''
     with rich_utils.client_status('Cleaning up existing GPU labeling '
@@ -62,12 +45,24 @@ def get_node_hash(node_name: str):
     return md5_hash[:32]
 
 
-def label():
-    deletion_success, reason = cleanup()
+def label(context: Optional[str] = None):
+    deletion_success, reason = cleanup(context=context)
     if not deletion_success:
         print(reason)
         return
 
+    unlabeled_gpu_nodes = kubernetes_utils.get_unlabeled_accelerator_nodes()
+
+    if not unlabeled_gpu_nodes:
+        print('No unlabeled GPU nodes found in the cluster. If you have '
+              'unlabeled GPU nodes, please ensure that they have the resource '
+              f'`{kubernetes_utils.get_gpu_resource_key()}: <number of GPUs>` '
+              'in their capacity.')
+        return
+
+    print(f'Found {len(unlabeled_gpu_nodes)} '
+          'unlabeled GPU nodes in the cluster')
+
     sky_dir = os.path.dirname(sky.__file__)
     manifest_dir = os.path.join(sky_dir, 'utils/kubernetes')
 
@@ -76,18 +71,17 @@ def label():
         rbac_manifest_path = os.path.join(manifest_dir,
                                           'k8s_gpu_labeler_setup.yaml')
         try:
-            subprocess.check_output(
-                ['kubectl', 'apply', '-f', rbac_manifest_path])
+            apply_command = ['kubectl', 'apply', '-f', rbac_manifest_path]
+            if context:
+                apply_command += ['--context', context]
+            subprocess.check_output(apply_command)
         except subprocess.CalledProcessError as e:
             output = e.output.decode('utf-8')
             print('Error setting up GPU labeling: ' + output)
             return
 
     with rich_utils.client_status('Creating GPU labeler jobs'):
-        config.load_kube_config()
-
-        v1 = client.CoreV1Api()
-        batch_v1 = client.BatchV1Api()
+        batch_v1 = kubernetes.batch_api(context=context)
         # Load the job manifest
         job_manifest_path = os.path.join(manifest_dir,
                                          'k8s_gpu_labeler_job.yaml')
@@ -95,20 +89,10 @@ def label():
         with open(job_manifest_path, 'r', encoding='utf-8') as file:
             job_manifest = yaml.safe_load(file)
 
-        # Iterate over nodes
-        nodes = v1.list_node().items
-
-        # Get the list of nodes with GPUs
-        gpu_nodes = []
-        for node in nodes:
-            if kubernetes_utils.get_gpu_resource_key() in node.status.capacity:
-                gpu_nodes.append(node)
-
-        print(f'Found {len(gpu_nodes)} GPU nodes in the cluster')
-
         # Check if the 'nvidia' RuntimeClass exists
         try:
-            nvidia_exists = kubernetes_utils.check_nvidia_runtime_class()
+            nvidia_exists = kubernetes_utils.check_nvidia_runtime_class(
+                context=context)
         except Exception as e:  # pylint: disable=broad-except
             print('Error occurred while checking for nvidia RuntimeClass: '
                   f'{str(e)}')
@@ -125,7 +109,7 @@ def label():
         else:
             print('Using default RuntimeClass for GPU labeling.')
 
-        for node in gpu_nodes:
+        for node in unlabeled_gpu_nodes:
             node_name = node.metadata.name
 
             # Modify the job manifest for the current node
@@ -139,17 +123,16 @@ def label():
             # Create the job for this node`
             batch_v1.create_namespaced_job(namespace, job_manifest)
             print(f'Created GPU labeler job for node {node_name}')
-    if not gpu_nodes:
-        print('No GPU nodes found in the cluster. If you have GPU nodes, '
-              'please ensure that they have the label '
-              f'`{kubernetes_utils.get_gpu_resource_key()}: <number of GPUs>`')
-    else:
-        print('GPU labeling started - this may take 10 min or more to complete.'
-              '\nTo check the status of GPU labeling jobs, run '
-              '`kubectl get jobs -n kube-system -l job=sky-gpu-labeler`'
-              '\nYou can check if nodes have been labeled by running '
-              '`kubectl describe nodes` and looking for labels of the format '
-              '`skypilot.co/accelerator: <gpu_name>`. ')
+
+    context_str = f' --context {context}' if context else ''
+    print(f'GPU labeling started - this may take 10 min or more to complete.'
+          '\nTo check the status of GPU labeling jobs, run '
+          f'`kubectl get jobs -n kube-system '
+          f'-l job=sky-gpu-labeler{context_str}`'
+          '\nYou can check if nodes have been labeled by running '
+          f'`kubectl describe nodes{context_str}` '
+          'and looking for labels of the format '
+          '`skypilot.co/accelerator: <gpu_name>`. ')
 
 
 def main():
@@ -165,18 +148,24 @@ def main():
                         action='store_true',
                         help='delete all GPU labeler resources in the '
                         'Kubernetes cluster.')
+    parser.add_argument('--context',
+                        type=str,
+                        help='the context to use for the Kubernetes cluster.')
     args = parser.parse_args()
+    context = None
+    if args.context:
+        context = args.context
 
     # Check if kubectl is installed and kubeconfig is set up
-    prereq_ok, reason = prerequisite_check()
+    prereq_ok, reason = kubernetes_utils.check_credentials(context=context)
     if not prereq_ok:
         print(reason)
         return
 
     if args.cleanup:
-        cleanup()
+        cleanup(context=context)
     else:
-        label()
+        label(context=context)
 
 
 if __name__ == '__main__':

{skypilot_nightly-1.0.0.dev20250401.dist-info → skypilot_nightly-1.0.0.dev20250403.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250401
+Version: 1.0.0.dev20250403
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -193,10 +193,6 @@ Dynamic: summary
     <img alt="Downloads" src="https://img.shields.io/pypi/dm/skypilot">
   </a>
 
-  <a href="https://buildkite.com/skypilot-1/full-smoke-tests-run">
-    <img alt="Smoke Tests" src="https://badge.buildkite.com/d3aa9d2370e4a9ac4fb5e210381f955082a63a9a46673b197a.svg?theme=github&branch=master">
-  </a>
-
 </p>
 
 <h3 align="center">

{skypilot_nightly-1.0.0.dev20250401.dist-info → skypilot_nightly-1.0.0.dev20250403.dist-info}/RECORD

@@ -1,20 +1,20 @@
-sky/__init__.py,sha256=tefeLNQrAbiS-ZFeb8jks_yUhK3yPUm9L-cZXxqdZgs,6428
+sky/__init__.py,sha256=jAriaLe3W4tnwwVPeiJ2hAxmMnsB0NRcf-MQ2yLMyQk,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=ND011K_-Ud1dVZF37A9KrwYir_ihJXcHc7iDWmuBc8Q,22872
-sky/check.py,sha256=iMGuM7yjUPRgDHc13Pf1_LSybBqIexq-6aXfnVcaI54,15898
-sky/cli.py,sha256=xXTc8ktbJy3DNmjud3Fwztwy9DajMn0vJ7J6snyfxmc,222415
-sky/cloud_stores.py,sha256=yo8xQGr8iabYGmn0E6o6jHsfJiomXCTCcR-WMLnNYTQ,26718
+sky/check.py,sha256=oktScSPsHIyO7ZrVHy3QaybB6-s_D6eMEjmICAiUtDo,15902
+sky/cli.py,sha256=Zcio2ak6zX_5_N_lshDUqCvoV6NEOmGS6Tp6AgS9VAk,222446
+sky/cloud_stores.py,sha256=cmKdSoB4bmwrd-Z1NCZBFb6IIJt0jKVxkGPoX86280s,26606
 sky/core.py,sha256=G3n6z0dyvoU4FJVGnnTu3kFdu_EtQC1l57er5voRAX0,47926
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
 sky/exceptions.py,sha256=EodMj6P0x0JYBiioLNuPoFMj12HzlQTQpfXNX6zHBoA,16837
 sky/execution.py,sha256=9L8NFOXNphtabnsL7mHGPJeGdw4n6gIIUEOzjW7CEHw,28294
-sky/global_user_state.py,sha256=uGmwtUL-HVKGqEwlEvIXFcQ_w2IxGE4msnffXUxCIGI,33574
-sky/models.py,sha256=4xSW05BdDPEjW8Ubvj3VlVOVnzv0TbrolsFvR5R5v1U,638
+sky/global_user_state.py,sha256=7HADn0mY-0omf5RbpmAZ88bjySzqKcmiD1MEmkrCoNU,33754
+sky/models.py,sha256=4Hq_JNpeQyvRlde_mf1T8H5iDS362TRORYXjtk2nmT4,735
 sky/optimizer.py,sha256=uutziDwxyq-f5n31YXgCF9fmqx5vndIU8g_avvGWuGc,58532
 sky/resources.py,sha256=2qc5U09MFDaJjI1dHcThcRodpMGY7HyXzQn8eC4lvbE,72402
 sky/sky_logging.py,sha256=pID2RINjH62n7SZpv70DuN8BSFYdCfTJ2ScGQpVmugg,5725
 sky/skypilot_config.py,sha256=CdaIbPL_7ECG5laOARca4p9df_6NLhT-bO8WnalxZAY,8839
-sky/task.py,sha256=B3eWFaTPEsAlHtXAt2ecm5uoKGv9V6v7rR2S-5YTnwo,55691
+sky/task.py,sha256=kepCHV_evVg9EKnXaKW9Stg1bQAZlaF4UQocwlR3M3g,55709
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=iH55Cm6eXWwufAG0Dgk7LTQcADawNa3ELrBH1m6yuSY,7617
 sky/adaptors/azure.py,sha256=r8xkjRgZZQfsSExNdguFa6c5fCLcUhZrFV8zs62VExo,21986
@@ -25,7 +25,7 @@ sky/adaptors/do.py,sha256=dJ0BYbkQoUWVu6_9Pxq3fOu6PngjZyyCQzgjnODXLCA,777
 sky/adaptors/docker.py,sha256=_kzpZ0fkWHqqQAVVl0llTsCE31KYz3Sjn8psTBQHVkA,468
 sky/adaptors/gcp.py,sha256=oEb9jClEtApw6PQnxdxDYxOCYsedvM3aiko1EW1FDVo,3501
 sky/adaptors/ibm.py,sha256=7YbHrWbYcZsJDgxMBNZr1yBI03mjs_C3pnCTCz-MNtQ,5068
-sky/adaptors/kubernetes.py,sha256=BfbUYMc9AMPvtoPX8f3qVI_tY2mLfGEJI9wSXLJftQs,6678
+sky/adaptors/kubernetes.py,sha256=pbrMAO_jaHFPdIKryNALY0jvXujC3nUKnYlq0Sj6I2M,6884
 sky/adaptors/nebius.py,sha256=B3KxPYhTJA3zjQ7x9k8_sqFTAkPYXldnM7PAuwy02gA,7512
 sky/adaptors/oci.py,sha256=LfMSFUmkkNT6Yoz9FZHNl6UFSg4X1lJO4-x4ZbDdXTs,2831
 sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
@@ -34,7 +34,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
 sky/backends/backend_utils.py,sha256=ndY4IPs1F9QovyiKAnB1FNYGWm52_ylwf_K7wY50cv0,134922
-sky/backends/cloud_vm_ray_backend.py,sha256=rWz9eUSq9TauA0HWhwYWm2DaGT2p8yDRxXvigcgx-uA,249578
+sky/backends/cloud_vm_ray_backend.py,sha256=uRz6XUvrTHGsF6LYIuSjFBUP51BSD5_yTj4R5-3DLo0,251922
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=meypuMaygSXXjGdXfq6dhWl-OrpAybg9KVRoup4D0wU,9098
@@ -43,7 +43,7 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=7rf-iHt6RXZ_pnBBWOMwcdodHQW69x27xNyx0yVog1U,26385
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=xXTc8ktbJy3DNmjud3Fwztwy9DajMn0vJ7J6snyfxmc,222415
+sky/client/cli.py,sha256=Zcio2ak6zX_5_N_lshDUqCvoV6NEOmGS6Tp6AgS9VAk,222446
 sky/client/common.py,sha256=YMgYoSYlV3B_scfCIJRKHvDq6JKvnSvRZDhlRjRmzu0,14780
 sky/client/sdk.py,sha256=36tvJou2IaG8H5lrqMicv-gSJn6VUDHUtB7V20t4lA8,68620
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
@@ -55,7 +55,7 @@ sky/clouds/do.py,sha256=P38l4otp2AuDReUH9Ii621ht9s-NIyb7-R37jbtjHk8,11580
 sky/clouds/fluidstack.py,sha256=jIqW1MLe55MVME1PATZm8e6_FsiTnJawW7OdytPW0aM,12666
 sky/clouds/gcp.py,sha256=sUJ9LXUnMxYm6OYZ5P-z1dJHxgVILuC3OW3eFSTNCv8,56919
 sky/clouds/ibm.py,sha256=XtuPN8QgrwJdb1qb_b-7KwAE2tf_N9wh9eEfi2tcg-s,22013
-sky/clouds/kubernetes.py,sha256=ivk6GlAbhj3oDaFfcIzwz2x7qRBrzShglV4FfzikqS0,36169
+sky/clouds/kubernetes.py,sha256=VuPHstRysRZH0UTYZIExY2Gtd2ItQn9I5EboqxyuvV0,36717
 sky/clouds/lambda_cloud.py,sha256=rR2YrZ6flEbKKpQAm60eKNjiMDYvH2hqzaCo3Hx4Ffw,12916
 sky/clouds/nebius.py,sha256=4D7C2NQYI-BNhXWNOyAXNAZj7-5nN33VQW1sxfSGt9w,14662
 sky/clouds/oci.py,sha256=YO4kjSsHBmAVH4z1TuVP72zfmC0BXte4E0xIyZir9N4,27622
@@ -103,13 +103,13 @@ sky/clouds/utils/oci_utils.py,sha256=KbmwTWlEO3stcjM4jN2agSFi2w9WNc2NYA9Gr5jMAvE
 sky/clouds/utils/scp_utils.py,sha256=MqawUhhFHHxVnn29nOI4gJ_nF665ich4Po7bsy1afsA,15948
 sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=-JcnVa_LT0kQejcSCnBwYtxhuuaNDPf_Q5oz62p186c,11973
-sky/data/data_utils.py,sha256=m1rrspfkgux7awWwTvQLFBjVix2KbKgRwgnhJDv3pcw,29895
-sky/data/mounting_utils.py,sha256=gLplBNKRQvVUBdkIRZO-wW4wpfY_F8xmkMyMfSZasTQ,16832
-sky/data/storage.py,sha256=AfSZ5gAtfpBXmo5mXHYk1thvq8l-I3CZbcax8C4OPmE,233456
-sky/data/storage_utils.py,sha256=0r0_I4pCnRYK2vf7syGtvIVyuSeI8w_gPnyxfYGEbcU,12698
+sky/data/data_utils.py,sha256=ryKUPgNBdeDmGIttqK-J7AKdfc70INTuYH5GOWm3C9g,33581
+sky/data/mounting_utils.py,sha256=ph2p8cYB28FODgxK5ibiD4B4iMD7T3or99zNQaD9HLs,20162
+sky/data/storage.py,sha256=85LcC64yxfd5bzTijGZVyMZV41NyzUhOn0xJZieK2Dc,236652
+sky/data/storage_utils.py,sha256=fDEEErxu97XhOtwPdnNBqRukWcfRT4eTBUhrSGrAvsY,13255
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
 sky/jobs/constants.py,sha256=1XiIqdR5dEgGgepLKWkZCRT3MYSsMBR-dO7N4RTsjwg,3088
-sky/jobs/controller.py,sha256=4G1CKI7M7D1BgJLbJMeqzg0iDDv7FR4ObB1BKZFFjhk,29585
+sky/jobs/controller.py,sha256=d5qQYHadesfFgU7-dYtt2trZwyd5IzvlVJeNh5O8OiA,31386
 sky/jobs/recovery_strategy.py,sha256=RLrqq8B1likxTknPzt3_BqO26sFVpoatxzUuGfwc18A,26170
 sky/jobs/scheduler.py,sha256=luQgrCDaDP6bI7oIbaqzxg4qMJtUIVswypnOGUklGtw,13270
 sky/jobs/state.py,sha256=tDULLH6DVs4oKUIKhh0UAn3RzyVGuIUtEq5kW7K1Ojw,44585
@@ -122,7 +122,7 @@ sky/jobs/dashboard/templates/index.html,sha256=NrlTDiEHJDt7sViwWgXUSxVCyVl_IEukE
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 sky/jobs/server/core.py,sha256=IHbGMEEbEdNS_ivIStv3rYVH6XodFgufrzZb3HGF9uo,25257
 sky/jobs/server/dashboard_utils.py,sha256=2Mbx40W1pQqPEPHsSDbHeaF0j5cgyKy-_A9Owdwp_AQ,2315
-sky/jobs/server/server.py,sha256=pLobr0ITZJmhWUX-QrUwpZMgm7qy4_PHFuAnp4Xikac,8386
+sky/jobs/server/server.py,sha256=jk6IopWbn9DbzNzLxM4EcYzWCRnXPXthfNqMDFmtonc,8386
 sky/provision/__init__.py,sha256=LzOo5LjkRXwSf29dUqN14YbjzQu3liXLQcmweTeZ4dE,6457
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
@@ -166,7 +166,7 @@ sky/provision/kubernetes/constants.py,sha256=dZCUV8FOO9Gct80sdqeubKnxeW3CGl-u5mx
 sky/provision/kubernetes/instance.py,sha256=oag17OtuiqU-1RjkgW9NvEpxSGUFIYdI7M61S-YmPu8,50503
 sky/provision/kubernetes/network.py,sha256=AtcOM8wPs_-UlQJhGEQGP6Lh4HIgdx63Y0iWEhP5jyc,12673
 sky/provision/kubernetes/network_utils.py,sha256=6uck1aBkgtm-gGBitU3_hEUp8j14ZuG_4Xo70ReZYXs,11654
-sky/provision/kubernetes/utils.py,sha256=37esnQgQgeXfxwWyMqAo6SXCz0EyVvKECy2DOWISDEo,124630
+sky/provision/kubernetes/utils.py,sha256=IR2AKbZA-ZuKtLRuqHZsiA1N2HmQTJKz5IqOQIVEk4U,125753
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -232,14 +232,14 @@ sky/serve/server/server.py,sha256=gQGVU9nHYdGbaLhGjIUNIYn4xwKjRASRJkiiTL5AI1Y,32
 sky/server/__init__.py,sha256=MPPBqFzXz6Jv5QSk6td_IcvnfXfNErDZVcizu4MLRow,27
 sky/server/common.py,sha256=6FQ-2X4AagshuXNxKxjgFRKT_mtgTGh7kG1GOJ2tZIM,19185
 sky/server/constants.py,sha256=_ZNrxYh8vmgbf3DmkGDduxjvO2y43ZSPTkH5rCNsVjU,770
-sky/server/server.py,sha256=b0D1lxZgozC1ny7xDf1dVAMQaF5u386PT2QETa9dmIk,44479
+sky/server/server.py,sha256=7aK3Nv_aHIYX2ZB_NFy-5Cn-DbSxUPMeiWaDY_2DLXE,44674
 sky/server/stream_utils.py,sha256=4JMHgtoXPpCT8JwtqyUcDQ9IdZFir9om0JaCRr8rvbQ,5849
 sky/server/uvicorn.py,sha256=wajwPHJ3IEEP3GMNOCc0S81-1v2qT5F-ejUkLFVhUzk,2953
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/event_loop.py,sha256=OhpPbuce65bbjpGRlcJa78AVnYSm08SzFKt70ypCUuQ,1211
 sky/server/requests/executor.py,sha256=wW-8s8APEakYRJKNiWIhHg2vrn2UUteseUTsqm9q3Fg,21693
-sky/server/requests/payloads.py,sha256=K9CzEqRLDdU2R7s3yxPDD2vC-h6beD_RPdSRpjoQzu4,16529
+sky/server/requests/payloads.py,sha256=3sF36Z9_PLzpEncW0AplJtOz-_nsn5PJaM5lS-3Y8bw,16558
 sky/server/requests/preconditions.py,sha256=ipxIb_3JXG6S3-ymcOdqQNb7VDvoPqADxu9ZK7-nQWc,7179
 sky/server/requests/requests.py,sha256=9ovdQE-zv_Mvc6IbGATHVyQlOxSKjg_OankZbgDVGeE,21338
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -255,7 +255,7 @@ sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=kGUnHm-jpF4zl3UJfB-4pnoldWpnVeR96WwYGSw7em0,4630
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=1Yzzb6bwECy3gP7BYaMnz2uve5x0bChOn-84pwJgSng,18333
+sky/skylet/constants.py,sha256=uGhQdxdxO42rwQyu_gg0aeiSFIXxUK2jPoDOaGaMb2M,18550
 sky/skylet/events.py,sha256=pnV3ZiwWhXqTHpU5B5Y9Xwam_7FQDI6IrxgSx7X_NVA,12743
 sky/skylet/job_lib.py,sha256=FUpaCSg5yl4ptFb3_GOeUunHTz4qJvwJJjeqrpiRUTk,44470
 sky/skylet/log_lib.py,sha256=DzOrgY8C7RdEMLC9O9kEKV-iLMb9wVMPSnDha8eMx28,20900
@@ -318,7 +318,7 @@ sky/utils/common.py,sha256=P4oVXFATUYgkruHX92cN12SJBtfb8DiOOYZtbN1kvP0,1927
 sky/utils/common_utils.py,sha256=s5YIo9wtFCwWLfLRW7fCjlC9BzqQKPGatWQjrEyYqpc,31680
 sky/utils/config_utils.py,sha256=VQ2E3DQ2XysD-kul-diSrxn_pXWsDMfKAev91OiJQ1Q,9041
 sky/utils/control_master_utils.py,sha256=iD4M0onjYOdZ2RuxjwMBl4KhafHXJzuHjvqlBUnu-VE,1450
-sky/utils/controller_utils.py,sha256=Bfwd7yWgcMgzwKz_B39UMGDIP6SQ2Xelqertf7CtwlI,49809
+sky/utils/controller_utils.py,sha256=mrmkerYyeu7gsCQ56cB3AjCz0r9WaN7teqXUItA47oQ,49805
 sky/utils/dag_utils.py,sha256=sAus0aL1wtuuFZSDnpO4LY-6WK4u5iJY952oWQzHo3Y,7532
 sky/utils/db_utils.py,sha256=K2-OHPg0FeHCarevMdWe0IWzm6wWumViEeYeJuGoFUE,3747
 sky/utils/env_options.py,sha256=aaD6GoYK0LaZIqjOEZ-R7eccQuiRriW3EuLWtOI5En8,1578
@@ -344,15 +344,15 @@ sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=EQMBC0VUqe3UaiYvwkNq4P6U9bk
 sky/utils/kubernetes/exec_kubeconfig_converter.py,sha256=fE1SnteoxI05EaugnWeV82hXwZTVHmbXsh1aaZAgF3c,2548
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=MBvXJio0PeujZSCXiRKE_pa6HCTiU9qBzR1WrXccVSY,10477
-sky/utils/kubernetes/gpu_labeler.py,sha256=9VZkC4UuR4vtE6TAz5P0i279Rn8KEaHE35JA9OVHxqI,7006
+sky/utils/kubernetes/gpu_labeler.py,sha256=MKsEiWitn-LOG3-OEdBZ-MXJd_L3ztdB9JrhD6D2T1w,6912
 sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=HPVgNt-wbCVPd9dpDFiA7t2mzQLpjXHJ61eiwRbEr-c,10378
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250401.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250401.dist-info/METADATA,sha256=xgKQsVQCmVNo7_dZ-gCLaDm24D6Iqx1VezZQcF0wPnk,18771
-skypilot_nightly-1.0.0.dev20250401.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-skypilot_nightly-1.0.0.dev20250401.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250401.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250401.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250403.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250403.dist-info/METADATA,sha256=YkgmWuWgAbuQRFP3Zfk3QpnqS_0KN7x50GuSoVI9JwU,18552
+skypilot_nightly-1.0.0.dev20250403.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250403.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250403.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250403.dist-info/RECORD,,