skypilot-nightly 1.0.0.dev20241011__py3-none-any.whl → 1.0.0.dev20241013__py3-none-any.whl

sky/utils/log_utils.py

@@ -9,6 +9,7 @@ import prettytable
 
 from sky import sky_logging
 from sky.utils import rich_utils
+from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -37,30 +38,34 @@ class RayUpLineProcessor(LineProcessor):
         RUNTIME_SETUP = 1
         PULLING_DOCKER_IMAGES = 2
 
+    def __init__(self, log_path: str):
+        self.log_path = log_path
+
     def __enter__(self) -> None:
         self.state = self.ProvisionStatus.LAUNCH
-        self.status_display = rich_utils.safe_status('[bold cyan]Launching')
+        self.status_display = rich_utils.safe_status(
+            ux_utils.spinner_message('Launching', self.log_path))
         self.status_display.start()
 
     def process_line(self, log_line: str) -> None:
         if ('Success.' in log_line and
                 self.state == self.ProvisionStatus.LAUNCH):
-            logger.info(f'{colorama.Fore.GREEN}Head node is up.'
-                        f'{colorama.Style.RESET_ALL}')
+            logger.info('  Head VM is up.')
             self.status_display.update(
-                '[bold cyan]Launching - Preparing SkyPilot runtime')
+                ux_utils.spinner_message(
+                    'Launching - Preparing SkyPilot runtime', self.log_path))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
         if ('Pulling from' in log_line and
                 self.state == self.ProvisionStatus.RUNTIME_SETUP):
             self.status_display.update(
-                '[bold cyan]Launching - Pulling docker images')
+                ux_utils.spinner_message(
+                    'Launching - Initializing docker container', self.log_path))
             self.state = self.ProvisionStatus.PULLING_DOCKER_IMAGES
         if ('Status: Downloaded newer image' in log_line and
                 self.state == self.ProvisionStatus.PULLING_DOCKER_IMAGES):
-            logger.info(f'{colorama.Fore.GREEN}Docker image is downloaded.'
-                        f'{colorama.Style.RESET_ALL}')
             self.status_display.update(
-                '[bold cyan]Launching - Preparing SkyPilot runtime')
+                ux_utils.spinner_message(
+                    'Launching - Preparing SkyPilot runtime', self.log_path))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
 
     def __exit__(self, except_type: Optional[Type[BaseException]],
@@ -73,9 +78,10 @@ class RayUpLineProcessor(LineProcessor):
 class SkyLocalUpLineProcessor(LineProcessor):
     """A processor for `sky local up` log lines."""
 
-    def __enter__(self) -> None:
-        status = rich_utils.safe_status('[bold cyan]Creating local cluster - '
-                                        'initializing Kubernetes')
+    def __enter__(self):
+        status = rich_utils.safe_status(
+            ux_utils.spinner_message('Creating local cluster - '
+                                     'initializing Kubernetes'))
         self.status_display = status
         self.status_display.start()
 
@@ -84,31 +90,37 @@ class SkyLocalUpLineProcessor(LineProcessor):
             logger.info(f'{colorama.Fore.GREEN}Kubernetes is running.'
                         f'{colorama.Style.RESET_ALL}')
         if 'Installing NVIDIA GPU operator...' in log_line:
-            self.status_display.update('[bold cyan]Creating local cluster - '
-                                       'Installing NVIDIA GPU operator')
+            self.status_display.update(
+                ux_utils.spinner_message('Creating local cluster - '
+                                         'Installing NVIDIA GPU operator'))
         if 'Starting wait for GPU operator installation...' in log_line:
             self.status_display.update(
-                '[bold cyan]Creating local cluster - '
-                'waiting for NVIDIA GPU operator installation to complete')
+                ux_utils.spinner_message(
+                    'Creating local cluster - '
+                    'waiting for NVIDIA GPU operator installation to complete'))
             logger.info('To check NVIDIA GPU operator status, '
                         'see pods: kubectl get pods -n gpu-operator')
         if 'GPU operator installed' in log_line:
             logger.info(f'{colorama.Fore.GREEN}NVIDIA GPU Operator installed.'
                         f'{colorama.Style.RESET_ALL}')
         if 'Pulling SkyPilot GPU image...' in log_line:
-            self.status_display.update('[bold cyan]Creating local cluster - '
-                                       'pulling and loading SkyPilot GPU image')
+            self.status_display.update(
+                ux_utils.spinner_message(
+                    'Creating local cluster - '
+                    'pulling and loading SkyPilot GPU image'))
         if 'SkyPilot GPU image loaded into kind cluster' in log_line:
             logger.info(f'{colorama.Fore.GREEN}SkyPilot GPU image pulled.'
                         f'{colorama.Style.RESET_ALL}')
         if 'Labelling nodes with GPUs...' in log_line:
-            self.status_display.update('[bold cyan]Creating local cluster - '
-                                       'launching GPU labelling jobs')
+            self.status_display.update(
+                ux_utils.spinner_message('Creating local cluster - '
+                                         'launching GPU labelling jobs'))
         if ('Starting wait for SkyPilot GPU labeling jobs to complete'
                 in log_line):
             self.status_display.update(
-                '[bold cyan]Creating local cluster - '
-                'waiting for GPU labelling jobs to complete')
+                ux_utils.spinner_message(
+                    'Creating local cluster - '
+                    'waiting for GPU labelling jobs to complete'))
             logger.info(
                 'To check GPU labelling status, see jobs: '
                 'kubectl get jobs -n kube-system -l job=sky-gpu-labeler')
@@ -116,14 +128,17 @@ class SkyLocalUpLineProcessor(LineProcessor):
             logger.info(f'{colorama.Fore.GREEN}GPU labelling complete.'
                         f'{colorama.Style.RESET_ALL}')
         if 'Pulling SkyPilot CPU image...' in log_line:
-            self.status_display.update('[bold cyan]Creating local cluster - '
-                                       'pulling and loading SkyPilot CPU image')
+            self.status_display.update(
+                ux_utils.spinner_message(
+                    'Creating local cluster - '
+                    'pulling and loading SkyPilot CPU image'))
         if 'SkyPilot CPU image loaded into kind cluster' in log_line:
             logger.info(f'{colorama.Fore.GREEN}SkyPilot CPU image pulled.'
                         f'{colorama.Style.RESET_ALL}')
         if 'Starting installation of Nginx Ingress Controller...' in log_line:
             self.status_display.update(
-                '[bold cyan]Creating Nginx Ingress Controller')
+                ux_utils.spinner_message('Creating local cluster - '
+                                         'creating Nginx Ingress Controller'))
         if 'Nginx Ingress Controller installed' in log_line:
             logger.info(
                 f'{colorama.Fore.GREEN}Nginx Ingress Controller installed.'

sky/utils/resources_utils.py

@@ -6,6 +6,8 @@ import re
 import typing
 from typing import List, Optional, Set
 
+from sky import skypilot_config
+from sky.clouds import cloud_registry
 from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
@@ -177,3 +179,24 @@ class FeasibleResources:
     resources_list: List['resources_lib.Resources']
     fuzzy_candidate_list: List[str]
     hint: Optional[str]
+
+
+def need_to_query_reservations() -> bool:
+    """Checks if we need to query reservations from cloud APIs.
+
+    We need to query reservations if:
+    - The cloud has specific reservations.
+    - The cloud prioritizes reservations over on-demand instances.
+
+    This is useful to skip the potentially expensive reservation query for
+    clouds that do not use reservations.
+    """
+    for cloud_str in cloud_registry.CLOUD_REGISTRY.keys():
+        cloud_specific_reservations = skypilot_config.get_nested(
+            (cloud_str, 'specific_reservations'), None)
+        cloud_prioritize_reservations = skypilot_config.get_nested(
+            (cloud_str, 'prioritize_reservations'), False)
+        if (cloud_specific_reservations is not None or
+                cloud_prioritize_reservations):
+            return True
+    return False

sky/utils/rich_utils.py

@@ -5,8 +5,9 @@ from typing import Union
 
 import rich.console as rich_console
 
-console = rich_console.Console()
+console = rich_console.Console(soft_wrap=True)
 _status = None
+_status_nesting_level = 0
 
 _logging_lock = threading.RLock()
 
@@ -30,19 +31,68 @@ class _NoOpConsoleStatus:
         pass
 
 
+class _RevertibleStatus:
+    """A wrapper for status that can revert to previous message after exit."""
+
+    def __init__(self, message: str):
+        if _status is not None:
+            self.previous_message = _status.status
+        else:
+            self.previous_message = None
+        self.message = message
+
+    def __enter__(self):
+        global _status_nesting_level
+        _status.update(self.message)
+        _status_nesting_level += 1
+        _status.__enter__()
+        return _status
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        global _status_nesting_level, _status
+        _status_nesting_level -= 1
+        if _status_nesting_level <= 0:
+            _status_nesting_level = 0
+            if _status is not None:
+                _status.__exit__(exc_type, exc_val, exc_tb)
+                _status = None
+        else:
+            _status.update(self.previous_message)
+
+    def update(self, *args, **kwargs):
+        _status.update(*args, **kwargs)
+
+    def stop(self):
+        _status.stop()
+
+    def start(self):
+        _status.start()
+
+
 def safe_status(msg: str) -> Union['rich_console.Status', _NoOpConsoleStatus]:
     """A wrapper for multi-threaded console.status."""
     from sky import sky_logging  # pylint: disable=import-outside-toplevel
+    global _status
     if (threading.current_thread() is threading.main_thread() and
             not sky_logging.is_silent()):
-        global _status
         if _status is None:
-            _status = console.status(msg)
-        _status.update(msg)
-        return _status
+            _status = console.status(msg, refresh_per_second=8)
+        return _RevertibleStatus(msg)
     return _NoOpConsoleStatus()
 
 
+def stop_safe_status():
+    """Stops all nested statuses.
+
+    This is useful when we need to stop all statuses, e.g., when we are going to
+    stream logs from user program and do not want it to interfere with the
+    spinner display.
+    """
+    if (threading.current_thread() is threading.main_thread() and
+            _status is not None):
+        _status.stop()
+
+
 def force_update_status(msg: str):
     """Update the status message even if sky_logging.is_silent() is true."""
     if (threading.current_thread() is threading.main_thread() and

sky/utils/ux_utils.py

@@ -1,9 +1,12 @@
 """Utility functions for UX."""
 import contextlib
+import os
 import sys
 import traceback
-from typing import Callable
+import typing
+from typing import Callable, Optional, Union
 
+import colorama
 import rich.console as rich_console
 
 from sky import sky_logging
@@ -11,11 +14,25 @@ from sky.utils import common_utils
 from sky.utils import env_options
 from sky.utils import ux_utils
 
+if typing.TYPE_CHECKING:
+    import pathlib
+
 console = rich_console.Console()
 
+INDENT_SYMBOL = f'{colorama.Style.DIM}├── {colorama.Style.RESET_ALL}'
+INDENT_LAST_SYMBOL = f'{colorama.Style.DIM}└── {colorama.Style.RESET_ALL}'
+
+# Console formatting constants
+BOLD = '\033[1m'
+RESET_BOLD = '\033[0m'
+
+# Log path hint in the spinner during launching
+_LOG_PATH_HINT = (f'{colorama.Style.DIM}View logs at: {{log_path}}'
+                  f'{colorama.Style.RESET_ALL}')
+
 
 def console_newline():
-    """Print a newline to the console using rich.
+    """Prints a newline to the console using rich.
 
     Useful when catching exceptions inside console.status()
     """
@@ -50,7 +67,7 @@ def print_exception_no_traceback():
 
 @contextlib.contextmanager
 def enable_traceback():
-    """Revert the effect of print_exception_no_traceback().
+    """Reverts the effect of print_exception_no_traceback().
 
     This is used for usage_lib to collect the full traceback.
     """
@@ -61,7 +78,7 @@ def enable_traceback():
 
 
 class RedirectOutputForProcess:
-    """Redirect stdout and stderr to a file.
+    """Redirects stdout and stderr to a file.
 
     This class enabled output redirect for multiprocessing.Process.
     Example usage:
@@ -102,3 +119,45 @@ class RedirectOutputForProcess:
                 with ux_utils.enable_traceback():
                     logger.error(f'  Traceback:\n{traceback.format_exc()}')
                 raise
+
+
+def starting_message(message: str) -> str:
+    """Gets the starting message for the given message."""
+    return f'⚙︎ {message}'
+
+
+def log_path_hint(log_path: Union[str, 'pathlib.Path']) -> str:
+    """Gets the log path hint for the given log path."""
+    log_path = str(log_path)
+    expanded_home = os.path.expanduser('~')
+    if log_path.startswith(expanded_home):
+        log_path = '~' + log_path[len(expanded_home):]
+    return _LOG_PATH_HINT.format(log_path=log_path)
+
+
+def finishing_message(
+        message: str,
+        log_path: Optional[Union[str, 'pathlib.Path']] = None) -> str:
+    """Gets the finishing message for the given message."""
+    success_prefix = (f'{colorama.Fore.GREEN}✓ {message}'
+                      f'{colorama.Style.RESET_ALL}')
+    if log_path is None:
+        return success_prefix
+    path_hint = log_path_hint(log_path)
+    return f'{success_prefix}  {path_hint}'
+
+
+def retry_message(message: str) -> str:
+    """Gets the retry message for the given message."""
+    return f'{colorama.Fore.YELLOW}↺{colorama.Style.RESET_ALL} {message}'
+
+
+def spinner_message(
+        message: str,
+        log_path: Optional[Union[str, 'pathlib.Path']] = None) -> str:
+    """Gets the spinner message for the given message and log path."""
+    colored_spinner = f'[bold cyan]{message}[/]'
+    if log_path is None:
+        return colored_spinner
+    path_hint = log_path_hint(log_path)
+    return f'{colored_spinner}  {path_hint}'

{skypilot_nightly-1.0.0.dev20241011.dist-info → skypilot_nightly-1.0.0.dev20241013.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241011
+Version: 1.0.0.dev20241013
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241011.dist-info → skypilot_nightly-1.0.0.dev20241013.dist-info}/RECORD

@@ -1,25 +1,25 @@
-sky/__init__.py,sha256=4R-I1YE1smfwcbO4zosKE8PEWzVuCTgo37088CS77fo,5854
+sky/__init__.py,sha256=ndLgWYv4TTixGE1nwht71nQtqsMq3f2yJGRuLR5yiyg,5854
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=TfKkVnmRIetATSEVQFp-rOOIRGqVig2i8faSQQt_ixA,20974
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
-sky/cli.py,sha256=19YLRhF3p8wOfKoN0vICUJ9itFKDKTyhG3mV8UsnR-U,206326
+sky/cli.py,sha256=eC-uC2_OhuDephCdejr70VeB1bo1KGwHC8et5VR_lto,210828
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
-sky/core.py,sha256=YF_6kwj8Ja171Oycb8L25SZ7V_ylZYovFS_jpnjwGo0,34408
+sky/core.py,sha256=I7ZxjrJmrM-zAWVnmUyz0kzS9fjKaaLehDdWMm3H0x8,34443
 sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
-sky/exceptions.py,sha256=s7j0iCa1Ec0rU1ABb9EAhqn2qFm22bmKQV_ckgRlMGk,8720
-sky/execution.py,sha256=97yhNh5BKBh2ZJW8GefGID_4KCYU-arnvemSJB9rf6U,25552
+sky/exceptions.py,sha256=D7WARzYRt4dGjXo6gI-gzkoodZbKF1D-qncm_DbHB28,8846
+sky/execution.py,sha256=CbrKMgfc2JgLqZqwPvmYKxbWAQKYqHpOLpUEOb-k2m0,24679
 sky/global_user_state.py,sha256=PywEmUutF97XBgRMClR6IS5_KM8JJC0oA1LsPUZebp0,28681
-sky/optimizer.py,sha256=YGBhJPlcvylYON7MLrYEMtBOqJLt4LdlguQclVvvl4E,58677
+sky/optimizer.py,sha256=OzxWiA6ZC0tyJ1eNMy4e72vitjfLKfbOLF9ywZOccXU,59343
 sky/resources.py,sha256=b9yaZvZkL-QZdElQLHsEZ2jhKgId2ixG8M2Z8DLBBKU,67450
-sky/sky_logging.py,sha256=I59__M9taBjDim15ie0m25Vtn6itLtR9Ao8W9FS36Xs,4253
+sky/sky_logging.py,sha256=oLmTmwkuucIto3LHXLJfMcyRpYSkmZAZa5XzQPA5IHk,4434
 sky/skypilot_config.py,sha256=E3g65cX3P3dT9b5N0GgFBG6yB0FXwIGpisKoozmJmWU,9094
 sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
 sky/task.py,sha256=KDsTBIxYpkCOPHv3_ei5H3LDMiGHvDeS9_2HeL6yyLA,49766
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=jz3E8YyeGscgLZvFE6A1SkxpH6o_inZ-0NiXdRnxSGA,6863
-sky/adaptors/azure.py,sha256=AmiNSSbgLrBEA_GefyRx5EKr8rWlvrClt1PLogFpkd0,21309
+sky/adaptors/azure.py,sha256=edWy5YgW6dAUl8pPWvSJ8x5mRgCew1KmDpntTw65vHo,21464
 sky/adaptors/cloudflare.py,sha256=0JA41HeZeYoqcJoEyol01caYgt8UrlpcZJJ6ToU-Fg8,7594
-sky/adaptors/common.py,sha256=Tz98vqZrYMf_EoIIqaZv8DQcbW5a_xSGf7RoAhCEDlo,2354
+sky/adaptors/common.py,sha256=VhgAFPx-GmhXoob8B1YvqGC2BXs2HwtqoA8oFMaqkbc,2551
 sky/adaptors/cudo.py,sha256=WGvIQrlzJkGDe02Ve7pygA56tHwUc4kwS3XHW8kMFAA,239
 sky/adaptors/docker.py,sha256=_kzpZ0fkWHqqQAVVl0llTsCE31KYz3Sjn8psTBQHVkA,468
 sky/adaptors/gcp.py,sha256=OQ9RaqjR0r0iaWYpjvEtIx5vnEhyB4LhUCwbtdxsmVk,3115
@@ -29,16 +29,16 @@ sky/adaptors/oci.py,sha256=n_zcrippTZRbTIhN3euD5sqNYn43G397zMavaJyEYbk,1480
 sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
-sky/backends/backend.py,sha256=xtxR6boDv1o-uSCjbJhOMkKMnZvBZh3gExx4khFWPTI,5932
-sky/backends/backend_utils.py,sha256=_gp6nL4evexSoXUWdA4rI7mJ2XQt59bYQ0LRAD5QuKI,126826
-sky/backends/cloud_vm_ray_backend.py,sha256=NUT8bp-mifTLGzZFsA9kXqFPue4ZhbX-5pX55evdeXI,233984
+sky/backends/backend.py,sha256=wwfbrxPhjMPs6PSyy3tAHI8WJhl-xhgzWBsAZjmJJ6g,6249
+sky/backends/backend_utils.py,sha256=Kp3HRqjrisnDYzLURYSbtyZ7WGMKHSVfMSwRj5_aOnc,126770
+sky/backends/cloud_vm_ray_backend.py,sha256=otDjYA4FtnlFGU_xNaxhBM5wFQ8fuF4kPeYQmf1Me3E,236901
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
-sky/backends/local_docker_backend.py,sha256=H4GBo0KFUC_EEf-ziv1OUbfAkOI5BrwkYs9fYOxSoNw,16741
+sky/backends/local_docker_backend.py,sha256=0JL5m0YUgOmOL4aWEUe4tmt89dsxjk4_WXkPwgEKEis,16801
 sky/backends/wheel_utils.py,sha256=3QS4T_Ydvo4DbYhogtyADyNBEf04I6jUCL71M285shQ,7963
 sky/backends/monkey_patches/monkey_patch_ray_up.py,sha256=76-y2fCaE3JINj8lEwHT1eirYzCbpD8O1ySsysuGu8o,3450
 sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
-sky/benchmark/benchmark_utils.py,sha256=oJOzJ4fs2sruxYh4Tl1NZ5fi2-3oWfXtoeCIAq2hgjw,26136
+sky/benchmark/benchmark_utils.py,sha256=eb-i6zYoo-Zkod-T9qtCu1FcYLw--Yyos1SyibUPZNE,26194
 sky/clouds/__init__.py,sha256=WuNIJEnZmBO72tU5awgaaL3rdvFRSkgaYNNeuY68dXo,1356
 sky/clouds/aws.py,sha256=uwz4aesgA2o6WNDhZlrXafQFbILv4EiUPNP67Cc8WGE,48978
 sky/clouds/azure.py,sha256=Yp_a1Lzvq4s47eRMeyVheDv9pC0hSPogCiTMYf-a5ZE,28687
@@ -56,12 +56,12 @@ sky/clouds/runpod.py,sha256=lstUC6f4JDhtcH9NfwkbpCJMmfmvMigoanhPXPbTYds,11540
 sky/clouds/scp.py,sha256=2KLTuNSMdBzK8CLwSesv7efOuiLidIMoyNG4AOt5Sqw,15870
 sky/clouds/vsphere.py,sha256=7eZFYIDtY5sX_ATr8h7kwwkY9t8Z-EYMJ9HCjoRBoxI,12309
 sky/clouds/service_catalog/__init__.py,sha256=e0K-c64jQV9d6zly5OnIXMsYaZXs_Ko9osAbDaRlOOw,14743
-sky/clouds/service_catalog/aws_catalog.py,sha256=OURG9JeSGQ_zNO-E2KwDgUJeQuJEvMa7arzhf4Kza6M,12549
+sky/clouds/service_catalog/aws_catalog.py,sha256=Ab8VAmSiuV37k6LLNBtjTYoe_UDMJRgXe1Vk3wiUO_U,12605
 sky/clouds/service_catalog/azure_catalog.py,sha256=VJi3yfhZy9Sc6UfcLAc8xIoTlUlUr090TODkCZyyHFw,7311
-sky/clouds/service_catalog/common.py,sha256=LED9Yzh1nsurvjCtqXGcsbUI8N-YFx05kazBgLF-HIQ,27256
+sky/clouds/service_catalog/common.py,sha256=PA3llB0zZh4v0DO_gDDCKGhRIBx16CAp2WJZNxhjNOA,27266
 sky/clouds/service_catalog/config.py,sha256=ylzqewdEBjDg4awvFek6ldYmFrnvD2bVGLZuLPvEVYA,1793
 sky/clouds/service_catalog/constants.py,sha256=ai2yOlsVqBnEpbxaEHXt61COsHBLwOfw6GZXntEPj7k,411
-sky/clouds/service_catalog/cudo_catalog.py,sha256=WPLIQ9NVvuKAS48KYujV6dQsyeppIcUCVXpapy13bbI,4335
+sky/clouds/service_catalog/cudo_catalog.py,sha256=QXAOpx5fJ_cGCr5LbB7wpHMfKIla7G-q_mMJnv_ArTA,4652
 sky/clouds/service_catalog/fluidstack_catalog.py,sha256=c8MMTldG-q97MJ0zJymudQiOVQC_rxS7vqrZgLrgbQA,5038
 sky/clouds/service_catalog/gcp_catalog.py,sha256=MHWq_-jqm68oNpK1i8AlJIGBkSKT-P6xX7DkpvqvpHU,24323
 sky/clouds/service_catalog/ibm_catalog.py,sha256=0dzjmXABFECzaAuIa0E6pVINhVK6-G6U52Mj-L45gK8,4472
@@ -91,15 +91,15 @@ sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=MBmjey9_p2L3IKNKTi8um09SlZe32n4wK3CkVnlTVvo,7346
 sky/data/data_utils.py,sha256=-P5GsDH_m4slrCz4vHdgiFezIys8ufzvhEKePJwfjFc,28597
 sky/data/mounting_utils.py,sha256=44YkYIIgArEkyvxCtfmXXumybrU8bmn1TfLXWv_eldI,11480
-sky/data/storage.py,sha256=SzO2GefxfoYbKuWO4iRt_9At33s--k4q8htN8xy-vrM,162395
-sky/data/storage_utils.py,sha256=Rwj_Pt2Pl0e16dhHXyxiT500CYv1k7SWE_WE2jKepl0,9358
-sky/jobs/__init__.py,sha256=9cqFutVlfjQb7t8hzG-ZlQmMlbmfMirn0KNBxIFnJYQ,1398
+sky/data/storage.py,sha256=x8YYY4zVBdit_5oAR_MXV-TM9qDefV_ZV4z0irv6ZaU,163102
+sky/data/storage_utils.py,sha256=cM3kxlffYE7PnJySDu8huyUsMX_JYsf9uer8r5OYsjo,9556
+sky/jobs/__init__.py,sha256=yucibSB_ZimtJMvOhMxn6ZqwBIYNfcwmc6pSXtCqmNQ,1483
 sky/jobs/constants.py,sha256=YLgcCg_RHSYr_rfsI_4UIdXk78KKKOK29Oem88t5j8I,1350
 sky/jobs/controller.py,sha256=k28bbicxtML6p1YxSetk-1nhBHPCubpvLWJsh7TtU9c,26701
-sky/jobs/core.py,sha256=Q5ExRWnF7yAYWJxwnB9NfAGBVDNqKYBCrWsypiMLCpY,13637
+sky/jobs/core.py,sha256=RkBFaKDlovmdzqlOAgQ0xAimZFgo4pXq3qaQkAvGsGk,16908
 sky/jobs/recovery_strategy.py,sha256=G3iFicEajB-l9FefvcqjqPIazb1X8BJ_AgVmD5bDV2w,25556
 sky/jobs/state.py,sha256=C6R5Yq7ftBqGPa_71tUjflBMKAaJ1FTTdbgjAwmbJsI,23231
-sky/jobs/utils.py,sha256=ZB2dJxtJ4hbCRdxHmy8wrmtXIvvGGE80kk5BQTOQWkQ,35653
+sky/jobs/utils.py,sha256=lYfWkEAPVnYcj2nT6VYdM6PCaWKUH6_AD4TAV_sVCkY,36376
 sky/jobs/dashboard/dashboard.py,sha256=HFShuaxKir97QTeK2x37h6bsY6ncaFaNEg1USZqJPdc,3050
 sky/jobs/dashboard/static/favicon.ico,sha256=uYlvgxSM7gjBmXpZ8wydvZUPAbJiiix-rc2Xe5mma9s,15086
 sky/jobs/dashboard/templates/index.html,sha256=DBKMYEkkJ6sgLYod9ro7drgL8Y_neDsCx_WbwhWDsWM,9837
@@ -110,16 +110,16 @@ sky/provision/docker_utils.py,sha256=Z7vDUs9Yjqks_CsWrACcTgABIZuFi3EJVFwkU0WsdD0
 sky/provision/instance_setup.py,sha256=n1Px_KOYZl7Rf1WLXrfTTHyqxyA8_5QTN9BNLjQRkgc,22427
 sky/provision/logging.py,sha256=yZWgejrFBhhRjAtvFu5N5bRXIMK5TuwNjp1vKQqz2pw,2103
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
-sky/provision/provisioner.py,sha256=atW6P0XgBL_qxChp0klwgj6j2XE7Q9_EevPrFSJcoJc,25240
+sky/provision/provisioner.py,sha256=7BatzBobHGCLJf2u5hCt4VveS9tOPc5zL4wGHR6OcqE,24888
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
-sky/provision/aws/config.py,sha256=GfI6l5-ewhm2SS2XBnhVIFZCEueBkLgpvkf16xlSlG0,23017
+sky/provision/aws/config.py,sha256=ApEh63RR_KyCp9nPXX35z6jBREoulJPQ5st8K9Jlclo,23385
 sky/provision/aws/instance.py,sha256=eCslJ2XfJo_pkQMnKFQqhGnUIRvwKiT12oxBY5-klss,40750
 sky/provision/aws/utils.py,sha256=m49pS-SHGW7Au3bhDeTPsL8N5iRzbwOXzyEWRCc1Vho,3238
 sky/provision/azure/__init__.py,sha256=87cgk1_Ws7n9rqaDDPv-HpfrkVeSQMdFQnhnXwyx9g4,548
 sky/provision/azure/azure-config-template.json,sha256=dwTO-DG70UtBifN59NzsJwPZHZ4uCs7_oLooHgXG_N8,4349
 sky/provision/azure/azure-vm-template.json,sha256=k0A4xclQ-xQbAdcjo516sZqMDsU-n8_jeRuPxtGKBFI,10901
-sky/provision/azure/config.py,sha256=Fx_N_F9912q2gUHt0FhAxVDwlTbd9cLxrP5rii0KJOM,6772
-sky/provision/azure/instance.py,sha256=-95c__W4d0mgKweUPhT3tPLAdNtdruroHrfIvNTFQtY,32962
+sky/provision/azure/config.py,sha256=qNUNcX1HboosuZASEVgJ_K-g7CNNWExodWXFIsoq5V8,7288
+sky/provision/azure/instance.py,sha256=3xHRkWcElAgaOuOzcBdwfnLni8zD8KwNgYmj3A7gljA,33244
 sky/provision/cudo/__init__.py,sha256=KAEl26MVPsk7IoP9Gg-MOJJRIV6-X9B0fbyHdyJWdLo,741
 sky/provision/cudo/config.py,sha256=RYOVkV0MoUqVBJRZiKhBZhjFygeyFs7eUdVMdPg1vds,327
 sky/provision/cudo/cudo_machine_type.py,sha256=_VNXWPELmlFXbtdcnPvkuLuyE9CZ923BUCdiac-ClDY,696
@@ -138,10 +138,10 @@ sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHa
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
 sky/provision/kubernetes/config.py,sha256=WEKcFXXhe89bLGAvoMiBvTDxdxkpTIA6ezrj2vmzldc,29072
-sky/provision/kubernetes/instance.py,sha256=MdgyGcMUbhsSRdaTRV3IgHmiAj5goCDVhzDZ2PDVs_Y,38323
+sky/provision/kubernetes/instance.py,sha256=Qth9AWc8OBGB7WeGJ4ERlopNA8y2wg1AvS5XPJEuXXQ,38421
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
 sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMshsvsYmnMUcq8k,11399
-sky/provision/kubernetes/utils.py,sha256=iULhot4naFOsyzp53x4Q4qpsHXvz5-DMOIFFTR8ap9s,83609
+sky/provision/kubernetes/utils.py,sha256=oJgCrbR8IyTw_uMoM9oxYYYNN_tZ1yzppWNTDM-XqaM,84522
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/paperspace/__init__.py,sha256=1nbUPWio7UA5gCQkO_rfEDfgXT17u5OtuByxQx4Ez6g,598
@@ -172,7 +172,7 @@ sky/serve/__init__.py,sha256=Qg_XPOtQsUxiN-Q3njHZRfzoMcQ_KKU1QthkiTbESDw,1661
 sky/serve/autoscalers.py,sha256=khY1oZ22PRaUQNsLCoNKH178X_NiJw0LSLOKr7_LNgY,30275
 sky/serve/constants.py,sha256=OansIC7a0Pwat-Y5SF43T9phad_EvyjKO3peZgKFEHk,4367
 sky/serve/controller.py,sha256=gfE_gB7wxE1VxvnYqw_-KcMGc6X2kufl-NLR7sWdzdY,8172
-sky/serve/core.py,sha256=cW2SNMPMbGtOcqASHnL__B12BCIErUilGtFw3olQbjk,28947
+sky/serve/core.py,sha256=yebcgmafGwKppXA1vyJdnbWdOg5BSlh87pKL9gkzHPE,29066
 sky/serve/load_balancer.py,sha256=aUfDsgUT_fYrchCwJCeunMPXmAkwJAY58BEu-IN2FaA,11571
 sky/serve/load_balancing_policies.py,sha256=ExdwH_pxPYpJ6CkoTQCOPSa4lzwbq1LFFMKzmIu8ryk,2331
 sky/serve/replica_managers.py,sha256=dO962WZ_6YWRDpyNemY7SzC7fZHlNfoL4kUS3MaKwDo,57405
@@ -190,7 +190,7 @@ sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
 sky/skylet/constants.py,sha256=txAK0602kGtaD42JUYECq5u4rLIZFhOIWz-fLUV1KgA,14652
 sky/skylet/events.py,sha256=A09E7LmmwzcGrSG0n8K7d3EZ1ZJr1mmmzoGyhnArYJA,12303
 sky/skylet/job_lib.py,sha256=1XTXLStPiHk8X2SHe6x4PNQ01Rj6Cwagr5lLWjzpj6E,35198
-sky/skylet/log_lib.py,sha256=5akG6avIW5y9rO5pfgOCEtprPgasEo4vAj7I8Bbhxm4,19340
+sky/skylet/log_lib.py,sha256=7rdmEr5Wy9CIGLkWls5-FVeqTL1O2GPy44uDaEAqMS0,19312
 sky/skylet/log_lib.pyi,sha256=AHMkW2DGK2erFovb3ToZWxRiYaATlzkxKb5J9pkgF2Y,4295
 sky/skylet/skylet.py,sha256=U9plr5hmhD9-Nyy0LMCymlE8DWtRXTFXQvfbFsS746Y,1153
 sky/skylet/subprocess_daemon.py,sha256=IJwGAzOdERrhWJS7VYKAUByNsLyIkKkB0w5nk06okG8,2818
@@ -201,7 +201,7 @@ sky/skylet/providers/ibm/node_provider.py,sha256=olNtCoCxjXTT-C_youwdQ9UF1DPgO8O
 sky/skylet/providers/ibm/utils.py,sha256=63vhKqLLOhAZdibSp8VWWONeyCER9F6U2VLrSpzlizk,1292
 sky/skylet/providers/ibm/vpc_provider.py,sha256=GiOGlWYqqeBETfAeKqVj2-9shsMSP7z1WnO8UP5JTNo,34630
 sky/skylet/providers/lambda_cloud/__init__.py,sha256=DNxB-NL97FU7ptGcXTnIZZRt6ZLtNVtp3nayJNaxjhY,112
-sky/skylet/providers/lambda_cloud/node_provider.py,sha256=1zZFYl-09FDLjFr5gTus8LEgcDVUI9PFqi4S1tt7-84,13971
+sky/skylet/providers/lambda_cloud/node_provider.py,sha256=77AoTaXuSW5cr4Z2cEAbAsHx1e4l8c0fc1ABbdLduKE,14042
 sky/skylet/providers/oci/__init__.py,sha256=LRMTj6OhQoxiFJw4uNxG8cn6PllP8A-lGJL3Cs5DJok,91
 sky/skylet/providers/oci/node_provider.py,sha256=YPqiRag_cysvYMIMDGbMn6lOumvHad6FLJB5DGPr00Q,20492
 sky/skylet/providers/oci/query_helper.py,sha256=dUsvPGzWPNF5O2NjQvuC8tkilT4H11gMj6R7Qel2fDc,17202
@@ -244,24 +244,24 @@ sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
 sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
-sky/utils/command_runner.py,sha256=n1B2h_25G_xIz7ICClczR_fkVgy0-MTHc9907Uy_Wvc,34582
+sky/utils/command_runner.py,sha256=ZIu4aur4yxtjHu60Na9o90Iu-g48_yeWXo-NNfmzs-w,34634
 sky/utils/command_runner.pyi,sha256=mJOzCgcYZAfHwnY_6Wf1YwlTEJGb9ihzc2f0rE0Kw98,7751
-sky/utils/common_utils.py,sha256=O6PlZTCNhbuXOzjuV2DKw43niWE_qPfYZNGhnMtZzQg,24028
-sky/utils/controller_utils.py,sha256=32pVORm2cd42tg0srxGvmYV0kYTl67IFsw2EdXbdoR8,38042
+sky/utils/common_utils.py,sha256=MwFhIcvCEMBo7kbENUjN3qRNO5SoMV0fzAORc65c5x0,24525
+sky/utils/controller_utils.py,sha256=V05hiLJIjqqXssYzs_Gchk4-tijgpMgLJsRW8ymhS-E,40625
 sky/utils/dag_utils.py,sha256=gjGZiJj4_GYsraXX67e6ElvbmOByJcyjSfvVgYZiXvs,5588
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
-sky/utils/env_options.py,sha256=1VXyd3bhiUgGfCpmmTqM9PagRo1ILBH4-pzIxmIeE6E,861
+sky/utils/env_options.py,sha256=3oAaUPxowL6vI2XmxXrH56V7Myj9IJWsL-MXFmRFVdI,1294
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
-sky/utils/log_utils.py,sha256=yVu3etgKhiVYX8UG-JFPWZujxWBT4kwxZ5oAPIdjtGs,12054
-sky/utils/resources_utils.py,sha256=snByBxgx3Hnjfch2uysdAA3D-OAwrnuzTDHug36s5H4,6515
-sky/utils/rich_utils.py,sha256=5ZVhzlFx-nhqMXwv00eO9xC4rz7ibDlfD2lmGhZrJEY,1581
+sky/utils/log_utils.py,sha256=ptv2sbsiJSgk4NvdccrMsUR-MvOKnbu4BQiRSishgk0,12472
+sky/utils/resources_utils.py,sha256=sJuPextjJKHhvDGAaOPEzeEkteryF2fGNgNgBLqnLic,7419
+sky/utils/rich_utils.py,sha256=hmnI1X5dKvRIQzB7EyNb34FT97qFNve-0QHqM5r0mVk,3066
 sky/utils/schemas.py,sha256=QT0Fxri2o0SiWkky1DlZhA1dzQRQoB5OdVaej0wJvhc,28787
 sky/utils/subprocess_utils.py,sha256=3R54Elc2n8DQeO6Y8MCDJ6N6v27HDGpbNMIfCquqXYQ,6552
 sky/utils/timeline.py,sha256=ao_nm0y52ZQILfL7Y92c3pSEFRyPm_ElORC3DrI5BwQ,3936
-sky/utils/ux_utils.py,sha256=318TRunQCyJpJXonfiJ1SVotNA-6K4F2XgMEYjvWvsk,3264
+sky/utils/ux_utils.py,sha256=0Fy20UbM2rQ_aVIx_B2-oiLXGmzC1myn-0pnmb56pcA,5227
 sky/utils/validator.py,sha256=cAFERCoC7jH0DFKepcU4x9SYmdrYL1iVmW9tXA18hvo,701
 sky/utils/cli_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/utils/cli_utils/status_utils.py,sha256=9odkfXiXLMD14XJsqve6sGvHpe7ThHXpC6ic9RYtOqY,11032
+sky/utils/cli_utils/status_utils.py,sha256=zIaiGWtEACdczubiSBMTupueZStPt1VqgRrhQilLEaI,17954
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=VLXfazav9XCMQmeKVqhuOQzt2vM6G1jgnvvb0SHUFno,7773
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
@@ -273,9 +273,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=aRMa_0JRHtXFOPtEg4rFAwR1t57wvvAoGZhn3H3BtGk,1059
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241011.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241011.dist-info/METADATA,sha256=poy78UzKBy2gq6bEdz2LA7LJym4BgdYIdeMv9zk_Jaw,18945
-skypilot_nightly-1.0.0.dev20241011.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-skypilot_nightly-1.0.0.dev20241011.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241011.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241011.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241013.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241013.dist-info/METADATA,sha256=6Ayx_gke7hiH3XZ4bkFiW9b0jUTcXzqs6dk9p3ePwN4,18945
+skypilot_nightly-1.0.0.dev20241013.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+skypilot_nightly-1.0.0.dev20241013.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241013.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241013.dist-info/RECORD,,