skypilot-nightly 1.0.0.dev20241107__py3-none-any.whl → 1.0.0.dev20241109__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'fe2ce9a262c059722ddce46f5594fc2ca2370c0d'
+_SKYPILOT_COMMIT_SHA = '42c79e1d0a5e018e275705ada53957573f9a0181'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241107'
+__version__ = '1.0.0.dev20241109'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -3262,6 +3262,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
     ) -> None:
         """Executes generated code on the head node."""
         style = colorama.Style
+        fore = colorama.Fore
 
         script_path = os.path.join(SKY_REMOTE_APP_DIR, f'sky_job_{job_id}')
         remote_log_dir = self.log_dir
@@ -3373,9 +3374,9 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             controller = controller_utils.Controllers.from_name(name)
             if controller == controller_utils.Controllers.JOBS_CONTROLLER:
                 logger.info(
-                    f'\n📋 Useful Commands'
-                    f'\nManaged Job ID: '
+                    f'\n{fore.CYAN}Managed Job ID: '
                     f'{style.BRIGHT}{job_id}{style.RESET_ALL}'
+                    f'\n📋 Useful Commands'
                     f'\n{ux_utils.INDENT_SYMBOL}To cancel the job:\t\t\t'
                     f'{ux_utils.BOLD}sky jobs cancel {job_id}'
                     f'{ux_utils.RESET_BOLD}'
@@ -3392,8 +3393,9 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                     f'dashboard:\t{ux_utils.BOLD}sky jobs dashboard'
                     f'{ux_utils.RESET_BOLD}')
             elif controller is None:
-                logger.info(f'\n📋 Useful Commands'
-                            f'\nJob ID: {job_id}'
+                logger.info(f'\n{fore.CYAN}Job ID: '
+                            f'{style.BRIGHT}{job_id}{style.RESET_ALL}'
+                            f'\n📋 Useful Commands'
                             f'\n{ux_utils.INDENT_SYMBOL}To cancel the job:\t\t'
                             f'{ux_utils.BOLD}sky cancel {name} {job_id}'
                             f'{ux_utils.RESET_BOLD}'
@@ -3709,7 +3711,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                   handle: CloudVmRayResourceHandle,
                   job_id: Optional[int],
                   managed_job_id: Optional[int] = None,
-                  follow: bool = True) -> int:
+                  follow: bool = True,
+                  tail: int = 0) -> int:
         """Tail the logs of a job.
 
         Args:
@@ -3717,10 +3720,13 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             job_id: The job ID to tail the logs of.
             managed_job_id: The managed job ID for display purpose only.
             follow: Whether to follow the logs.
+            tail: The number of lines to display from the end of the
+                log file. If 0, print all lines.
         """
         code = job_lib.JobLibCodeGen.tail_logs(job_id,
                                                managed_job_id=managed_job_id,
-                                               follow=follow)
+                                               follow=follow,
+                                               tail=tail)
         if job_id is None and managed_job_id is None:
             logger.info(
                 'Job ID not provided. Streaming the logs of the latest job.')

sky/backends/wheel_utils.py

@@ -129,7 +129,11 @@ def _build_sky_wheel() -> pathlib.Path:
 
         wheel_dir = WHEEL_DIR / hash_of_latest_wheel
         wheel_dir.mkdir(parents=True, exist_ok=True)
-        shutil.move(str(wheel_path), wheel_dir)
+        # shutil.move will fail when the file already exists and is being
+        # moved across filesystems.
+        if not os.path.exists(
+                os.path.join(wheel_dir, os.path.basename(wheel_path))):
+            shutil.move(str(wheel_path), wheel_dir)
         return wheel_dir / wheel_path.name
 
 

sky/cli.py

@@ -46,6 +46,7 @@ from rich import progress as rich_progress
 import yaml
 
 import sky
+from sky import admin_policy
 from sky import backends
 from sky import check as sky_check
 from sky import clouds as sky_clouds
@@ -67,6 +68,7 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.skylet import log_lib
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import dag_utils
@@ -582,6 +584,15 @@ def _launch_with_confirm(
             with ux_utils.print_exception_no_traceback():
                 raise RuntimeError(f'{colorama.Fore.YELLOW}{e}'
                                    f'{colorama.Style.RESET_ALL}') from e
+        dag, _ = admin_policy_utils.apply(
+            dag,
+            request_options=admin_policy.RequestOptions(
+                cluster_name=cluster,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                down=down,
+                dryrun=dryrun,
+            ),
+        )
         dag = sky.optimize(dag)
     task = dag.tasks[0]
 
@@ -2011,6 +2022,12 @@ def queue(clusters: List[str], skip_finished: bool, all_users: bool):
     help=('Follow the logs of a job. '
           'If --no-follow is specified, print the log so far and exit. '
           '[default: --follow]'))
+@click.option(
+    '--tail',
+    default=0,
+    type=int,
+    help=('The number of lines to display from the end of the log file. '
+          'Default is 0, which means print all lines.'))
 @click.argument('cluster',
                 required=True,
                 type=str,
@@ -2024,6 +2041,7 @@ def logs(
     sync_down: bool,
     status: bool,  # pylint: disable=redefined-outer-name
     follow: bool,
+    tail: int,
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail the log of a job.
@@ -2090,7 +2108,7 @@ def logs(
                 click.secho(f'Job {id_str}not found', fg='red')
             sys.exit(1)
 
-    core.tail_logs(cluster, job_id, follow)
+    core.tail_logs(cluster, job_id, follow, tail)
 
 
 @cli.command()
@@ -3036,9 +3054,9 @@ def show_gpus(
     and spot instances. There may be multiple regions with the same lowest
     price.
 
-    If ``--cloud kubernetes`` is specified, it will show the maximum quantities
-    of the GPU available on a single node and the real-time availability of
-    the GPU across all nodes in the Kubernetes cluster.
+    If ``--cloud kubernetes`` or ``--cloud k8s`` is specified, it will show the
+    maximum quantities of the GPU available on a single node and the real-time
+    availability of the GPU across all nodes in the Kubernetes cluster.
 
     Definitions of certain fields:
 
@@ -3667,6 +3685,8 @@ def jobs_launch(
 
     click.secho(f'Managed job {dag.name!r} will be launched on (estimated):',
                 fg='cyan')
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     dag = sky.optimize(dag)
 
     if not yes:
@@ -4145,6 +4165,8 @@ def serve_up(
                 fg='cyan')
     with sky.Dag() as dag:
         dag.add(task)
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     sky.optimize(dag)
 
     if not yes:
@@ -4261,6 +4283,8 @@ def serve_update(
                 fg='cyan')
     with sky.Dag() as dag:
         dag.add(task)
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     sky.optimize(dag)
 
     if not yes:

sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py

@@ -20,7 +20,6 @@ DEFAULT_LAMBDA_KEYS_PATH = os.path.expanduser('~/.lambda_cloud/lambda_keys')
 
 # List of all possible regions.
 REGIONS = [
-    'australia-southeast-1',
     'europe-central-1',
     'asia-south-1',
     'me-west-1',
@@ -28,9 +27,12 @@ REGIONS = [
     'asia-northeast-1',
     'asia-northeast-2',
     'us-east-1',
+    'us-east-2',
     'us-west-2',
     'us-west-1',
     'us-south-1',
+    'us-south-2',
+    'us-south-3',
     'us-west-3',
     'us-midwest-1',
 ]

sky/core.py

@@ -742,7 +742,8 @@ def cancel(
 @usage_lib.entrypoint
 def tail_logs(cluster_name: str,
               job_id: Optional[int],
-              follow: bool = True) -> None:
+              follow: bool = True,
+              tail: int = 0) -> None:
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail the logs of a job.
 
@@ -775,7 +776,7 @@ def tail_logs(cluster_name: str,
         f'{colorama.Style.RESET_ALL}')
 
     usage_lib.record_cluster_name_for_current_operation(cluster_name)
-    backend.tail_logs(handle, job_id, follow=follow)
+    backend.tail_logs(handle, job_id, follow=follow, tail=tail)
 
 
 @usage_lib.entrypoint

sky/dag.py

@@ -23,6 +23,7 @@ class Dag:
 
         self.graph = nx.DiGraph()
         self.name: Optional[str] = None
+        self.policy_applied: bool = False
 
     def add(self, task: 'task.Task') -> None:
         self.graph.add_node(task)

sky/data/mounting_utils.py

@@ -276,23 +276,11 @@ def get_mounting_command(
     script = get_mounting_script(mount_path, mount_cmd, install_cmd,
                                  version_check_cmd)
 
-    # TODO(romilb): Get direct bash script to work like so:
-    # command = f'bash <<-\EOL' \
-    #           f'{script}' \
-    #           'EOL'
-
-    # TODO(romilb): This heredoc should have EOF after script, but it
-    #  fails with sky's ssh pipeline. Instead, we don't use EOF and use )
-    #  as the end of heredoc. This raises a warning (here-document delimited
-    #  by end-of-file) that can be safely ignored.
-
     # While these commands are run sequentially for each storage object,
     # we add random int to be on the safer side and avoid collisions.
     script_path = f'~/.sky/mount_{random.randint(0, 1000000)}.sh'
-    first_line = r'(cat <<-\EOF > {}'.format(script_path)
-    command = (f'{first_line}'
-               f'{script}'
-               f') && chmod +x {script_path}'
-               f' && bash {script_path}'
-               f' && rm {script_path}')
+    command = (f'echo {shlex.quote(script)} > {script_path} && '
+               f'chmod +x {script_path} && '
+               f'bash {script_path} && '
+               f'rm {script_path}')
     return command

sky/exceptions.py

@@ -3,6 +3,8 @@ import enum
 import typing
 from typing import List, Optional, Sequence
 
+from sky.utils import env_options
+
 if typing.TYPE_CHECKING:
     from sky import status_lib
     from sky.backends import backend
@@ -104,7 +106,8 @@ class CommandError(Exception):
         if not command:
             message = error_msg
         else:
-            if len(command) > 100:
+            if (len(command) > 100 and
+                    not env_options.Options.SHOW_DEBUG_INFO.get()):
                 # Chunck the command to avoid overflow.
                 command = command[:100] + '...'
             message = (f'Command {command} failed with return code '

sky/execution.py

@@ -160,14 +160,16 @@ def _execute(
     """
 
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
-    dag, _ = admin_policy_utils.apply(
-        dag,
-        request_options=admin_policy.RequestOptions(
-            cluster_name=cluster_name,
-            idle_minutes_to_autostop=idle_minutes_to_autostop,
-            down=down,
-            dryrun=dryrun,
-        ))
+    if not dag.policy_applied:
+        dag, _ = admin_policy_utils.apply(
+            dag,
+            request_options=admin_policy.RequestOptions(
+                cluster_name=cluster_name,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                down=down,
+                dryrun=dryrun,
+            ),
+        )
     assert len(dag) == 1, f'We support 1 task for now. {dag}'
     task = dag.tasks[0]
 

sky/jobs/core.py

@@ -59,8 +59,10 @@ def launch(
     """
     entrypoint = task
     dag_uuid = str(uuid.uuid4().hex[:4])
-
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
+    # Always apply the policy again here, even though it might have been applied
+    # in the CLI. This is to ensure that we apply the policy to the final DAG
+    # and get the mutated config.
     dag, mutated_user_config = admin_policy_utils.apply(
         dag, use_mutated_config_in_current_request=False)
     if not dag.is_chain():

sky/jobs/dashboard/dashboard.py

@@ -26,7 +26,8 @@ def _is_running_on_jobs_controller() -> bool:
     """
     if pathlib.Path('~/.sky/sky_ray.yml').expanduser().exists():
         config = yaml.safe_load(
-            pathlib.Path('~/.sky/sky_ray.yml').expanduser().read_text())
+            pathlib.Path('~/.sky/sky_ray.yml').expanduser().read_text(
+                encoding='utf-8'))
         cluster_name = config.get('cluster_name', '')
         candidate_controller_names = (
             controller_utils.Controllers.JOBS_CONTROLLER.value.

sky/jobs/recovery_strategy.py

@@ -36,6 +36,11 @@ DEFAULT_RECOVERY_STRATEGY = None
 # 10 * JOB_STARTED_STATUS_CHECK_GAP_SECONDS = 10 * 5 = 50 seconds
 MAX_JOB_CHECKING_RETRY = 10
 
+# Minutes to job cluster autodown. This should be significantly larger than
+# managed_job_utils.JOB_STATUS_CHECK_GAP_SECONDS, to avoid tearing down the
+# cluster before its status can be updated by the job controller.
+_AUTODOWN_MINUTES = 5
+
 
 def terminate_cluster(cluster_name: str, max_retry: int = 3) -> None:
     """Terminate the cluster."""
@@ -302,11 +307,17 @@ class StrategyExecutor:
                 usage_lib.messages.usage.set_internal()
                 # Detach setup, so that the setup failure can be detected
                 # by the controller process (job_status -> FAILED_SETUP).
-                sky.launch(self.dag,
-                           cluster_name=self.cluster_name,
-                           detach_setup=True,
-                           detach_run=True,
-                           _is_launched_by_jobs_controller=True)
+                sky.launch(
+                    self.dag,
+                    cluster_name=self.cluster_name,
+                    # We expect to tear down the cluster as soon as the job is
+                    # finished. However, in case the controller dies, set
+                    # autodown to try and avoid a resource leak.
+                    idle_minutes_to_autostop=_AUTODOWN_MINUTES,
+                    down=True,
+                    detach_setup=True,
+                    detach_run=True,
+                    _is_launched_by_jobs_controller=True)
                 logger.info('Managed job cluster launched.')
             except (exceptions.InvalidClusterNameError,
                     exceptions.NoCloudAccessError,

sky/jobs/state.py

@@ -12,6 +12,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 import colorama
 
 from sky import sky_logging
+from sky.utils import common_utils
 from sky.utils import db_utils
 
 if typing.TYPE_CHECKING:
@@ -22,23 +23,6 @@ CallbackType = Callable[[str], None]
 logger = sky_logging.init_logger(__name__)
 
 
-def _get_db_path() -> str:
-    """Workaround to collapse multi-step Path ops for type checker.
-    Ensures _DB_PATH is str, avoiding Union[Path, str] inference.
-    """
-    path = pathlib.Path('~/.sky/spot_jobs.db')
-    path = path.expanduser().absolute()
-    path.parents[0].mkdir(parents=True, exist_ok=True)
-    return str(path)
-
-
-_DB_PATH = _get_db_path()
-
-# Module-level connection/cursor; thread-safe as the module is only imported
-# once.
-_CONN = sqlite3.connect(_DB_PATH)
-_CURSOR = _CONN.cursor()
-
 # === Database schema ===
 # `spot` table contains all the finest-grained tasks, including all the
 # tasks of a managed job (called spot for legacy reason, as it is generalized
@@ -50,68 +34,99 @@ _CURSOR = _CONN.cursor()
 # identifier/primary key for all the tasks. We will use `spot_job_id`
 # to identify the spot job.
 # TODO(zhwu): schema migration may be needed.
-_CURSOR.execute("""\
-    CREATE TABLE IF NOT EXISTS spot (
-    job_id INTEGER PRIMARY KEY AUTOINCREMENT,
-    job_name TEXT,
-    resources TEXT,
-    submitted_at FLOAT,
-    status TEXT,
-    run_timestamp TEXT CANDIDATE KEY,
-    start_at FLOAT DEFAULT NULL,
-    end_at FLOAT DEFAULT NULL,
-    last_recovered_at FLOAT DEFAULT -1,
-    recovery_count INTEGER DEFAULT 0,
-    job_duration FLOAT DEFAULT 0,
-    failure_reason TEXT,
-    spot_job_id INTEGER,
-    task_id INTEGER DEFAULT 0,
-    task_name TEXT,
-    specs TEXT)""")
-_CONN.commit()
-
-db_utils.add_column_to_table(_CURSOR, _CONN, 'spot', 'failure_reason', 'TEXT')
-# Create a new column `spot_job_id`, which is the same for tasks of the
-# same managed job.
-# The original `job_id` no longer has an actual meaning, but only a legacy
-# identifier for all tasks in database.
-db_utils.add_column_to_table(_CURSOR,
-                             _CONN,
-                             'spot',
-                             'spot_job_id',
-                             'INTEGER',
-                             copy_from='job_id')
-db_utils.add_column_to_table(_CURSOR,
-                             _CONN,
-                             'spot',
-                             'task_id',
-                             'INTEGER DEFAULT 0',
-                             value_to_replace_existing_entries=0)
-db_utils.add_column_to_table(_CURSOR,
-                             _CONN,
-                             'spot',
-                             'task_name',
-                             'TEXT',
-                             copy_from='job_name')
-
-# Specs is some useful information about the task, e.g., the
-# max_restarts_on_errors value. It is stored in JSON format.
-db_utils.add_column_to_table(_CURSOR,
-                             _CONN,
-                             'spot',
-                             'specs',
-                             'TEXT',
-                             value_to_replace_existing_entries=json.dumps({
-                                 'max_restarts_on_errors': 0,
-                             }))
-
-# `job_info` contains the mapping from job_id to the job_name.
-# In the future, it may contain more information about each job.
-_CURSOR.execute("""\
-    CREATE TABLE IF NOT EXISTS job_info (
-    spot_job_id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT)""")
-_CONN.commit()
+def create_table(cursor, conn):
+    # Enable WAL mode to avoid locking issues.
+    # See: issue #3863, #1441 and PR #1509
+    # https://github.com/microsoft/WSL/issues/2395
+    # TODO(romilb): We do not enable WAL for WSL because of known issue in WSL.
+    #  This may cause the database locked problem from WSL issue #1441.
+    if not common_utils.is_wsl():
+        try:
+            cursor.execute('PRAGMA journal_mode=WAL')
+        except sqlite3.OperationalError as e:
+            if 'database is locked' not in str(e):
+                raise
+            # If the database is locked, it is OK to continue, as the WAL mode
+            # is not critical and is likely to be enabled by other processes.
+
+    cursor.execute("""\
+        CREATE TABLE IF NOT EXISTS spot (
+        job_id INTEGER PRIMARY KEY AUTOINCREMENT,
+        job_name TEXT,
+        resources TEXT,
+        submitted_at FLOAT,
+        status TEXT,
+        run_timestamp TEXT CANDIDATE KEY,
+        start_at FLOAT DEFAULT NULL,
+        end_at FLOAT DEFAULT NULL,
+        last_recovered_at FLOAT DEFAULT -1,
+        recovery_count INTEGER DEFAULT 0,
+        job_duration FLOAT DEFAULT 0,
+        failure_reason TEXT,
+        spot_job_id INTEGER,
+        task_id INTEGER DEFAULT 0,
+        task_name TEXT,
+        specs TEXT)""")
+    conn.commit()
+
+    db_utils.add_column_to_table(cursor, conn, 'spot', 'failure_reason', 'TEXT')
+    # Create a new column `spot_job_id`, which is the same for tasks of the
+    # same managed job.
+    # The original `job_id` no longer has an actual meaning, but only a legacy
+    # identifier for all tasks in database.
+    db_utils.add_column_to_table(cursor,
+                                 conn,
+                                 'spot',
+                                 'spot_job_id',
+                                 'INTEGER',
+                                 copy_from='job_id')
+    db_utils.add_column_to_table(cursor,
+                                 conn,
+                                 'spot',
+                                 'task_id',
+                                 'INTEGER DEFAULT 0',
+                                 value_to_replace_existing_entries=0)
+    db_utils.add_column_to_table(cursor,
+                                 conn,
+                                 'spot',
+                                 'task_name',
+                                 'TEXT',
+                                 copy_from='job_name')
+
+    # Specs is some useful information about the task, e.g., the
+    # max_restarts_on_errors value. It is stored in JSON format.
+    db_utils.add_column_to_table(cursor,
+                                 conn,
+                                 'spot',
+                                 'specs',
+                                 'TEXT',
+                                 value_to_replace_existing_entries=json.dumps({
+                                     'max_restarts_on_errors': 0,
+                                 }))
+
+    # `job_info` contains the mapping from job_id to the job_name.
+    # In the future, it may contain more information about each job.
+    cursor.execute("""\
+        CREATE TABLE IF NOT EXISTS job_info (
+        spot_job_id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT)""")
+    conn.commit()
+
+
+# Module-level connection/cursor; thread-safe as the module is only imported
+# once.
+def _get_db_path() -> str:
+    """Workaround to collapse multi-step Path ops for type checker.
+    Ensures _DB_PATH is str, avoiding Union[Path, str] inference.
+    """
+    path = pathlib.Path('~/.sky/spot_jobs.db')
+    path = path.expanduser().absolute()
+    path.parents[0].mkdir(parents=True, exist_ok=True)
+    return str(path)
+
+
+_DB_PATH = _get_db_path()
+db_utils.SQLiteConn(_DB_PATH, create_table)
 
 # job_duration is the time a job actually runs (including the
 # setup duration) before last_recover, excluding the provision

sky/jobs/utils.py

@@ -14,7 +14,7 @@ import shutil
 import textwrap
 import time
 import typing
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import colorama
 import filelock
@@ -487,6 +487,7 @@ def stream_logs(job_id: Optional[int],
         job_id = managed_job_state.get_latest_job_id()
         if job_id is None:
             return 'No managed job found.'
+
     if controller:
         if job_id is None:
             assert job_name is not None
@@ -494,16 +495,22 @@ def stream_logs(job_id: Optional[int],
             # We manually filter the jobs by name, instead of using
             # get_nonterminal_job_ids_by_name, as with `controller=True`, we
             # should be able to show the logs for jobs in terminal states.
-            managed_jobs = list(
-                filter(lambda job: job['job_name'] == job_name, managed_jobs))
-            if len(managed_jobs) == 0:
+            managed_job_ids: Set[int] = {
+                job['job_id']
+                for job in managed_jobs
+                if job['job_name'] == job_name
+            }
+            if len(managed_job_ids) == 0:
                 return f'No managed job found with name {job_name!r}.'
-            if len(managed_jobs) > 1:
-                job_ids_str = ', '.join(job['job_id'] for job in managed_jobs)
-                raise ValueError(
-                    f'Multiple managed jobs found with name {job_name!r} (Job '
-                    f'IDs: {job_ids_str}). Please specify the job_id instead.')
-            job_id = managed_jobs[0]['job_id']
+            if len(managed_job_ids) > 1:
+                job_ids_str = ', '.join(
+                    str(job_id) for job_id in managed_job_ids)
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError(
+                        f'Multiple managed jobs found with name {job_name!r} '
+                        f'(Job IDs: {job_ids_str}). Please specify the job_id '
+                        'instead.')
+            job_id = managed_job_ids.pop()
         assert job_id is not None, (job_id, job_name)
         # TODO: keep the following code sync with
         # job_lib.JobLibCodeGen.tail_logs, we do not directly call that function
@@ -849,6 +856,7 @@ class ManagedJobCodeGen:
 
         from sky.skylet import job_lib, log_lib
         from sky.skylet import constants
+        from sky.utils import ux_utils
         try:
             from sky.jobs.utils import stream_logs_by_id
         except ImportError:

sky/provision/aws/config.py

@@ -42,8 +42,9 @@ def _skypilot_log_error_and_exit_for_failover(error: str) -> None:
     Mainly used for handling VPC/subnet errors before nodes are launched.
     """
     # NOTE: keep. The backend looks for this to know no nodes are launched.
-    prefix = 'SKYPILOT_ERROR_NO_NODES_LAUNCHED: '
-    raise RuntimeError(prefix + error)
+    full_error = f'SKYPILOT_ERROR_NO_NODES_LAUNCHED: {error}'
+    logger.error(full_error)
+    raise RuntimeError(full_error)
 
 
 def bootstrap_instances(
@@ -222,10 +223,27 @@ def _configure_iam_role(iam) -> Dict[str, Any]:
 
 
 @functools.lru_cache(maxsize=128)  # Keep bounded.
-def _get_route_tables(ec2, vpc_id: Optional[str], main: bool) -> List[Any]:
+def _get_route_tables(ec2, vpc_id: Optional[str], region: str,
+                      main: bool) -> List[Any]:
+    """Get route tables associated with a VPC and region
+
+    Args:
+        ec2: ec2 resource object
+        vpc_id: vpc_id is optional, if not provided, all route tables in the
+                region will be returned
+        region: region is mandatory to allow the lru cache
+                   to return the corect results
+        main: if True, only main route tables will be returned otherwise
+                only non-main route tables will be returned
+
+    Returns:
+        A list of route tables associated with the options VPC and region
+    """
     filters = [{'Name': 'association.main', 'Values': [str(main).lower()]}]
     if vpc_id is not None:
         filters.append({'Name': 'vpc-id', 'Values': [vpc_id]})
+    logger.debug(
+        f'Getting route tables with filters: {filters} in region: {region}')
     return ec2.meta.client.describe_route_tables(Filters=filters).get(
         'RouteTables', [])
 
@@ -238,7 +256,8 @@ def _is_subnet_public(ec2, subnet_id, vpc_id: Optional[str]) -> bool:
     https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html
     """
     # Get the route tables associated with the subnet
-    all_route_tables = _get_route_tables(ec2, vpc_id, main=False)
+    region = ec2.meta.client.meta.region_name
+    all_route_tables = _get_route_tables(ec2, vpc_id, region, main=False)
     route_tables = [
         rt for rt in all_route_tables
         # An RT can be associated with multiple subnets, i.e.,
@@ -267,7 +286,8 @@ def _is_subnet_public(ec2, subnet_id, vpc_id: Optional[str]) -> bool:
     # subnets. Since the associations are implicit, the filter above won't find
     # any. Check there exists a main route table with routes pointing to an IGW.
     logger.debug('Checking main route table')
-    main_route_tables = _get_route_tables(ec2, vpc_id, main=True)
+    region = ec2.meta.client.meta.region_name
+    main_route_tables = _get_route_tables(ec2, vpc_id, region, main=True)
     return _has_igw_route(main_route_tables)
 
 

sky/provision/instance_setup.py

@@ -264,6 +264,7 @@ def start_ray_on_head_node(cluster_name: str, custom_resource: Optional[str],
         f'--disable-usage-stats '
         f'--port={constants.SKY_REMOTE_RAY_PORT} '
         f'--dashboard-port={constants.SKY_REMOTE_RAY_DASHBOARD_PORT} '
+        f'--min-worker-port 11002 '
         f'--object-manager-port=8076 '
         f'--temp-dir={constants.SKY_REMOTE_RAY_TEMPDIR}')
     if custom_resource:

sky/provision/runpod/instance.py

@@ -232,7 +232,12 @@ def query_ports(
         instances = _filter_instances(cluster_name_on_cloud,
                                       None,
                                       head_only=True)
-        assert len(instances) == 1
+        assert len(instances) <= 1
+        # It is possible that the instance is terminated on console by
+        # the user. In this case, the instance will not be found and we
+        # should return an empty dict.
+        if not instances:
+            return {}
         head_inst = list(instances.values())[0]
         ready_ports: Dict[int, List[common.Endpoint]] = {
             port: [common.SocketEndpoint(**endpoint)]

sky/serve/core.py

@@ -124,7 +124,9 @@ def up(
                              f'{constants.CLUSTER_NAME_VALID_REGEX}')
 
     _validate_service_task(task)
-
+    # Always apply the policy again here, even though it might have been applied
+    # in the CLI. This is to ensure that we apply the policy to the final DAG
+    # and get the mutated config.
     dag, mutated_user_config = admin_policy_utils.apply(
         task, use_mutated_config_in_current_request=False)
     task = dag.tasks[0]
@@ -319,6 +321,14 @@ def update(
         service_name: Name of the service.
     """
     _validate_service_task(task)
+    # Always apply the policy again here, even though it might have been applied
+    # in the CLI. This is to ensure that we apply the policy to the final DAG
+    # and get the mutated config.
+    # TODO(cblmemo,zhwu): If a user sets a new skypilot_config, the update
+    # will not apply the config.
+    dag, _ = admin_policy_utils.apply(
+        task, use_mutated_config_in_current_request=False)
+    task = dag.tasks[0]
     handle = backend_utils.is_controller_accessible(
         controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
         stopped_message=

sky/skylet/constants.py

@@ -79,7 +79,7 @@ SKYLET_VERSION = '8'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.
-SKYLET_LIB_VERSION = 1
+SKYLET_LIB_VERSION = 2
 SKYLET_VERSION_FILE = '~/.sky/skylet_version'
 
 # `sky jobs dashboard`-related

sky/skylet/job_lib.py

@@ -29,6 +29,7 @@ if typing.TYPE_CHECKING:
 
 logger = sky_logging.init_logger(__name__)
 
+_LINUX_NEW_LINE = '\n'
 _JOB_STATUS_LOCK = '~/.sky/locks/.job_{}.lock'
 
 
@@ -602,6 +603,7 @@ def update_job_status(job_ids: List[int],
                 # the pending table until appearing in ray jobs. For jobs
                 # submitted outside of the grace period, we will consider the
                 # ray job status.
+
                 if not (pending_job['submit'] > 0 and pending_job['submit'] <
                         ray_job_query_time - _PENDING_SUBMIT_GRACE_PERIOD):
                     # Reset the job status to PENDING even though it may not
@@ -903,14 +905,19 @@ class JobLibCodeGen:
     def tail_logs(cls,
                   job_id: Optional[int],
                   managed_job_id: Optional[int],
-                  follow: bool = True) -> str:
+                  follow: bool = True,
+                  tail: int = 0) -> str:
         # pylint: disable=line-too-long
+
         code = [
+            # We use != instead of is not because 1 is not None will print a warning:
+            # <stdin>:1: SyntaxWarning: "is not" with a literal. Did you mean "!="?
             f'job_id = {job_id} if {job_id} != None else job_lib.get_latest_job_id()',
             'run_timestamp = job_lib.get_run_timestamp(job_id)',
             f'log_dir = None if run_timestamp is None else os.path.join({constants.SKY_LOGS_DIRECTORY!r}, run_timestamp)',
-            f'log_lib.tail_logs(job_id=job_id, log_dir=log_dir, '
-            f'managed_job_id={managed_job_id!r}, follow={follow})',
+            f'tail_log_kwargs = {{"job_id": job_id, "log_dir": log_dir, "managed_job_id": {managed_job_id!r}, "follow": {follow}}}',
+            f'{_LINUX_NEW_LINE}if getattr(constants, "SKYLET_LIB_VERSION", 1) > 1: tail_log_kwargs["tail"] = {tail}',
+            f'{_LINUX_NEW_LINE}log_lib.tail_logs(**tail_log_kwargs)',
         ]
         return cls._build(code)
 

sky/skylet/log_lib.py

@@ -2,6 +2,7 @@
 
 This is a remote utility module that provides logging functionality.
 """
+import collections
 import copy
 import io
 import multiprocessing.pool
@@ -12,7 +13,8 @@ import sys
 import tempfile
 import textwrap
 import time
-from typing import Dict, Iterator, List, Optional, Tuple, Union
+from typing import (Deque, Dict, Iterable, Iterator, List, Optional, TextIO,
+                    Tuple, Union)
 
 import colorama
 
@@ -26,6 +28,9 @@ from sky.utils import ux_utils
 _SKY_LOG_WAITING_GAP_SECONDS = 1
 _SKY_LOG_WAITING_MAX_RETRY = 5
 _SKY_LOG_TAILING_GAP_SECONDS = 0.2
+# Peek the head of the lines to check if we need to start
+# streaming when tail > 0.
+PEEK_HEAD_LINES_FOR_START_STREAM = 20
 
 logger = sky_logging.init_logger(__name__)
 
@@ -330,6 +335,7 @@ def run_bash_command_with_log(bash_command: str,
 
 def _follow_job_logs(file,
                      job_id: int,
+                     start_streaming: bool,
                      start_streaming_at: str = '') -> Iterator[str]:
     """Yield each line from a file as they are written.
 
@@ -338,7 +344,6 @@ def _follow_job_logs(file,
     # No need to lock the status here, as the while loop can handle
     # the older status.
     status = job_lib.get_status_no_lock(job_id)
-    start_streaming = False
     wait_last_logs = True
     while True:
         tmp = file.readline()
@@ -378,10 +383,45 @@ def _follow_job_logs(file,
             status = job_lib.get_status_no_lock(job_id)
 
 
+def _peek_head_lines(log_file: TextIO) -> List[str]:
+    """Peek the head of the file."""
+    lines = [
+        log_file.readline() for _ in range(PEEK_HEAD_LINES_FOR_START_STREAM)
+    ]
+    # Reset the file pointer to the beginning
+    log_file.seek(0, os.SEEK_SET)
+    return [line for line in lines if line]
+
+
+def _should_stream_the_whole_tail_lines(head_lines_of_log_file: List[str],
+                                        tail_lines: Deque[str],
+                                        start_stream_at: str) -> bool:
+    """Check if the entire tail lines should be streamed."""
+    # See comment:
+    # https://github.com/skypilot-org/skypilot/pull/4241#discussion_r1833611567
+    # for more details.
+    # Case 1: If start_stream_at is found at the head of the tail lines,
+    # we should not stream the whole tail lines.
+    for index, line in enumerate(tail_lines):
+        if index >= PEEK_HEAD_LINES_FOR_START_STREAM:
+            break
+        if start_stream_at in line:
+            return False
+    # Case 2: If start_stream_at is found at the head of log file, but not at
+    # the tail lines, we need to stream the whole tail lines.
+    for line in head_lines_of_log_file:
+        if start_stream_at in line:
+            return True
+    # Case 3: If start_stream_at is not at the head, and not found at the tail
+    # lines, we should not stream the whole tail lines.
+    return False
+
+
 def tail_logs(job_id: Optional[int],
               log_dir: Optional[str],
               managed_job_id: Optional[int] = None,
-              follow: bool = True) -> None:
+              follow: bool = True,
+              tail: int = 0) -> None:
     """Tail the logs of a job.
 
     Args:
@@ -390,6 +430,8 @@ def tail_logs(job_id: Optional[int],
         managed_job_id: The managed job id (for logging info only to avoid
             confusion).
         follow: Whether to follow the logs or print the logs so far and exit.
+        tail: The number of lines to display from the end of the log file,
+            if 0, print all lines.
     """
     if job_id is None:
         # This only happens when job_lib.get_latest_job_id() returns None,
@@ -430,6 +472,8 @@ def tail_logs(job_id: Optional[int],
         status = job_lib.update_job_status([job_id], silent=True)[0]
 
     start_stream_at = 'Waiting for task resources on '
+    # Explicitly declare the type to avoid mypy warning.
+    lines: Iterable[str] = []
     if follow and status in [
             job_lib.JobStatus.SETTING_UP,
             job_lib.JobStatus.PENDING,
@@ -440,18 +484,43 @@ def tail_logs(job_id: Optional[int],
         with open(log_path, 'r', newline='', encoding='utf-8') as log_file:
             # Using `_follow` instead of `tail -f` to streaming the whole
             # log and creating a new process for tail.
+            start_streaming = False
+            if tail > 0:
+                head_lines_of_log_file = _peek_head_lines(log_file)
+                lines = collections.deque(log_file, maxlen=tail)
+                start_streaming = _should_stream_the_whole_tail_lines(
+                    head_lines_of_log_file, lines, start_stream_at)
+                for line in lines:
+                    if start_stream_at in line:
+                        start_streaming = True
+                    if start_streaming:
+                        print(line, end='')
+                # Flush the last n lines
+                print(end='', flush=True)
+            # Now, the cursor is at the end of the last lines
+            # if tail > 0
             for line in _follow_job_logs(log_file,
                                          job_id=job_id,
+                                         start_streaming=start_streaming,
                                          start_streaming_at=start_stream_at):
                 print(line, end='', flush=True)
     else:
         try:
-            start_stream = False
-            with open(log_path, 'r', encoding='utf-8') as f:
-                for line in f.readlines():
+            start_streaming = False
+            with open(log_path, 'r', encoding='utf-8') as log_file:
+                if tail > 0:
+                    # If tail > 0, we need to read the last n lines.
+                    # We use double ended queue to rotate the last n lines.
+                    head_lines_of_log_file = _peek_head_lines(log_file)
+                    lines = collections.deque(log_file, maxlen=tail)
+                    start_streaming = _should_stream_the_whole_tail_lines(
+                        head_lines_of_log_file, lines, start_stream_at)
+                else:
+                    lines = log_file
+                for line in lines:
                     if start_stream_at in line:
-                        start_stream = True
-                    if start_stream:
+                        start_streaming = True
+                    if start_streaming:
                         print(line, end='', flush=True)
         except FileNotFoundError:
             print(f'{colorama.Fore.RED}ERROR: Logs for job {job_id} (status:'

sky/templates/kubernetes-ray.yml.j2

@@ -324,6 +324,8 @@ available_node_types:
           command: ["/bin/bash", "-c", "--"]
           args: 
             - |
+              function mylsof { p=$(for pid in /proc/{0..9}*; do i=$(basename "$pid"); for file in "$pid"/fd/*; do link=$(readlink -e "$file"); if [ "$link" = "$1" ]; then echo "$i"; fi; done; done); echo "$p"; };
+
               # Tails file and checks every 5 sec for
               # open file handlers with write access
               # closes if none exist
@@ -333,7 +335,7 @@ available_node_types:
                 while kill -0 $TAIL_PID 2> /dev/null; do
                   # only two PIDs should be accessing the file
                   # the log appender and log tailer
-                  if [ $(lsof -w $file | wc -l) -lt 3 ]; then
+                  if [ $(mylsof $file | wc -l) -lt 2 ]; then
                     kill $TAIL_PID
                     break
                   fi

sky/utils/admin_policy_utils.py

@@ -142,4 +142,5 @@ def apply(
             importlib.reload(skypilot_config)
 
     logger.debug(f'Mutated user request: {mutated_user_request}')
+    mutated_dag.policy_applied = True
     return mutated_dag, mutated_config

sky/utils/command_runner.py

@@ -11,6 +11,7 @@ from sky import sky_logging
 from sky.skylet import constants
 from sky.skylet import log_lib
 from sky.utils import common_utils
+from sky.utils import control_master_utils
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 
@@ -104,13 +105,22 @@ def ssh_options_list(
     }
     # SSH Control will have a severe delay when using docker_ssh_proxy_command.
     # TODO(tian): Investigate why.
+    #
+    # We disable ControlMaster when ssh_proxy_command is used, because the
+    # master connection will be idle although the connection might be shared
+    # by other ssh commands that is not idle. In that case, user's custom proxy
+    # command may drop the connection due to idle timeout, since it will only
+    # see the idle master connection. It is an issue even with the
+    # ServerAliveInterval set, since the keepalive message may not be recognized
+    # by the custom proxy command, such as AWS SSM Session Manager.
+    #
     # We also do not use ControlMaster when we use `kubectl port-forward`
     # to access Kubernetes pods over SSH+Proxycommand. This is because the
     # process running ProxyCommand is kept running as long as the ssh session
     # is running and the ControlMaster keeps the session, which results in
     # 'ControlPersist' number of seconds delay per ssh commands ran.
     if (ssh_control_name is not None and docker_ssh_proxy_command is None and
-            not disable_control_master):
+            ssh_proxy_command is None and not disable_control_master):
         arg_dict.update({
             # Control path: important optimization as we do multiple ssh in one
             # sky.launch().
@@ -459,7 +469,9 @@ class SSHCommandRunner(CommandRunner):
             None if ssh_control_name is None else hashlib.md5(
                 ssh_control_name.encode()).hexdigest()[:_HASH_MAX_LENGTH])
         self._ssh_proxy_command = ssh_proxy_command
-        self.disable_control_master = disable_control_master
+        self.disable_control_master = (
+            disable_control_master or
+            control_master_utils.should_disable_control_master())
         if docker_user is not None:
             assert port is None or port == 22, (
                 f'port must be None or 22 for docker_user, got {port}.')

sky/utils/control_master_utils.py

@@ -0,0 +1,49 @@
+"""Utils to check if the ssh control master should be disabled."""
+
+import functools
+
+from sky import sky_logging
+from sky.utils import subprocess_utils
+
+logger = sky_logging.init_logger(__name__)
+
+
+def is_tmp_9p_filesystem() -> bool:
+    """Check if the /tmp filesystem is 9p.
+
+    Returns:
+        bool: True if the /tmp filesystem is 9p, False otherwise.
+    """
+
+    result = subprocess_utils.run(['df', '-T', '/tmp'],
+                                  capture_output=True,
+                                  text=True,
+                                  shell=None,
+                                  check=False,
+                                  executable=None)
+
+    if result.returncode != 0:
+        return False
+
+    filesystem_infos = result.stdout.strip().split('\n')
+    if len(filesystem_infos) < 2:
+        return False
+    filesystem_types = filesystem_infos[1].split()
+    if len(filesystem_types) < 2:
+        return False
+    return filesystem_types[1].lower() == '9p'
+
+
+@functools.lru_cache
+def should_disable_control_master() -> bool:
+    """Whether disable ssh control master based on file system.
+
+    Returns:
+        bool: True if the ssh control master should be disabled,
+        False otherwise.
+    """
+    if is_tmp_9p_filesystem():
+        return True
+    # there may be additional criteria to disable ssh control master
+    # in the future. They should be checked here
+    return False

{skypilot_nightly-1.0.0.dev20241107.dist-info → skypilot_nightly-1.0.0.dev20241109.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241107
+Version: 1.0.0.dev20241109
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241107.dist-info → skypilot_nightly-1.0.0.dev20241109.dist-info}/RECORD

@@ -1,13 +1,13 @@
-sky/__init__.py,sha256=LDYVc006Bm6m_yCUJiTKF3oPp3_O3ODjp1KhoU5meCE,5882
+sky/__init__.py,sha256=vuaxCFFtQHJTriSEGG_wKshl6nmhDcnt70q66x1rkvA,5882
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=pAdCT60OxxiXI9KXDyP2lQ9u9vMc6aMtq5Xi2h_hbdw,20984
 sky/check.py,sha256=D3Y3saIFAYVvPxuBHnVgJEO0fUVDxgjwuMBaO-D778k,9472
-sky/cli.py,sha256=6umPcFovU5sHIUdC0B9lfOstzWLA0DPS5x6dg1EOkeQ,211193
+sky/cli.py,sha256=jEjXs5Z0u263eJIsTHoKyG9oOY6giqw19s2di9kEv1s,212088
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
-sky/core.py,sha256=DW9OGE2kS2CmsvQ1grrpRnNFS3woMGWSHu5GE99e-I4,38190
-sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
-sky/exceptions.py,sha256=KBIEJHgrw6OMBL8H65o-Gk6qYQEV1SR9gBwMjnMnxxg,8858
-sky/execution.py,sha256=HF76sz-gCEZPGkuL48jJaLOTqjuHg0KysgKaPw-hn84,25997
+sky/core.py,sha256=0-4W_DKJZgbwXuzNZKQ2R_qJxqxbqqNfyi0U0PQBKvQ,38230
+sky/dag.py,sha256=O9g8NnO8L1SGUEDyqW9W341AH4Wvd3nJs54niR-pkrk,2822
+sky/exceptions.py,sha256=E3C2Ejcc8RUDAUQn7ar_Jr97C_AxD2rKKMmJOfLJ9d0,8965
+sky/execution.py,sha256=TwcorzFxR_0m8uazPdeKltU3g3ikgUSqqzcSBrHp7K4,26070
 sky/global_user_state.py,sha256=PywEmUutF97XBgRMClR6IS5_KM8JJC0oA1LsPUZebp0,28681
 sky/optimizer.py,sha256=tXGrFpc6xNtKH34qjBAMd4jTuWcDZTPnGFwEtuCQFmk,59702
 sky/resources.py,sha256=Zt8mCCmdvZ5ZCqY-l3KXlx_lkUesAopRtaEcEsrRFZo,68465
@@ -31,10 +31,10 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=wwfbrxPhjMPs6PSyy3tAHI8WJhl-xhgzWBsAZjmJJ6g,6249
 sky/backends/backend_utils.py,sha256=2myfryj1zG9xxPaX6XYYJruxAOGNGbpsy2ckT4A77sE,121813
-sky/backends/cloud_vm_ray_backend.py,sha256=jlX1atSF4L31ZMzC_tnBaWnxvc2Wb8DRwt5G_ukrlJk,232799
+sky/backends/cloud_vm_ray_backend.py,sha256=6Ew9Ej92KGlumlCnyDcGSEbHInj7g2Shqwx4oxRkWVQ,233122
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=0JL5m0YUgOmOL4aWEUe4tmt89dsxjk4_WXkPwgEKEis,16801
-sky/backends/wheel_utils.py,sha256=3QS4T_Ydvo4DbYhogtyADyNBEf04I6jUCL71M285shQ,7963
+sky/backends/wheel_utils.py,sha256=CUVOwlBtQjOMv-RSDGx2jMQ0M1D0w9ZPm0TDafJwBDI,8180
 sky/backends/monkey_patches/monkey_patch_ray_up.py,sha256=76-y2fCaE3JINj8lEwHT1eirYzCbpD8O1ySsysuGu8o,3450
 sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
@@ -79,7 +79,7 @@ sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=L1JsX1YrhpyI7ylzE
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=35nO_VaDOgp5W13kt_lIANSk_CNf7gBiZGJ5fGyZu6o,6808
 sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=mDAN98T58h1g_LLyppSEUVDlsbLhk2454Nhmg5-aw0Q,32670
-sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=B7H14so38zayuJGgUrD1PJYJKiVZHGnwH6JJop3F7o0,4918
+sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=yOPmmckiQ0HU6bKXWd7YdTrsF2sql3Bs_jYNpuxlo0I,4942
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=SF_gTU74qg6L-DSWneCAbqP0lwZXaaDi5otiMIJbrw0,21462
 sky/clouds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/clouds/utils/aws_utils.py,sha256=W5BRC-2F_VY4BymRA1kS6-MufsI3V8cfY_hv--4gJBU,1986
@@ -90,29 +90,29 @@ sky/clouds/utils/scp_utils.py,sha256=RUp7NwyhKygOoVOwvdAOGdoQNSJjryOG6WSExCf-yas
 sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=MBmjey9_p2L3IKNKTi8um09SlZe32n4wK3CkVnlTVvo,7346
 sky/data/data_utils.py,sha256=-P5GsDH_m4slrCz4vHdgiFezIys8ufzvhEKePJwfjFc,28597
-sky/data/mounting_utils.py,sha256=44YkYIIgArEkyvxCtfmXXumybrU8bmn1TfLXWv_eldI,11480
+sky/data/mounting_utils.py,sha256=HwBGg1NmX-2IJZV_6h2r1U3ajTGOyfmA3MqboA7znqU,11004
 sky/data/storage.py,sha256=OQ_kznF-P50Jq0feO5FBqm97QGhfbsZ2dX-Ar3sVWr4,163903
 sky/data/storage_utils.py,sha256=cM3kxlffYE7PnJySDu8huyUsMX_JYsf9uer8r5OYsjo,9556
 sky/jobs/__init__.py,sha256=yucibSB_ZimtJMvOhMxn6ZqwBIYNfcwmc6pSXtCqmNQ,1483
 sky/jobs/constants.py,sha256=YLgcCg_RHSYr_rfsI_4UIdXk78KKKOK29Oem88t5j8I,1350
 sky/jobs/controller.py,sha256=sirpi730_GfKfPZeZ2PvCXnJWger0r6AyLSOx2sLd6A,27368
-sky/jobs/core.py,sha256=w7PancHi8_-afLKZQ3HHMD1sEDoepm1vEMxyDlXdo64,17155
-sky/jobs/recovery_strategy.py,sha256=FpPK6e2PT61cZPDUJqIfo6g53uSRTBh7dOTbfR1DLVE,26672
-sky/jobs/state.py,sha256=exN6BdJlLBzFTccJCSHN4dNjVeYFgTgqgxOaHwLw2IQ,24307
-sky/jobs/utils.py,sha256=pF4Kyl4v1M_Bmm2jIRlXGTSdII5BJ3f4qwex_oCFgBk,37742
-sky/jobs/dashboard/dashboard.py,sha256=FXVQAWjAuQQTfAGlTCD-Xb9LckC5I4NhGwiBZy8Avo8,3186
+sky/jobs/core.py,sha256=Lk_zKizc9a7O-8WHhh4-VXBS5kT0jRpwmNNA7S4ueIo,17347
+sky/jobs/recovery_strategy.py,sha256=O_DouAfWx8FNdQxXsr2msMwlKCIodS99cW6V4Lf1vMo,27219
+sky/jobs/state.py,sha256=DE02bCZc9bPbbuayb3Zml553mb4pEV7Z8t1pt8IGbYM,25252
+sky/jobs/utils.py,sha256=Ff3TttIEdVeM1_kOVkviqIDjeVfBPIXVE8i-yP1VDM8,37976
+sky/jobs/dashboard/dashboard.py,sha256=KMSarpVcfnc-ELPFvy1M9_I1k4kSeXubTk3ibQC67Tg,3219
 sky/jobs/dashboard/static/favicon.ico,sha256=uYlvgxSM7gjBmXpZ8wydvZUPAbJiiix-rc2Xe5mma9s,15086
 sky/jobs/dashboard/templates/index.html,sha256=su1tqgcsXNl1lGl9hfIR6ig1f531OO57x1Tc2mNDK7U,11139
 sky/provision/__init__.py,sha256=UhYsGRribEyK1--PPT0Dom9051jlpdn8UCNhO8qpPOc,6262
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
 sky/provision/docker_utils.py,sha256=cKYasCwbMf6C2_0vTxg2GvbrnhFvko-xDl1frfm7wxc,19199
-sky/provision/instance_setup.py,sha256=c6i_NC6GrW4hXAQIU5_dUBbnThjZQNS3cL2M6yMtzes,23616
+sky/provision/instance_setup.py,sha256=gI739UMCqtPqdA522D92bPu5sA3OHBMDmIGmqqxsIwY,23652
 sky/provision/logging.py,sha256=yZWgejrFBhhRjAtvFu5N5bRXIMK5TuwNjp1vKQqz2pw,2103
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
 sky/provision/provisioner.py,sha256=mTvtBjS-Xz64LJcyeHx_-wdM8Gin8D49YRaV_TADaz4,25334
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
-sky/provision/aws/config.py,sha256=ApEh63RR_KyCp9nPXX35z6jBREoulJPQ5st8K9Jlclo,23385
+sky/provision/aws/config.py,sha256=dbwulPxXGIJjKJddv85PbtlXOjwLemaD65j3DISNsK0,24214
 sky/provision/aws/instance.py,sha256=eCslJ2XfJo_pkQMnKFQqhGnUIRvwKiT12oxBY5-klss,40750
 sky/provision/aws/utils.py,sha256=m49pS-SHGW7Au3bhDeTPsL8N5iRzbwOXzyEWRCc1Vho,3238
 sky/provision/azure/__init__.py,sha256=87cgk1_Ws7n9rqaDDPv-HpfrkVeSQMdFQnhnXwyx9g4,548
@@ -154,7 +154,7 @@ sky/provision/paperspace/instance.py,sha256=q_V01DZSMXLfy63Zwt6AQotq02JuXQZb5CHS
 sky/provision/paperspace/utils.py,sha256=uOmxbDKjV6skFizC4gYXSxRuEqso5ck2kF7MbtNmhEs,9580
 sky/provision/runpod/__init__.py,sha256=6HYvHI27EaLrX1SS0vWVhdLu5HDBeZCdvAeDJuwM5pk,556
 sky/provision/runpod/config.py,sha256=9ulZJVL7nHuxhTdoj8D7lNn7SdicJ5zc6FIcHIG9tcg,321
-sky/provision/runpod/instance.py,sha256=ucmFQEzapbxylsl6K9EUo7bHTZYzvfECo6tpJc-MFrw,9577
+sky/provision/runpod/instance.py,sha256=AIWzTHuAu2dw8Rk-AHc7-14hUAYPEKh_UMzAhMzjDh0,9807
 sky/provision/runpod/utils.py,sha256=ZjrcpjKzwS2nXQ21dW405PLxBl_V9awcfRjucGB3alw,6795
 sky/provision/vsphere/__init__.py,sha256=5icB8-kfs926S9DVfNJSCBVr7z7cmCEDr04-YHX89_4,788
 sky/provision/vsphere/config.py,sha256=f_ojGmi_vbnwJ8Ri48cqhZHBOuIkj41j9bFbq-ldPOo,504
@@ -175,7 +175,7 @@ sky/serve/__init__.py,sha256=gFZt7W3UPMi4qvYe2xgkHg1VxbR1WGavKyWLBUD3mpg,1731
 sky/serve/autoscalers.py,sha256=khY1oZ22PRaUQNsLCoNKH178X_NiJw0LSLOKr7_LNgY,30275
 sky/serve/constants.py,sha256=7MflfgTHO9gDSux93U4BmNeEMWXxZB4q7I54KUwgp-s,4651
 sky/serve/controller.py,sha256=R5iIEGEEFtbm_6MvSGelYZP-vSmW0cSFuy64OexUc4g,11719
-sky/serve/core.py,sha256=jwrgglvtFqbD9Y4pzXmuso5hKc0OQcTWJ-AkvypiQII,30986
+sky/serve/core.py,sha256=hszs95BwtC4wIJujGNokvFC46VjojgRz1BbYOIIPh6k,31601
 sky/serve/load_balancer.py,sha256=aUfDsgUT_fYrchCwJCeunMPXmAkwJAY58BEu-IN2FaA,11571
 sky/serve/load_balancing_policies.py,sha256=ExdwH_pxPYpJ6CkoTQCOPSa4lzwbq1LFFMKzmIu8ryk,2331
 sky/serve/replica_managers.py,sha256=1xYDK9Te5wFEF5hUK0gyNIUib0MY-HScLHUBDlTSl-k,57774
@@ -190,10 +190,10 @@ sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=JPDHmByuhoNYXSUHl-OnyeJUkOFWn7gDM1FrS7Kr3E8,4478
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=TL-O0ZoxA1ZeNvKXzzA_UyIMXsma7flbsDZ1N_o9dKg,14468
+sky/skylet/constants.py,sha256=w05Enrg9RhGp99P1WDYMKK_ki0M-e0bS8Wr-VZR0Vn8,14468
 sky/skylet/events.py,sha256=A09E7LmmwzcGrSG0n8K7d3EZ1ZJr1mmmzoGyhnArYJA,12303
-sky/skylet/job_lib.py,sha256=-SCbpJRiWMSwvhDjUwfwnvBap7Y5B3ol1l_PDPra3XI,36860
-sky/skylet/log_lib.py,sha256=Jyj3h2yMBlheFX53AabXEiPaKyCbu06hLEhay5_ZRN0,18734
+sky/skylet/job_lib.py,sha256=FD1n9vE0daOEUKSH3lnccfBh7Vs81R8s4ILZyKu2o7M,37275
+sky/skylet/log_lib.py,sha256=BmhAgcLvlin3szhj33IH0kbdCALacVisF2x61BQpZdY,21888
 sky/skylet/log_lib.pyi,sha256=AHMkW2DGK2erFovb3ToZWxRiYaATlzkxKb5J9pkgF2Y,4295
 sky/skylet/skylet.py,sha256=U9plr5hmhD9-Nyy0LMCymlE8DWtRXTFXQvfbFsS746Y,1153
 sky/skylet/subprocess_daemon.py,sha256=IJwGAzOdERrhWJS7VYKAUByNsLyIkKkB0w5nk06okG8,2818
@@ -228,7 +228,7 @@ sky/templates/jobs-controller.yaml.j2,sha256=Gu3ogFxFYr09VEXP-6zEbrCUOFo1aYxWEjA
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=HlG7CPBBedCVBlL9qv0erW_eKm6Irj0LFyaAWuJW_lc,3148
-sky/templates/kubernetes-ray.yml.j2,sha256=Wq9luXc6-t141uyHbtOy1IDmLMM0PBbePTZfZEtAKw0,18160
+sky/templates/kubernetes-ray.yml.j2,sha256=dsWlkX-0b1igeZI4c0u0Jzia5I_9gezCiewR6pX1LlY,18374
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
 sky/templates/lambda-ray.yml.j2,sha256=HyvO_tX2vxwSsc4IFVSqGuIbjLMk0bevP9bcxb8ZQII,4498
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
@@ -243,11 +243,12 @@ sky/usage/constants.py,sha256=8xpg9vhDU9A3eObtpkNFjwa42oCazqGEv4yw_vJSO7U,590
 sky/usage/usage_lib.py,sha256=mxsbwUMEQjesUOIv4Yne-Ze7rVxSQYr3_wBXruifGRA,17898
 sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
-sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
+sky/utils/admin_policy_utils.py,sha256=_Vt_jTTYCXmMdryj0vrrumFPewa93qHnzUqBDXjAhRU,5981
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
-sky/utils/command_runner.py,sha256=seU7uX9CrxiC8WOWBKHW94m67-V6DYghqRXhYdUIdQI,35756
+sky/utils/command_runner.py,sha256=GHTZxoJQ3V8WVSRAaOA4JpRTxtCtuq36H9U8kOfWUwc,36450
 sky/utils/command_runner.pyi,sha256=mJOzCgcYZAfHwnY_6Wf1YwlTEJGb9ihzc2f0rE0Kw98,7751
 sky/utils/common_utils.py,sha256=Qy25LuIoTT0qg391EWyT9i5D6fwk1S4OdFwRpCTZ9Vk,24657
+sky/utils/control_master_utils.py,sha256=90hnxiAUP20gbJ9e3MERh7rb04ZO_I3LsljNjR26H5I,1416
 sky/utils/controller_utils.py,sha256=wF4_y1PCsLAWoo3XEtECwkNYTN6hO3vn_cxGxgQYcd8,43268
 sky/utils/dag_utils.py,sha256=pVX3lGDDcYTcGoH_1jEWzl9767Y4mwlIEYIzoyHO6gM,6105
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
@@ -274,9 +275,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7Z
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=hyYDaYSNxYaNvzUQBzC8AidB7nDeojizjkzc_CTxycY,1077
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241107.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241107.dist-info/METADATA,sha256=ICnKtcpMVvZVf_1H6k63r29XgS_-heZ4BcgH-p5J5s4,19708
-skypilot_nightly-1.0.0.dev20241107.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-skypilot_nightly-1.0.0.dev20241107.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241107.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241107.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241109.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241109.dist-info/METADATA,sha256=YM8C71GXOj5CoHQlj5yNYhL8UkZ75DL-qMMTPXCOmXY,19708
+skypilot_nightly-1.0.0.dev20241109.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+skypilot_nightly-1.0.0.dev20241109.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241109.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241109.dist-info/RECORD,,