skypilot-nightly 1.0.0.dev20250228__py3-none-any.whl → 1.0.0.dev20250301__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '0f178b2af47ec2b185bc685cce6965c675613cc7'
+_SKYPILOT_COMMIT_SHA = 'cefc23846d47d7dacf960093c2001912a83c1162'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250228'
+__version__ = '1.0.0.dev20250301'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/backend_utils.py

@@ -1280,12 +1280,17 @@ def parallel_data_transfer_to_nodes(
                                             stream_logs=stream_logs,
                                             require_outputs=True,
                                             source_bashrc=source_bashrc)
-            err_msg = ('Failed to run command before rsync '
+            err_msg = (f'{colorama.Style.RESET_ALL}{colorama.Style.DIM}'
+                       f'----- CMD -----\n'
+                       f'{cmd}\n'
+                       f'----- CMD END -----\n'
+                       f'{colorama.Style.RESET_ALL}'
+                       f'{colorama.Fore.RED}'
+                       f'Failed to run command before rsync '
                        f'{origin_source} -> {target}. '
-                       'Ensure that the network is stable, then retry. '
-                       f'{cmd}')
+                       f'{colorama.Style.RESET_ALL}')
             if log_path != os.devnull:
-                err_msg += f' See logs in {log_path}'
+                err_msg += ux_utils.log_path_hint(log_path)
             subprocess_utils.handle_returncode(rc,
                                                cmd,
                                                err_msg,

sky/cli.py

@@ -5419,11 +5419,16 @@ def local():
 @click.option('--cleanup',
               is_flag=True,
               help='Clean up the remote cluster instead of deploying it.')
+@click.option(
+    '--context-name',
+    type=str,
+    required=False,
+    help='Name to use for the kubeconfig context. Defaults to "default".')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, async_call: bool):
+             cleanup: bool, context_name: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5468,7 +5473,8 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
             raise click.BadParameter(
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
-    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup)
+    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
+                              context_name)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 
@@ -5504,10 +5510,19 @@ def api():
               required=False,
               help=('The host to deploy the SkyPilot API server. To allow '
                     'remote access, set this to 0.0.0.0'))
+@click.option('--foreground',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Run the SkyPilot API server in the foreground and output '
+              'its logs to stdout/stderr. Allowing external systems '
+              'to manage the process lifecycle and collect logs directly. '
+              'This is useful when the API server is managed by systems '
+              'like systemd and Kubernetes.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str]):
+def api_start(deploy: bool, host: Optional[str], foreground: bool):
     """Starts the SkyPilot API server locally."""
-    sdk.api_start(deploy=deploy, host=host)
+    sdk.api_start(deploy=deploy, host=host, foreground=foreground)
 
 
 @api.command('stop', cls=_DocumentedCodeCommand)

sky/client/cli.py

@@ -5419,11 +5419,16 @@ def local():
 @click.option('--cleanup',
               is_flag=True,
               help='Clean up the remote cluster instead of deploying it.')
+@click.option(
+    '--context-name',
+    type=str,
+    required=False,
+    help='Name to use for the kubeconfig context. Defaults to "default".')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, async_call: bool):
+             cleanup: bool, context_name: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5468,7 +5473,8 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
             raise click.BadParameter(
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
-    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup)
+    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
+                              context_name)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 
@@ -5504,10 +5510,19 @@ def api():
               required=False,
               help=('The host to deploy the SkyPilot API server. To allow '
                     'remote access, set this to 0.0.0.0'))
+@click.option('--foreground',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Run the SkyPilot API server in the foreground and output '
+              'its logs to stdout/stderr. Allowing external systems '
+              'to manage the process lifecycle and collect logs directly. '
+              'This is useful when the API server is managed by systems '
+              'like systemd and Kubernetes.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str]):
+def api_start(deploy: bool, host: Optional[str], foreground: bool):
     """Starts the SkyPilot API server locally."""
-    sdk.api_start(deploy=deploy, host=host)
+    sdk.api_start(deploy=deploy, host=host, foreground=foreground)
 
 
 @api.command('stop', cls=_DocumentedCodeCommand)

sky/client/sdk.py

@@ -1263,8 +1263,12 @@ def storage_delete(name: str) -> server_common.RequestId:
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
-             ssh_key: Optional[str], cleanup: bool) -> server_common.RequestId:
+def local_up(gpus: bool,
+             ips: Optional[List[str]],
+             ssh_user: Optional[str],
+             ssh_key: Optional[str],
+             cleanup: bool,
+             context_name: Optional[str] = None) -> server_common.RequestId:
     """Launches a Kubernetes cluster on local machines.
 
     Returns:
@@ -1282,7 +1286,8 @@ def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
                                 ips=ips,
                                 ssh_user=ssh_user,
                                 ssh_key=ssh_key,
-                                cleanup=cleanup)
+                                cleanup=cleanup,
+                                context_name=context_name)
     response = requests.post(f'{server_common.get_server_url()}/local_up',
                              json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)
@@ -1611,6 +1616,7 @@ def api_start(
     *,
     deploy: bool = False,
     host: str = '127.0.0.1',
+    foreground: bool = False,
 ) -> None:
     """Starts the API server.
 
@@ -1622,7 +1628,8 @@ def api_start(
             resources of the machine.
         host: The host to deploy the API server. It will be set to 0.0.0.0
             if deploy is True, to allow remote access.
-
+        foreground: Whether to run the API server in the foreground (run in
+            the current process).
     Returns:
         None
     """
@@ -1641,7 +1648,10 @@ def api_start(
                              'from the config file and/or unset the '
                              'SKYPILOT_API_SERVER_ENDPOINT environment '
                              'variable.')
-    server_common.check_server_healthy_or_start_fn(deploy, host)
+    server_common.check_server_healthy_or_start_fn(deploy, host, foreground)
+    if foreground:
+        # Explain why current process exited
+        logger.info('API server is already running:')
     logger.info(f'{ux_utils.INDENT_SYMBOL}SkyPilot API server: '
                 f'{server_common.get_server_url(host)}\n'
                 f'{ux_utils.INDENT_LAST_SYMBOL}'

sky/cloud_stores.py

@@ -54,8 +54,9 @@ class S3CloudStorage(CloudStorage):
 
     # List of commands to install AWS CLI
     _GET_AWSCLI = [
-        'aws --version >/dev/null 2>&1 || '
-        f'{constants.SKY_UV_PIP_CMD} install awscli',
+        'awscli_path=$(which aws) || '
+        f'{{ {constants.SKY_UV_PIP_CMD} install awscli && '
+        f'awscli_path={constants.SKY_REMOTE_PYTHON_ENV}/bin/aws; }}',
     ]
 
     def is_directory(self, url: str) -> bool:
@@ -85,8 +86,7 @@ class S3CloudStorage(CloudStorage):
         # AWS Sync by default uses 10 threads to upload files to the bucket.
         # To increase parallelism, modify max_concurrent_requests in your
         # aws config file (Default path: ~/.aws/config).
-        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
-                               'sync --no-follow-symlinks '
+        download_via_awscli = (f'$awscli_path s3 sync --no-follow-symlinks '
                                f'{source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
@@ -95,8 +95,7 @@ class S3CloudStorage(CloudStorage):
 
     def make_sync_file_command(self, source: str, destination: str) -> str:
         """Downloads a file using AWS CLI."""
-        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
-                               f'cp {source} {destination}')
+        download_via_awscli = (f'$awscli_path s3 cp {source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
         all_commands.append(download_via_awscli)

sky/clouds/gcp.py

@@ -67,8 +67,20 @@ _GCLOUD_VERSION = '424.0.0'
 GOOGLE_SDK_INSTALLATION_COMMAND: str = f'pushd /tmp &>/dev/null && \
     {{ gcloud --help > /dev/null 2>&1 || \
     {{ mkdir -p {os.path.dirname(_GCLOUD_INSTALLATION_LOG)} && \
-    wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-{_GCLOUD_VERSION}-linux-x86_64.tar.gz > {_GCLOUD_INSTALLATION_LOG} && \
-    tar xzf google-cloud-sdk-{_GCLOUD_VERSION}-linux-x86_64.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
+    ARCH=$(uname -m) && \
+    if [ "$ARCH" = "x86_64" ]; then \
+        echo "Installing Google Cloud SDK for $ARCH" > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="x86_64"; \
+    elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
+        echo "Installing Google Cloud SDK for $ARCH" > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="arm"; \
+    else \
+        echo "Architecture $ARCH not supported by Google Cloud SDK. Defaulting to x86_64." > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="x86_64"; \
+    fi && \
+    echo "Detected architecture: $ARCH, using package: $ARCH_SUFFIX" >> {_GCLOUD_INSTALLATION_LOG} && \
+    wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-{_GCLOUD_VERSION}-linux-${{ARCH_SUFFIX}}.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
+    tar xzf google-cloud-sdk-{_GCLOUD_VERSION}-linux-${{ARCH_SUFFIX}}.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
     rm -rf ~/google-cloud-sdk >> {_GCLOUD_INSTALLATION_LOG}  && \
     mv google-cloud-sdk ~/ && \
     ~/google-cloud-sdk/install.sh -q >> {_GCLOUD_INSTALLATION_LOG} 2>&1 && \

sky/core.py

@@ -1007,8 +1007,12 @@ def realtime_kubernetes_gpu_availability(
 # = Local Cluster =
 # =================
 @usage_lib.entrypoint
-def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
-             ssh_key: Optional[str], cleanup: bool) -> None:
+def local_up(gpus: bool,
+             ips: Optional[List[str]],
+             ssh_user: Optional[str],
+             ssh_key: Optional[str],
+             cleanup: bool,
+             context_name: Optional[str] = None) -> None:
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key, cleanup):
@@ -1034,7 +1038,7 @@ def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
     if ips:
         assert ssh_user is not None and ssh_key is not None
         kubernetes_deploy_utils.deploy_remote_cluster(ips, ssh_user, ssh_key,
-                                                      cleanup)
+                                                      cleanup, context_name)
     else:
         # Run local deployment (kind) if no remote args are specified
         kubernetes_deploy_utils.deploy_local_cluster(gpus)

sky/data/mounting_utils.py

@@ -24,7 +24,14 @@ RCLONE_VERSION = 'v1.68.2'
 
 def get_s3_mount_install_cmd() -> str:
     """Returns a command to install S3 mount utility goofys."""
-    install_cmd = ('sudo wget -nc https://github.com/romilbhardwaj/goofys/'
+    install_cmd = ('ARCH=$(uname -m) && '
+                   'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                   '  echo "goofys is not supported on $ARCH" && '
+                   f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+                   'else '
+                   '  ARCH_SUFFIX="amd64"; '
+                   'fi && '
+                   'sudo wget -nc https://github.com/romilbhardwaj/goofys/'
                    'releases/download/0.24.0-romilb-upstream/goofys '
                    '-O /usr/local/bin/goofys && '
                    'sudo chmod 755 /usr/local/bin/goofys')
@@ -49,9 +56,15 @@ def get_s3_mount_cmd(bucket_name: str,
 
 def get_gcs_mount_install_cmd() -> str:
     """Returns a command to install GCS mount utility gcsfuse."""
-    install_cmd = ('wget -nc https://github.com/GoogleCloudPlatform/gcsfuse'
+    install_cmd = ('ARCH=$(uname -m) && '
+                   'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                   '  ARCH_SUFFIX="arm64"; '
+                   'else '
+                   '  ARCH_SUFFIX="amd64"; '
+                   'fi && '
+                   'wget -nc https://github.com/GoogleCloudPlatform/gcsfuse'
                    f'/releases/download/v{GCSFUSE_VERSION}/'
-                   f'gcsfuse_{GCSFUSE_VERSION}_amd64.deb '
+                   f'gcsfuse_{GCSFUSE_VERSION}_${{ARCH_SUFFIX}}.deb '
                    '-O /tmp/gcsfuse.deb && '
                    'sudo dpkg --install /tmp/gcsfuse.deb')
     return install_cmd
@@ -77,16 +90,24 @@ def get_gcs_mount_cmd(bucket_name: str,
 
 def get_az_mount_install_cmd() -> str:
     """Returns a command to install AZ Container mount utility blobfuse2."""
-    install_cmd = ('sudo apt-get update; '
-                   'sudo apt-get install -y '
-                   '-o Dpkg::Options::="--force-confdef" '
-                   'fuse3 libfuse3-dev && '
-                   'wget -nc https://github.com/Azure/azure-storage-fuse'
-                   f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
-                   f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
-                   '-O /tmp/blobfuse2.deb && '
-                   'sudo dpkg --install /tmp/blobfuse2.deb && '
-                   f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
+    install_cmd = (
+        'sudo apt-get update; '
+        'sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        'fuse3 libfuse3-dev && '
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  echo "blobfuse2 is not supported on $ARCH" && '
+        f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'else '
+        '  ARCH_SUFFIX="x86_64"; '
+        'fi && '
+        'wget -nc https://github.com/Azure/azure-storage-fuse'
+        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
+        f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
+        '-O /tmp/blobfuse2.deb && '
+        'sudo dpkg --install /tmp/blobfuse2.deb && '
+        f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
 
     return install_cmd
 
@@ -207,14 +228,20 @@ def get_rclone_install_cmd() -> str:
     """
     # pylint: disable=line-too-long
     install_cmd = (
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  ARCH_SUFFIX="arm"; '
+        'else '
+        '  ARCH_SUFFIX="amd64"; '
+        'fi && '
         f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-amd64.deb'
-        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-amd64.deb'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-amd64.deb)))'
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
         f' || (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-amd64.rpm'
-        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-amd64.rpm -y'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-amd64.rpm))')
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
+        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm))')
     return install_cmd
 
 

sky/exceptions.py

@@ -22,6 +22,8 @@ MOUNT_PATH_NON_EMPTY_CODE = 42
 INSUFFICIENT_PRIVILEGES_CODE = 52
 # Return code when git command is ran in a dir that is not git repo
 GIT_FATAL_EXIT_CODE = 128
+# Architecture, such as arm64, not supported by the dependency
+ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 
 
 def is_safe_exception(exc: Exception) -> bool:

sky/server/common.py

@@ -145,57 +145,6 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     return ApiServerInfo(status=ApiServerStatus.UNHEALTHY, api_version=None)
 
 
-def start_api_server_in_background(deploy: bool = False,
-                                   host: str = '127.0.0.1'):
-    if not is_api_server_local():
-        raise RuntimeError(
-            f'Cannot start API server: {get_server_url()} is not a local URL')
-
-    # Check available memory before starting the server.
-    avail_mem_size_gb: float = common_utils.get_mem_size_gb()
-    if avail_mem_size_gb <= server_constants.MIN_AVAIL_MEM_GB:
-        logger.warning(
-            f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only has '
-            f'{avail_mem_size_gb:.1f}GB memory available. '
-            f'At least {server_constants.MIN_AVAIL_MEM_GB}GB is recommended to '
-            f'support higher load with better performance.'
-            f'{colorama.Style.RESET_ALL}')
-    log_path = os.path.expanduser(constants.API_SERVER_LOGS)
-    os.makedirs(os.path.dirname(log_path), exist_ok=True)
-
-    api_server_cmd = API_SERVER_CMD
-    if deploy:
-        api_server_cmd += ' --deploy'
-    if host is not None:
-        api_server_cmd += f' --host {host}'
-    cmd = f'{sys.executable} {api_server_cmd} > {log_path} 2>&1 < /dev/null'
-
-    # Start the API server process in the background and don't wait for it.
-    # If this is called from a CLI invocation, we need start_new_session=True so
-    # that SIGINT on the CLI will not also kill the API server.
-    subprocess.Popen(cmd, shell=True, start_new_session=True)
-
-    # Wait for the server to start until timeout.
-    # Conservative upper time bound for starting the server based on profiling.
-    timeout_sec = 12
-    start_time = time.time()
-    while True:
-        api_server_info = get_api_server_status()
-        assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
-            f'API server version mismatch when starting the server. '
-            f'Server version: {api_server_info.api_version} '
-            f'Client version: {server_constants.API_VERSION}')
-        if api_server_info.status == ApiServerStatus.HEALTHY:
-            break
-        elif time.time() - start_time >= timeout_sec:
-            with ux_utils.print_exception_no_traceback():
-                raise RuntimeError(
-                    'Failed to start SkyPilot API server at '
-                    f'{get_server_url(host)}'
-                    f'\nView logs at: {constants.API_SERVER_LOGS}')
-        time.sleep(0.5)
-
-
 def handle_request_error(response: requests.Response) -> None:
     if response.status_code != 200:
         with ux_utils.print_exception_no_traceback():
@@ -218,10 +167,10 @@ def get_request_id(response: requests.Response) -> RequestId:
     return request_id
 
 
-def _start_api_server(deploy: bool = False, host: str = '127.0.0.1'):
+def _start_api_server(deploy: bool = False,
+                      host: str = '127.0.0.1',
+                      foreground: bool = False):
     """Starts a SkyPilot API server locally."""
-    # Lock to prevent multiple processes from starting the server at the
-    # same time, causing issues with database initialization.
     server_url = get_server_url(host)
     assert server_url in AVAILABLE_LOCAL_API_SERVER_URLS, (
         f'server url {server_url} is not a local url')
@@ -230,7 +179,60 @@ def _start_api_server(deploy: bool = False, host: str = '127.0.0.1'):
                     f'SkyPilot API server at {server_url}. '
                     'Starting a local server.'
                     f'{colorama.Style.RESET_ALL}')
-        start_api_server_in_background(deploy=deploy, host=host)
+        if not is_api_server_local():
+            raise RuntimeError(f'Cannot start API server: {get_server_url()} '
+                               'is not a local URL')
+
+        # Check available memory before starting the server.
+        avail_mem_size_gb: float = common_utils.get_mem_size_gb()
+        if avail_mem_size_gb <= server_constants.MIN_AVAIL_MEM_GB:
+            logger.warning(
+                f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only '
+                f'has {avail_mem_size_gb:.1f}GB memory available. '
+                f'At least {server_constants.MIN_AVAIL_MEM_GB}GB is '
+                'recommended to support higher load with better performance.'
+                f'{colorama.Style.RESET_ALL}')
+
+        args = [sys.executable, *API_SERVER_CMD.split()]
+        if deploy:
+            args += ['--deploy']
+        if host is not None:
+            args += [f'--host={host}']
+
+        if foreground:
+            # Replaces the current process with the API server
+            os.execvp(args[0], args)
+
+        log_path = os.path.expanduser(constants.API_SERVER_LOGS)
+        os.makedirs(os.path.dirname(log_path), exist_ok=True)
+        cmd = f'{" ".join(args)} > {log_path} 2>&1 < /dev/null'
+
+        # Start the API server process in the background and don't wait for it.
+        # If this is called from a CLI invocation, we need
+        # start_new_session=True so that SIGINT on the CLI will not also kill
+        # the API server.
+        subprocess.Popen(cmd, shell=True, start_new_session=True)
+
+        # Wait for the server to start until timeout.
+        # Conservative upper time bound for starting the server based on
+        # profiling.
+        timeout_sec = 12
+        start_time = time.time()
+        while True:
+            api_server_info = get_api_server_status()
+            assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
+                f'API server version mismatch when starting the server. '
+                f'Server version: {api_server_info.api_version} '
+                f'Client version: {server_constants.API_VERSION}')
+            if api_server_info.status == ApiServerStatus.HEALTHY:
+                break
+            elif time.time() - start_time >= timeout_sec:
+                with ux_utils.print_exception_no_traceback():
+                    raise RuntimeError(
+                        'Failed to start SkyPilot API server at '
+                        f'{get_server_url(host)}'
+                        f'\nView logs at: {constants.API_SERVER_LOGS}')
+            time.sleep(0.5)
         logger.info(ux_utils.finishing_message('SkyPilot API server started.'))
 
 
@@ -260,7 +262,8 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
 
 
 def check_server_healthy_or_start_fn(deploy: bool = False,
-                                     host: str = '127.0.0.1'):
+                                     host: str = '127.0.0.1',
+                                     foreground: bool = False):
     try:
         check_server_healthy()
     except exceptions.ApiServerConnectionError as exc:
@@ -268,13 +271,15 @@ def check_server_healthy_or_start_fn(deploy: bool = False,
         if not is_api_server_local():
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.ApiServerConnectionError(endpoint) from exc
+        # Lock to prevent multiple processes from starting the server at the
+        # same time, causing issues with database initialization.
         with filelock.FileLock(
                 os.path.expanduser(constants.API_SERVER_CREATION_LOCK_PATH)):
             # Check again if server is already running. Other processes may
             # have started the server while we were waiting for the lock.
             api_server_info = get_api_server_status(endpoint)
             if api_server_info.status == ApiServerStatus.UNHEALTHY:
-                _start_api_server(deploy, host)
+                _start_api_server(deploy, host, foreground)
 
 
 def check_server_healthy_or_start(func):

sky/server/requests/executor.py

@@ -371,7 +371,8 @@ def request_worker(worker: RequestWorker, max_parallel_size: int) -> None:
             # TODO(aylei): crash the API server or recreate the worker process
             # to avoid broken state.
             logger.error(f'[{worker}] Worker process interrupted')
-            raise
+            with ux_utils.print_exception_no_traceback():
+                raise
         except (Exception, SystemExit) as e:  # pylint: disable=broad-except
             # Catch any other exceptions to avoid crashing the worker process.
             logger.error(
@@ -408,6 +409,7 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
         f'long requests and will allow at max '
         f'{max_parallel_for_short} short requests in parallel.')
 
+    sub_procs = []
     # Setup the queues.
     if queue_backend == QueueBackend.MULTIPROCESSING:
         logger.info('Creating shared request queues')
@@ -424,27 +426,26 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
         queue_server = multiprocessing.Process(
             target=mp_queue.start_queue_manager, args=(queue_names, port))
         queue_server.start()
-
+        sub_procs.append(queue_server)
         mp_queue.wait_for_queues_to_be_ready(queue_names, port=port)
 
     logger.info('Request queues created')
 
-    worker_procs = []
     for worker_id in range(max_parallel_for_long):
         worker = RequestWorker(id=worker_id,
                                schedule_type=api_requests.ScheduleType.LONG)
         worker_proc = multiprocessing.Process(target=request_worker,
                                               args=(worker, 1))
         worker_proc.start()
-        worker_procs.append(worker_proc)
+        sub_procs.append(worker_proc)
 
     # Start a worker for short requests.
     worker = RequestWorker(id=1, schedule_type=api_requests.ScheduleType.SHORT)
     worker_proc = multiprocessing.Process(target=request_worker,
                                           args=(worker, max_parallel_for_short))
     worker_proc.start()
-    worker_procs.append(worker_proc)
-    return worker_procs
+    sub_procs.append(worker_proc)
+    return sub_procs
 
 
 @annotations.lru_cache(scope='global', maxsize=1)

sky/server/requests/payloads.py

@@ -450,6 +450,7 @@ class LocalUpBody(RequestBody):
     ssh_user: Optional[str] = None
     ssh_key: Optional[str] = None
     cleanup: bool = False
+    context_name: Optional[str] = None
 
 
 class ServeTerminateReplicaBody(RequestBody):

sky/server/requests/serializers/decoders.py

@@ -172,12 +172,15 @@ def decode_storage_ls(
 
 @register_decoders('job_status')
 def decode_job_status(
-    return_value: Dict[int, Optional[str]]
+    return_value: Dict[str, Optional[str]]
 ) -> Dict[int, Optional['job_lib.JobStatus']]:
     job_statuses: Dict[int, Optional['job_lib.JobStatus']] = {}
-    for job_id in return_value.keys():
-        if return_value[job_id] is not None:
-            job_statuses[job_id] = job_lib.JobStatus(return_value[job_id])
+    for job_id_str, status_str in return_value.items():
+        # When we json serialize the job ID for storing in the requests db,
+        # the job_id gets converted to a string. Here we convert it back to int.
+        job_id = int(job_id_str)
+        if status_str is not None:
+            job_statuses[job_id] = job_lib.JobStatus(status_str)
         else:
             job_statuses[job_id] = None
     return job_statuses

sky/server/server.py

@@ -1081,14 +1081,14 @@ if __name__ == '__main__':
     # that it is shown only when the API server is started.
     usage_lib.maybe_show_privacy_policy()
 
-    num_workers = None
+    num_workers = 1
     if cmd_args.deploy:
-        num_workers = os.cpu_count()
+        num_workers = common_utils.get_cpu_count()
 
-    workers = []
+    sub_procs = []
     try:
-        workers = executor.start(cmd_args.deploy)
-        logger.info('Starting SkyPilot API server')
+        sub_procs = executor.start(cmd_args.deploy)
+        logger.info(f'Starting SkyPilot API server, workers={num_workers}')
         # We don't support reload for now, since it may cause leakage of request
         # workers or interrupt running requests.
         uvicorn.run('sky.server.server:app',
@@ -1101,5 +1101,6 @@ if __name__ == '__main__':
         raise
     finally:
         logger.info('Shutting down SkyPilot API server...')
-        for worker in workers:
-            worker.terminate()
+        for sub_proc in sub_procs:
+            sub_proc.terminate()
+            sub_proc.join()

sky/setup_files/dependencies.py

@@ -123,7 +123,7 @@ extras_require: Dict[str, List[str]] = {
         'ibm-cloud-sdk-core', 'ibm-vpc', 'ibm-platform-services', 'ibm-cos-sdk'
     ] + local_ray,
     'docker': ['docker'] + local_ray,
-    'lambda': local_ray,
+    'lambda': [],  # No dependencies needed for lambda
     'cloudflare': aws_dependencies,
     'scp': local_ray,
     'oci': ['oci'] + local_ray,

sky/skylet/constants.py

@@ -144,12 +144,14 @@ DISABLE_GPU_ECC_COMMAND = (
 CONDA_INSTALLATION_COMMANDS = (
     'which conda > /dev/null 2>&1 || '
     '{ '
-    'curl https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-Linux-x86_64.sh -o Miniconda3-Linux-x86_64.sh && '  # pylint: disable=line-too-long
+    # Use uname -m to get the architecture of the machine and download the
+    # corresponding Miniconda3-Linux.sh script.
+    'curl https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-Linux-$(uname -m).sh -o Miniconda3-Linux.sh && '  # pylint: disable=line-too-long
     # We do not use && for installation of conda and the following init commands
     # because for some images, conda is already installed, but not initialized.
     # In this case, we need to initialize conda and set auto_activate_base to
     # true.
-    '{ bash Miniconda3-Linux-x86_64.sh -b; '
+    '{ bash Miniconda3-Linux.sh -b; '
     'eval "$(~/miniconda3/bin/conda shell.bash hook)" && conda init && '
     # Caller should replace {conda_auto_activate} with either true or false.
     'conda config --set auto_activate_base {conda_auto_activate} && '

sky/utils/controller_utils.py

@@ -124,7 +124,7 @@ class Controllers(enum.Enum):
         cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
         in_progress_hint=(
             f'* To see detailed service status: {colorama.Style.BRIGHT}'
-            f'sky serve status -a{colorama.Style.RESET_ALL}'),
+            f'sky serve status -v{colorama.Style.RESET_ALL}'),
         decline_cancel_hint=(
             'Cancelling the sky serve controller\'s jobs is not allowed.'),
         _decline_down_when_failed_to_fetch_status_hint=(
@@ -246,11 +246,15 @@ def _get_cloud_dependencies_installation_commands(
                 'apt install curl socat netcat -y &> /dev/null; '
                 'fi" && '
                 # Install kubectl
+                'ARCH=$(uname -m) && '
+                'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                '  ARCH="arm64"; '
+                'else '
+                '  ARCH="amd64"; '
+                'fi && '
                 '(command -v kubectl &>/dev/null || '
-                '(curl -s -LO "https://dl.k8s.io/release/'
-                '$(curl -L -s https://dl.k8s.io/release/stable.txt)'
-                '/bin/linux/amd64/kubectl" && '
-                'sudo install -o root -g root -m 0755 '
+                '("https://dl.k8s.io/release/v1.31.6/bin/linux/$ARCH/kubectl" '
+                '&& sudo install -o root -g root -m 0755 '
                 'kubectl /usr/local/bin/kubectl))')
         elif isinstance(cloud, clouds.Cudo):
             step_prefix = prefix_str.replace('<step>', str(len(commands) + 1))

sky/utils/kubernetes/deploy_remote_cluster.sh

@@ -9,21 +9,38 @@ YELLOW='\033[1;33m'
 NC='\033[0m' # No color
 
 # Variables
+CLEANUP=false
+INSTALL_GPU=false
+POSITIONAL_ARGS=()
+
+# Process all arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --cleanup)
+            CLEANUP=true
+            shift
+            ;;
+        *)
+            POSITIONAL_ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+
+# Restore positional arguments in correct order
+set -- "${POSITIONAL_ARGS[@]}"
+
+# Assign positional arguments to variables
 IPS_FILE=$1
 USER=$2
 SSH_KEY=$3
+CONTEXT_NAME=${4:-default}
 K3S_TOKEN=mytoken  # Any string can be used as the token
-CLEANUP=false
-INSTALL_GPU=false
-
-if [[ "$4" == "--cleanup" ]]; then
-    CLEANUP=true
-fi
 
 # Basic argument checks
 if [ -z "$IPS_FILE" ] || [ -z "$USER" ] || [ -z "$SSH_KEY" ]; then
     >&2 echo -e "${RED}Error: Missing required arguments.${NC}"
-    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [--cleanup]"
+    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [context-name] [--cleanup]"
     exit 1
 fi
 
@@ -116,6 +133,17 @@ if [ "$CLEANUP" == "true" ]; then
         cleanup_agent_node "$NODE"
     done
 
+    # Remove the context from local kubeconfig if it exists
+    if [ -f "$HOME/.kube/config" ]; then
+        progress_message "Removing context '$CONTEXT_NAME' from local kubeconfig..."
+        kubectl config delete-context "$CONTEXT_NAME" 2>/dev/null || true
+        kubectl config delete-cluster "$CONTEXT_NAME" 2>/dev/null || true
+        kubectl config delete-user "$CONTEXT_NAME" 2>/dev/null || true
+        # Update the current context to the first available context
+        kubectl config use-context $(kubectl config view -o jsonpath='{.contexts[0].name}') 2>/dev/null || true
+        success_message "Context '$CONTEXT_NAME' removed from local kubeconfig."
+    fi
+
     echo -e "${GREEN}Cleanup completed successfully.${NC}"
     exit 0
 fi
@@ -167,22 +195,25 @@ for NODE in $WORKER_NODES; do
 done
 # Step 3: Configure local kubectl to connect to the cluster
 progress_message "Configuring local kubectl to connect to the cluster..."
-scp -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "$SSH_KEY" "$USER@$HEAD_NODE":~/.kube/config ~/.kube/config
 
-# Back up the original kubeconfig file if it exists
+# Create temporary directory for kubeconfig operations
+TEMP_DIR=$(mktemp -d)
+TEMP_KUBECONFIG="$TEMP_DIR/kubeconfig"
+
+# Get the kubeconfig from remote server
+scp -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "$SSH_KEY" "$USER@$HEAD_NODE":~/.kube/config "$TEMP_KUBECONFIG"
+
+# Create .kube directory if it doesn't exist
+mkdir -p "$HOME/.kube"
+
+# Create empty kubeconfig if it doesn't exist
 KUBECONFIG_FILE="$HOME/.kube/config"
-if [[ -f "$KUBECONFIG_FILE" ]]; then
-    echo "Backing up existing kubeconfig to $KUBECONFIG_FILE.bak"
-    cp "$KUBECONFIG_FILE" "$KUBECONFIG_FILE.bak"
+if [[ ! -f "$KUBECONFIG_FILE" ]]; then
+    touch "$KUBECONFIG_FILE"
 fi
 
-# Update kubeconfig for the local machine to use the master node's IP
-# Temporary file to hold the modified kubeconfig
-TEMP_FILE=$(mktemp)
-
-# Remove the certificate-authority-data, and replace the server with the master address
-awk '
-  BEGIN { in_cluster = 0 }
+# Modify the temporary kubeconfig to update server address and context name
+awk -v context="$CONTEXT_NAME" '
   /^clusters:/ { in_cluster = 1 }
   /^users:/ { in_cluster = 0 }
   in_cluster && /^ *certificate-authority-data:/ { next }
@@ -191,13 +222,24 @@ awk '
     print "    insecure-skip-tls-verify: true"
     next
   }
+  /name: default/ { sub("name: default", "name: " context) }
+  /cluster: default/ { sub("cluster: default", "cluster: " context) }
+  /user: default/ { sub("user: default", "user: " context) }
+  /current-context: default/ { sub("current-context: default", "current-context: " context) }
   { print }
-' "$KUBECONFIG_FILE" > "$TEMP_FILE"
+' "$TEMP_KUBECONFIG" > "$TEMP_DIR/modified_config"
+
+# Merge the configurations using kubectl
+KUBECONFIG="$KUBECONFIG_FILE:$TEMP_DIR/modified_config" kubectl config view --flatten > "$TEMP_DIR/merged_config"
+mv "$TEMP_DIR/merged_config" "$KUBECONFIG_FILE"
+
+# Set the new context as the current context
+kubectl config use-context "$CONTEXT_NAME"
 
-# Replace the original kubeconfig with the modified one
-mv "$TEMP_FILE" "$KUBECONFIG_FILE"
+# Clean up temporary files
+rm -rf "$TEMP_DIR"
 
-success_message "kubectl configured to connect to the cluster."
+success_message "kubectl configured with new context '$CONTEXT_NAME'."
 
 echo "Cluster deployment completed. You can now run 'kubectl get nodes' to verify the setup."
 

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -3,7 +3,7 @@ import os
 import shlex
 import subprocess
 import tempfile
-from typing import List
+from typing import List, Optional
 
 from sky import check as sky_check
 from sky import sky_logging
@@ -19,8 +19,11 @@ from sky.utils import ux_utils
 logger = sky_logging.init_logger(__name__)
 
 
-def deploy_remote_cluster(ip_list: List[str], ssh_user: str, ssh_key: str,
-                          cleanup: bool):
+def deploy_remote_cluster(ip_list: List[str],
+                          ssh_user: str,
+                          ssh_key: str,
+                          cleanup: bool,
+                          context_name: Optional[str] = None):
     success = False
     path_to_package = os.path.dirname(__file__)
     up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.sh')
@@ -41,6 +44,8 @@ def deploy_remote_cluster(ip_list: List[str], ssh_user: str, ssh_key: str,
 
         deploy_command = (f'{up_script_path} {ip_file.name} '
                           f'{ssh_user} {key_file.name}')
+        if context_name is not None:
+            deploy_command += f' {context_name}'
         if cleanup:
             deploy_command += ' --cleanup'
 
@@ -52,10 +57,6 @@ def deploy_remote_cluster(ip_list: List[str], ssh_user: str, ssh_key: str,
         log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
                                 'local_up.log')
 
-        # Check if ~/.kube/config exists:
-        if os.path.exists(os.path.expanduser('~/.kube/config')):
-            logger.info('Found existing kube config. '
-                        'It will be backed up to ~/.kube/config.bak.')
         if cleanup:
             msg_str = 'Cleaning up remote cluster...'
         else:

{skypilot_nightly-1.0.0.dev20250228.dist-info → skypilot_nightly-1.0.0.dev20250301.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250228
+Version: 1.0.0.dev20250301
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -75,7 +75,6 @@ Provides-Extra: docker
 Requires-Dist: docker; extra == "docker"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "docker"
 Provides-Extra: lambda
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "lambda"
 Provides-Extra: cloudflare
 Requires-Dist: urllib3<2; extra == "cloudflare"
 Requires-Dist: awscli>=1.27.10; extra == "cloudflare"
@@ -132,7 +131,6 @@ Requires-Dist: ibm-cos-sdk; extra == "all"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: docker; extra == "all"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: urllib3<2; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"

{skypilot_nightly-1.0.0.dev20250228.dist-info → skypilot_nightly-1.0.0.dev20250301.dist-info}/RECORD

@@ -1,12 +1,12 @@
-sky/__init__.py,sha256=jLsjhG2RasaqPtD_RYBd_LfTFzhUYIB8j6WkrjZVbKY,6428
+sky/__init__.py,sha256=zwz3HHeG9IfscR8U6_UAUwRDZFPP6FQWHQeMW5WTGhk,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=hCEqi77nprQEg3ktfRL51xiiw16zwZOmFEDB_Z7fWVU,22384
 sky/check.py,sha256=NDKx_Zm7YRxPjMv82wz3ESLnGIPljaACyqVdVNM0PzY,11258
-sky/cli.py,sha256=K28Bowflmjhir2e3hIPbZhsnfqvBsANSeBSnEXFoy10,219929
-sky/cloud_stores.py,sha256=-95XIqi_ouo7hvoN5mQNP6bGm07MyF6Yk-YP4Txb5wg,24034
-sky/core.py,sha256=X83hdpPTiWyEJLamrguCd03PUjkRiGgqTFfEBEQkzWc,45471
+sky/cli.py,sha256=-3dv9uu0jp3uRgZYsdI_1SQTE6o0CD1XBql6gMydgHE,220651
+sky/cloud_stores.py,sha256=kEHXd2divyra-1c3EusHxKyM5yTQlTXc6cKVXofsefA,23978
+sky/core.py,sha256=fDCit8n9BAXxJ9Dexq_CCOTmY8uZGK73wEaouNCRAbA,45573
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
-sky/exceptions.py,sha256=cGepNlBkjjgvi3fazc3DbdYLKhhF_sHCuGX0-hu_QMQ,13685
+sky/exceptions.py,sha256=RCAojf2r9sUOdFYEhOoB05G-VP0xwkEVfsYdh2TZypA,13783
 sky/execution.py,sha256=0M4RTEzWn-B9oz221XdZOIGH12XOACmNq0j-WGUT_No,28023
 sky/global_user_state.py,sha256=sUDdSsJeiJkbgmZNwy8YGFK0XeNh-RBr1VDUvbmjf0g,33246
 sky/models.py,sha256=4xSW05BdDPEjW8Ubvj3VlVOVnzv0TbrolsFvR5R5v1U,638
@@ -33,7 +33,7 @@ sky/adaptors/vast.py,sha256=tpvmHi7IkQNzbbHVkeo04kUSajoEpSzXr2XgeO_I1LU,695
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
-sky/backends/backend_utils.py,sha256=egi4xOu1NKZi8HBUTz8hhCOLx-QtEokcM_v6Ix6OUdA,133199
+sky/backends/backend_utils.py,sha256=B_46tG9PyrppxLWdg4mWGuuIr3TEcWTz6qhYXjAY2bw,133452
 sky/backends/cloud_vm_ray_backend.py,sha256=B8sH-m5pujvkTN_sLENsHw-SWpbaWwlHda26yI_1wnk,247459
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
@@ -43,9 +43,9 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=o4RymqSceq5mLEZL0upQM6NVEzJJQzj9s9tTm49uUTc,26365
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=K28Bowflmjhir2e3hIPbZhsnfqvBsANSeBSnEXFoy10,219929
+sky/client/cli.py,sha256=-3dv9uu0jp3uRgZYsdI_1SQTE6o0CD1XBql6gMydgHE,220651
 sky/client/common.py,sha256=axDic7WOG1e78SdFm5XIwdhX7YNvf3g4k7INrsW3X4s,14611
-sky/client/sdk.py,sha256=U4v8Khu1lf1oUoBuJUhIFnjsFhYM9x8XcKsnVRMtihI,66990
+sky/client/sdk.py,sha256=zNjZFDTecloxdkGqmSq9wrdKjX9pEMJLghu5bUThJWk,67406
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
 sky/clouds/aws.py,sha256=J8tczaTDL239UowN9tUlhI92SeHw01wtFucSckvG63w,54112
 sky/clouds/azure.py,sha256=bawEw6wOLAVyrjxMD-4UjLCuMj1H5_jH8qggpfZYS54,31703
@@ -53,7 +53,7 @@ sky/clouds/cloud.py,sha256=Ej6WH6VElYdG3PG1-Sp6lFVsJ42uskV4dAg7kmoY4JA,35376
 sky/clouds/cudo.py,sha256=femv17IUM1TOXuCAg6zljqyFcBGfofbXCNGckpXFHzc,13127
 sky/clouds/do.py,sha256=hmksx0XML0dVHUZBMV2Wr3a5VilOsYfxX2dSBV_XK5o,11487
 sky/clouds/fluidstack.py,sha256=Eb0nlfU_EwTtGtV0nPKS2ueBlB0nYiDAN9swA-jjQV0,12446
-sky/clouds/gcp.py,sha256=0QpsI0Dso1xs3LhGlUq-Sq6WK-u11wN-57-vfcyhI5I,55154
+sky/clouds/gcp.py,sha256=FUCUq94yGUZ_yyKxA3prRKTqetObbIMkfjAPTPbhXyA,55824
 sky/clouds/ibm.py,sha256=R4JR96YfXstZ2B_IgFNVEX2SBAq3q0lSWz4y7FoFoeE,21474
 sky/clouds/kubernetes.py,sha256=7ki_zJZKnkhOPrHgVFq6azy5UhNKeeBOCSTjKCgj3vk,31709
 sky/clouds/lambda_cloud.py,sha256=ejqA_Wj5-325Y_QjQ__FY4HMO8sv_2tSRsufmaldcmI,12699
@@ -103,7 +103,7 @@ sky/clouds/utils/scp_utils.py,sha256=r4lhRLtNgoz5nmkfN2ctAXYugF_-Et8TYH6ZlbbFfo8
 sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=wixC4_3_JaeJFdGKOp-O5ulcsMugDSgrCR0SnPpugGc,8946
 sky/data/data_utils.py,sha256=HjcgMDuWRR_fNQ9gjuROi9GgPVvTGApiJwxGtdb2_UU,28860
-sky/data/mounting_utils.py,sha256=la21kp7k51zGoFp9WxT5hf38P_XTqcq-Hm1bJZsPnkg,14949
+sky/data/mounting_utils.py,sha256=i79Y-DZXVR88fjG_MBPB8EgsZBnHdpf1LGnJSm_VhAg,16063
 sky/data/storage.py,sha256=mTgMGdfSV6Gia076Dvgmc18ZlqF6eObima558UShiXA,207165
 sky/data/storage_utils.py,sha256=zB99nRTJjh8isU0UmqERmlwwRNgfig91IwrwVH8CcNw,12383
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
@@ -228,29 +228,29 @@ sky/serve/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 sky/serve/server/core.py,sha256=pRvFadEIH_WTUkTtSmuFoPBP4JFq8Obt68ifi9DWuog,36865
 sky/serve/server/server.py,sha256=gQGVU9nHYdGbaLhGjIUNIYn4xwKjRASRJkiiTL5AI1Y,3283
 sky/server/__init__.py,sha256=MPPBqFzXz6Jv5QSk6td_IcvnfXfNErDZVcizu4MLRow,27
-sky/server/common.py,sha256=0LphKrp89_sGI-xDakK2uEqI-zKuvbc4OTcuLCiKfmQ,17560
+sky/server/common.py,sha256=pEa-q3P5aOm6RMlit0pVzlDoJnZU_6zViO7aK_7htn0,17843
 sky/server/constants.py,sha256=89jKE3SIe1T3_7j6ECTy4pZnhZZD7fBwsWOCOkTban8,770
-sky/server/server.py,sha256=4ipJG67sBFWylNYdPD1FUhth36yX23XbcROXipRSZsw,42438
+sky/server/server.py,sha256=IDVx6qHl75PRSyc23NLMi0JLjqrTjOHWiCeNvK1D60U,42511
 sky/server/stream_utils.py,sha256=-3IX1YCgxAFfcvQIV0TCvOn1wbRLWovAx3ckCrsExWU,5651
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/server/requests/executor.py,sha256=TWX2jYkkstgRyRkWNE19Mgw4_CfzadebOW30iTGdK_Q,19693
-sky/server/requests/payloads.py,sha256=QYgEz85jswXkEYxO1mkwPA8MWXD_pETs-g_JH_Tlm_w,16038
+sky/server/requests/executor.py,sha256=Jk8RJoQlicDqaHhgVWMH3UiL-dJS7lGSGd05GPv-Lrc,19781
+sky/server/requests/payloads.py,sha256=75bHfIJ5TR8z3BVVKMYLKxK5BzjtxwawCpwN8B88-FA,16077
 sky/server/requests/requests.py,sha256=aMdjiK5kjSYP36pxdXFU6qgKOXcOmtViHbFm3V8Dvf8,19590
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/queues/mp_queue.py,sha256=_7AFas__0b1L8e7Bwy4lu0VYU18R85YwMlDHPhQCfh0,2998
 sky/server/requests/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/server/requests/serializers/decoders.py,sha256=iChUnvvXwnAArWm0AVT5eTLt5ZmPfkr4TT32orCSJCM,6624
+sky/server/requests/serializers/decoders.py,sha256=0cpg80uAqkdK_LqcQPkpKswhcNUUztG9luDLm_0eUow,6811
 sky/server/requests/serializers/encoders.py,sha256=i4SAb5Oyp00CyMkyidbdA9dtxAzxZl40KTpL_x6pH0w,5679
 sky/setup_files/MANIFEST.in,sha256=cHYG6IdIp7RsDapL4Lrs-WTeYJftHn6qystSolmyyk8,581
-sky/setup_files/dependencies.py,sha256=CbuSc7D1YFlonN-TF8qI0khq3jE0K7ueUEPG10RUNIY,6283
+sky/setup_files/dependencies.py,sha256=cm2qeYFeVDv5tHlcSw0xtZzTI6uz4m_WaF2GUIFIgCk,6313
 sky/setup_files/setup.py,sha256=Q9f0RvsdPC0FLvyTKW-upQtRuA81jRO4TtN3VK-mP-Y,7436
 sky/skylet/LICENSE,sha256=BnFrJSvUFpMUoH5mOpWnEvaC5R6Uux8W6WXgrte8iYg,12381
 sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=W4CtMira6QnmYToFT5kYTGjNPRZNC-bZPfsF1k3tluE,4480
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=WB7bKWvMkgyEf9tAzfKwtM8eYR9OEgJtqwgBT3BEoEM,17923
+sky/skylet/constants.py,sha256=wELTCOQLIafwtl7_L8vjDwdqYQQxG-Sup3H7vxmcyB4,18037
 sky/skylet/events.py,sha256=pnV3ZiwWhXqTHpU5B5Y9Xwam_7FQDI6IrxgSx7X_NVA,12743
 sky/skylet/job_lib.py,sha256=8W6GM2zxqGMIoD3AGiXcKsK_7-qouuTojiVL6upSeoA,43728
 sky/skylet/log_lib.py,sha256=DzOrgY8C7RdEMLC9O9kEKV-iLMb9wVMPSnDha8eMx28,20900
@@ -313,7 +313,7 @@ sky/utils/common.py,sha256=P4oVXFATUYgkruHX92cN12SJBtfb8DiOOYZtbN1kvP0,1927
 sky/utils/common_utils.py,sha256=1KfqF_hgH9l1ieyV-_3fJd6ma41-tOstj-ihAQcEDIM,31162
 sky/utils/config_utils.py,sha256=VQ2E3DQ2XysD-kul-diSrxn_pXWsDMfKAev91OiJQ1Q,9041
 sky/utils/control_master_utils.py,sha256=iD4M0onjYOdZ2RuxjwMBl4KhafHXJzuHjvqlBUnu-VE,1450
-sky/utils/controller_utils.py,sha256=Wth_esy2NX9nco-MK01bgQMIChAYky0Uq4T35jQkXxY,48472
+sky/utils/controller_utils.py,sha256=FETdTx1om3Qmxw5W9BNazhV0dWoNdMWYopIEQJRcKSA,48614
 sky/utils/dag_utils.py,sha256=sAus0aL1wtuuFZSDnpO4LY-6WK4u5iJY952oWQzHo3Y,7532
 sky/utils/db_utils.py,sha256=K2-OHPg0FeHCarevMdWe0IWzm6wWumViEeYeJuGoFUE,3747
 sky/utils/env_options.py,sha256=aaD6GoYK0LaZIqjOEZ-R7eccQuiRriW3EuLWtOI5En8,1578
@@ -334,19 +334,19 @@ sky/utils/cli_utils/status_utils.py,sha256=LwGXzMgvnQeGR1fCC24q38hRLuAPeeSDkQ387
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=VLXfazav9XCMQmeKVqhuOQzt2vM6G1jgnvvb0SHUFno,7773
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
-sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=7FN1Y10fE24GntL_sbQU7z_103rKvXrneru0rjuDhkU,8595
+sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=SGnqa5ks5Og-F96S_PIUbKzuKAtTF5cYOrXIg5XWx_E,10179
 sky/utils/kubernetes/exec_kubeconfig_converter.py,sha256=fE1SnteoxI05EaugnWeV82hXwZTVHmbXsh1aaZAgF3c,2548
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=MBvXJio0PeujZSCXiRKE_pa6HCTiU9qBzR1WrXccVSY,10477
 sky/utils/kubernetes/gpu_labeler.py,sha256=4px7FyfsukacPEvKwTLUNb3WwacMIUrHWjP93qTi3kE,6998
 sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
-sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=iAjfyPclOs8qlALACcfxLpRAO9CZ-h16leFqXZ6tNaY,10096
+sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=otzHzpliHDCpzYT-nU9Q0ZExbiFpDPWvhxwkvchZj7k,10073
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250228.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250228.dist-info/METADATA,sha256=IBTEc-5NtZ9KP63RfsWsMmXI6qzs7ZBSdi9bAN8QfAc,19236
-skypilot_nightly-1.0.0.dev20250228.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-skypilot_nightly-1.0.0.dev20250228.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250228.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250228.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250301.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250301.dist-info/METADATA,sha256=ImCAjTZcD8ZiM3E8or9csIop8EZDekZHswvPqBMLlAo,19115
+skypilot_nightly-1.0.0.dev20250301.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+skypilot_nightly-1.0.0.dev20250301.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250301.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250301.dist-info/RECORD,,