skypilot-nightly 1.0.0.dev20250227__tar.gz → 1.0.0.dev20250301__tar.gz

{skypilot_nightly-1.0.0.dev20250227/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250301}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250227
+Version: 1.0.0.dev20250301
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -75,7 +75,6 @@ Provides-Extra: docker
 Requires-Dist: docker; extra == "docker"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "docker"
 Provides-Extra: lambda
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "lambda"
 Provides-Extra: cloudflare
 Requires-Dist: urllib3<2; extra == "cloudflare"
 Requires-Dist: awscli>=1.27.10; extra == "cloudflare"
@@ -132,7 +131,6 @@ Requires-Dist: ibm-cos-sdk; extra == "all"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: docker; extra == "all"
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: urllib3<2; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '2c4849b6f73499740f495f84a29ac4af98d25073'
+_SKYPILOT_COMMIT_SHA = 'cefc23846d47d7dacf960093c2001912a83c1162'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250227'
+__version__ = '1.0.0.dev20250301'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/backends/backend_utils.py

@@ -1280,12 +1280,17 @@ def parallel_data_transfer_to_nodes(
                                             stream_logs=stream_logs,
                                             require_outputs=True,
                                             source_bashrc=source_bashrc)
-            err_msg = ('Failed to run command before rsync '
+            err_msg = (f'{colorama.Style.RESET_ALL}{colorama.Style.DIM}'
+                       f'----- CMD -----\n'
+                       f'{cmd}\n'
+                       f'----- CMD END -----\n'
+                       f'{colorama.Style.RESET_ALL}'
+                       f'{colorama.Fore.RED}'
+                       f'Failed to run command before rsync '
                        f'{origin_source} -> {target}. '
-                       'Ensure that the network is stable, then retry. '
-                       f'{cmd}')
+                       f'{colorama.Style.RESET_ALL}')
             if log_path != os.devnull:
-                err_msg += f' See logs in {log_path}'
+                err_msg += ux_utils.log_path_hint(log_path)
             subprocess_utils.handle_returncode(rc,
                                                cmd,
                                                err_msg,

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/cli.py

@@ -133,7 +133,11 @@ def _get_cluster_records_and_set_ssh_config(
     # Update the SSH config for all clusters
     for record in cluster_records:
         handle = record['handle']
-        if handle is not None and handle.cached_external_ips is not None:
+        # During the failover, even though a cluster does not exist, the handle
+        # can still exist in the record, and we check for credentials to avoid
+        # updating the SSH config for non-existent clusters.
+        if (handle is not None and handle.cached_external_ips is not None and
+                'credentials' in record):
             credentials = record['credentials']
             if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
                 # Replace the proxy command to proxy through the SkyPilot API
@@ -169,9 +173,9 @@ def _get_cluster_records_and_set_ssh_config(
                 handle.ssh_user,
             )
         else:
-            # If the cluster is not UP or does not have IPs, we need to remove
-            # the cluster from the SSH config.
-            cluster_utils.SSHConfigHelper.remove_cluster(handle.cluster_name)
+            # If the cluster is not UP or does not have credentials available,
+            # we need to remove the cluster from the SSH config.
+            cluster_utils.SSHConfigHelper.remove_cluster(record['name'])
 
     # Clean up SSH configs for clusters that do not exist.
     #
@@ -5415,11 +5419,16 @@ def local():
 @click.option('--cleanup',
               is_flag=True,
               help='Clean up the remote cluster instead of deploying it.')
+@click.option(
+    '--context-name',
+    type=str,
+    required=False,
+    help='Name to use for the kubeconfig context. Defaults to "default".')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, async_call: bool):
+             cleanup: bool, context_name: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5464,7 +5473,8 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
             raise click.BadParameter(
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
-    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup)
+    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
+                              context_name)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 
@@ -5500,10 +5510,19 @@ def api():
               required=False,
               help=('The host to deploy the SkyPilot API server. To allow '
                     'remote access, set this to 0.0.0.0'))
+@click.option('--foreground',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Run the SkyPilot API server in the foreground and output '
+              'its logs to stdout/stderr. Allowing external systems '
+              'to manage the process lifecycle and collect logs directly. '
+              'This is useful when the API server is managed by systems '
+              'like systemd and Kubernetes.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str]):
+def api_start(deploy: bool, host: Optional[str], foreground: bool):
     """Starts the SkyPilot API server locally."""
-    sdk.api_start(deploy=deploy, host=host)
+    sdk.api_start(deploy=deploy, host=host, foreground=foreground)
 
 
 @api.command('stop', cls=_DocumentedCodeCommand)

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/client/cli.py

@@ -133,7 +133,11 @@ def _get_cluster_records_and_set_ssh_config(
     # Update the SSH config for all clusters
     for record in cluster_records:
         handle = record['handle']
-        if handle is not None and handle.cached_external_ips is not None:
+        # During the failover, even though a cluster does not exist, the handle
+        # can still exist in the record, and we check for credentials to avoid
+        # updating the SSH config for non-existent clusters.
+        if (handle is not None and handle.cached_external_ips is not None and
+                'credentials' in record):
             credentials = record['credentials']
             if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
                 # Replace the proxy command to proxy through the SkyPilot API
@@ -169,9 +173,9 @@ def _get_cluster_records_and_set_ssh_config(
                 handle.ssh_user,
             )
         else:
-            # If the cluster is not UP or does not have IPs, we need to remove
-            # the cluster from the SSH config.
-            cluster_utils.SSHConfigHelper.remove_cluster(handle.cluster_name)
+            # If the cluster is not UP or does not have credentials available,
+            # we need to remove the cluster from the SSH config.
+            cluster_utils.SSHConfigHelper.remove_cluster(record['name'])
 
     # Clean up SSH configs for clusters that do not exist.
     #
@@ -5415,11 +5419,16 @@ def local():
 @click.option('--cleanup',
               is_flag=True,
               help='Clean up the remote cluster instead of deploying it.')
+@click.option(
+    '--context-name',
+    type=str,
+    required=False,
+    help='Name to use for the kubeconfig context. Defaults to "default".')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, async_call: bool):
+             cleanup: bool, context_name: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5464,7 +5473,8 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
             raise click.BadParameter(
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
-    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup)
+    request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
+                              context_name)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 
@@ -5500,10 +5510,19 @@ def api():
               required=False,
               help=('The host to deploy the SkyPilot API server. To allow '
                     'remote access, set this to 0.0.0.0'))
+@click.option('--foreground',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Run the SkyPilot API server in the foreground and output '
+              'its logs to stdout/stderr. Allowing external systems '
+              'to manage the process lifecycle and collect logs directly. '
+              'This is useful when the API server is managed by systems '
+              'like systemd and Kubernetes.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str]):
+def api_start(deploy: bool, host: Optional[str], foreground: bool):
     """Starts the SkyPilot API server locally."""
-    sdk.api_start(deploy=deploy, host=host)
+    sdk.api_start(deploy=deploy, host=host, foreground=foreground)
 
 
 @api.command('stop', cls=_DocumentedCodeCommand)

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/client/sdk.py

@@ -1263,8 +1263,12 @@ def storage_delete(name: str) -> server_common.RequestId:
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
-             ssh_key: Optional[str], cleanup: bool) -> server_common.RequestId:
+def local_up(gpus: bool,
+             ips: Optional[List[str]],
+             ssh_user: Optional[str],
+             ssh_key: Optional[str],
+             cleanup: bool,
+             context_name: Optional[str] = None) -> server_common.RequestId:
     """Launches a Kubernetes cluster on local machines.
 
     Returns:
@@ -1282,7 +1286,8 @@ def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
                                 ips=ips,
                                 ssh_user=ssh_user,
                                 ssh_key=ssh_key,
-                                cleanup=cleanup)
+                                cleanup=cleanup,
+                                context_name=context_name)
     response = requests.post(f'{server_common.get_server_url()}/local_up',
                              json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)
@@ -1611,6 +1616,7 @@ def api_start(
     *,
     deploy: bool = False,
     host: str = '127.0.0.1',
+    foreground: bool = False,
 ) -> None:
     """Starts the API server.
 
@@ -1622,7 +1628,8 @@ def api_start(
             resources of the machine.
         host: The host to deploy the API server. It will be set to 0.0.0.0
             if deploy is True, to allow remote access.
-
+        foreground: Whether to run the API server in the foreground (run in
+            the current process).
     Returns:
         None
     """
@@ -1641,7 +1648,10 @@ def api_start(
                              'from the config file and/or unset the '
                              'SKYPILOT_API_SERVER_ENDPOINT environment '
                              'variable.')
-    server_common.check_server_healthy_or_start_fn(deploy, host)
+    server_common.check_server_healthy_or_start_fn(deploy, host, foreground)
+    if foreground:
+        # Explain why current process exited
+        logger.info('API server is already running:')
     logger.info(f'{ux_utils.INDENT_SYMBOL}SkyPilot API server: '
                 f'{server_common.get_server_url(host)}\n'
                 f'{ux_utils.INDENT_LAST_SYMBOL}'

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/cloud_stores.py

@@ -54,8 +54,9 @@ class S3CloudStorage(CloudStorage):
 
     # List of commands to install AWS CLI
     _GET_AWSCLI = [
-        'aws --version >/dev/null 2>&1 || '
-        f'{constants.SKY_UV_PIP_CMD} install awscli',
+        'awscli_path=$(which aws) || '
+        f'{{ {constants.SKY_UV_PIP_CMD} install awscli && '
+        f'awscli_path={constants.SKY_REMOTE_PYTHON_ENV}/bin/aws; }}',
     ]
 
     def is_directory(self, url: str) -> bool:
@@ -85,8 +86,7 @@ class S3CloudStorage(CloudStorage):
         # AWS Sync by default uses 10 threads to upload files to the bucket.
         # To increase parallelism, modify max_concurrent_requests in your
         # aws config file (Default path: ~/.aws/config).
-        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
-                               'sync --no-follow-symlinks '
+        download_via_awscli = (f'$awscli_path s3 sync --no-follow-symlinks '
                                f'{source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
@@ -95,8 +95,7 @@ class S3CloudStorage(CloudStorage):
 
     def make_sync_file_command(self, source: str, destination: str) -> str:
         """Downloads a file using AWS CLI."""
-        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
-                               f'cp {source} {destination}')
+        download_via_awscli = (f'$awscli_path s3 cp {source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
         all_commands.append(download_via_awscli)

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/clouds/gcp.py

@@ -67,8 +67,20 @@ _GCLOUD_VERSION = '424.0.0'
 GOOGLE_SDK_INSTALLATION_COMMAND: str = f'pushd /tmp &>/dev/null && \
     {{ gcloud --help > /dev/null 2>&1 || \
     {{ mkdir -p {os.path.dirname(_GCLOUD_INSTALLATION_LOG)} && \
-    wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-{_GCLOUD_VERSION}-linux-x86_64.tar.gz > {_GCLOUD_INSTALLATION_LOG} && \
-    tar xzf google-cloud-sdk-{_GCLOUD_VERSION}-linux-x86_64.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
+    ARCH=$(uname -m) && \
+    if [ "$ARCH" = "x86_64" ]; then \
+        echo "Installing Google Cloud SDK for $ARCH" > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="x86_64"; \
+    elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
+        echo "Installing Google Cloud SDK for $ARCH" > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="arm"; \
+    else \
+        echo "Architecture $ARCH not supported by Google Cloud SDK. Defaulting to x86_64." > {_GCLOUD_INSTALLATION_LOG} && \
+        ARCH_SUFFIX="x86_64"; \
+    fi && \
+    echo "Detected architecture: $ARCH, using package: $ARCH_SUFFIX" >> {_GCLOUD_INSTALLATION_LOG} && \
+    wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-{_GCLOUD_VERSION}-linux-${{ARCH_SUFFIX}}.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
+    tar xzf google-cloud-sdk-{_GCLOUD_VERSION}-linux-${{ARCH_SUFFIX}}.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
     rm -rf ~/google-cloud-sdk >> {_GCLOUD_INSTALLATION_LOG}  && \
     mv google-cloud-sdk ~/ && \
     ~/google-cloud-sdk/install.sh -q >> {_GCLOUD_INSTALLATION_LOG} 2>&1 && \

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/core.py

@@ -1007,8 +1007,12 @@ def realtime_kubernetes_gpu_availability(
 # = Local Cluster =
 # =================
 @usage_lib.entrypoint
-def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
-             ssh_key: Optional[str], cleanup: bool) -> None:
+def local_up(gpus: bool,
+             ips: Optional[List[str]],
+             ssh_user: Optional[str],
+             ssh_key: Optional[str],
+             cleanup: bool,
+             context_name: Optional[str] = None) -> None:
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key, cleanup):
@@ -1034,7 +1038,7 @@ def local_up(gpus: bool, ips: Optional[List[str]], ssh_user: Optional[str],
     if ips:
         assert ssh_user is not None and ssh_key is not None
         kubernetes_deploy_utils.deploy_remote_cluster(ips, ssh_user, ssh_key,
-                                                      cleanup)
+                                                      cleanup, context_name)
     else:
         # Run local deployment (kind) if no remote args are specified
         kubernetes_deploy_utils.deploy_local_cluster(gpus)

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/data/mounting_utils.py

@@ -24,7 +24,14 @@ RCLONE_VERSION = 'v1.68.2'
 
 def get_s3_mount_install_cmd() -> str:
     """Returns a command to install S3 mount utility goofys."""
-    install_cmd = ('sudo wget -nc https://github.com/romilbhardwaj/goofys/'
+    install_cmd = ('ARCH=$(uname -m) && '
+                   'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                   '  echo "goofys is not supported on $ARCH" && '
+                   f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+                   'else '
+                   '  ARCH_SUFFIX="amd64"; '
+                   'fi && '
+                   'sudo wget -nc https://github.com/romilbhardwaj/goofys/'
                    'releases/download/0.24.0-romilb-upstream/goofys '
                    '-O /usr/local/bin/goofys && '
                    'sudo chmod 755 /usr/local/bin/goofys')
@@ -49,9 +56,15 @@ def get_s3_mount_cmd(bucket_name: str,
 
 def get_gcs_mount_install_cmd() -> str:
     """Returns a command to install GCS mount utility gcsfuse."""
-    install_cmd = ('wget -nc https://github.com/GoogleCloudPlatform/gcsfuse'
+    install_cmd = ('ARCH=$(uname -m) && '
+                   'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                   '  ARCH_SUFFIX="arm64"; '
+                   'else '
+                   '  ARCH_SUFFIX="amd64"; '
+                   'fi && '
+                   'wget -nc https://github.com/GoogleCloudPlatform/gcsfuse'
                    f'/releases/download/v{GCSFUSE_VERSION}/'
-                   f'gcsfuse_{GCSFUSE_VERSION}_amd64.deb '
+                   f'gcsfuse_{GCSFUSE_VERSION}_${{ARCH_SUFFIX}}.deb '
                    '-O /tmp/gcsfuse.deb && '
                    'sudo dpkg --install /tmp/gcsfuse.deb')
     return install_cmd
@@ -77,16 +90,24 @@ def get_gcs_mount_cmd(bucket_name: str,
 
 def get_az_mount_install_cmd() -> str:
     """Returns a command to install AZ Container mount utility blobfuse2."""
-    install_cmd = ('sudo apt-get update; '
-                   'sudo apt-get install -y '
-                   '-o Dpkg::Options::="--force-confdef" '
-                   'fuse3 libfuse3-dev && '
-                   'wget -nc https://github.com/Azure/azure-storage-fuse'
-                   f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
-                   f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
-                   '-O /tmp/blobfuse2.deb && '
-                   'sudo dpkg --install /tmp/blobfuse2.deb && '
-                   f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
+    install_cmd = (
+        'sudo apt-get update; '
+        'sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        'fuse3 libfuse3-dev && '
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  echo "blobfuse2 is not supported on $ARCH" && '
+        f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'else '
+        '  ARCH_SUFFIX="x86_64"; '
+        'fi && '
+        'wget -nc https://github.com/Azure/azure-storage-fuse'
+        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
+        f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
+        '-O /tmp/blobfuse2.deb && '
+        'sudo dpkg --install /tmp/blobfuse2.deb && '
+        f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
 
     return install_cmd
 
@@ -207,14 +228,20 @@ def get_rclone_install_cmd() -> str:
     """
     # pylint: disable=line-too-long
     install_cmd = (
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  ARCH_SUFFIX="arm"; '
+        'else '
+        '  ARCH_SUFFIX="amd64"; '
+        'fi && '
         f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-amd64.deb'
-        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-amd64.deb'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-amd64.deb)))'
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
         f' || (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-amd64.rpm'
-        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-amd64.rpm -y'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-amd64.rpm))')
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
+        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm))')
     return install_cmd
 
 

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/exceptions.py

@@ -22,6 +22,8 @@ MOUNT_PATH_NON_EMPTY_CODE = 42
 INSUFFICIENT_PRIVILEGES_CODE = 52
 # Return code when git command is ran in a dir that is not git repo
 GIT_FATAL_EXIT_CODE = 128
+# Architecture, such as arm64, not supported by the dependency
+ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 
 
 def is_safe_exception(exc: Exception) -> bool:

{skypilot_nightly-1.0.0.dev20250227 → skypilot_nightly-1.0.0.dev20250301}/sky/server/common.py

@@ -145,57 +145,6 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     return ApiServerInfo(status=ApiServerStatus.UNHEALTHY, api_version=None)
 
 
-def start_api_server_in_background(deploy: bool = False,
-                                   host: str = '127.0.0.1'):
-    if not is_api_server_local():
-        raise RuntimeError(
-            f'Cannot start API server: {get_server_url()} is not a local URL')
-
-    # Check available memory before starting the server.
-    avail_mem_size_gb: float = common_utils.get_mem_size_gb()
-    if avail_mem_size_gb <= server_constants.MIN_AVAIL_MEM_GB:
-        logger.warning(
-            f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only has '
-            f'{avail_mem_size_gb:.1f}GB memory available. '
-            f'At least {server_constants.MIN_AVAIL_MEM_GB}GB is recommended to '
-            f'support higher load with better performance.'
-            f'{colorama.Style.RESET_ALL}')
-    log_path = os.path.expanduser(constants.API_SERVER_LOGS)
-    os.makedirs(os.path.dirname(log_path), exist_ok=True)
-
-    api_server_cmd = API_SERVER_CMD
-    if deploy:
-        api_server_cmd += ' --deploy'
-    if host is not None:
-        api_server_cmd += f' --host {host}'
-    cmd = f'{sys.executable} {api_server_cmd} > {log_path} 2>&1 < /dev/null'
-
-    # Start the API server process in the background and don't wait for it.
-    # If this is called from a CLI invocation, we need start_new_session=True so
-    # that SIGINT on the CLI will not also kill the API server.
-    subprocess.Popen(cmd, shell=True, start_new_session=True)
-
-    # Wait for the server to start until timeout.
-    # Conservative upper time bound for starting the server based on profiling.
-    timeout_sec = 12
-    start_time = time.time()
-    while True:
-        api_server_info = get_api_server_status()
-        assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
-            f'API server version mismatch when starting the server. '
-            f'Server version: {api_server_info.api_version} '
-            f'Client version: {server_constants.API_VERSION}')
-        if api_server_info.status == ApiServerStatus.HEALTHY:
-            break
-        elif time.time() - start_time >= timeout_sec:
-            with ux_utils.print_exception_no_traceback():
-                raise RuntimeError(
-                    'Failed to start SkyPilot API server at '
-                    f'{get_server_url(host)}'
-                    f'\nView logs at: {constants.API_SERVER_LOGS}')
-        time.sleep(0.5)
-
-
 def handle_request_error(response: requests.Response) -> None:
     if response.status_code != 200:
         with ux_utils.print_exception_no_traceback():
@@ -218,10 +167,10 @@ def get_request_id(response: requests.Response) -> RequestId:
     return request_id
 
 
-def _start_api_server(deploy: bool = False, host: str = '127.0.0.1'):
+def _start_api_server(deploy: bool = False,
+                      host: str = '127.0.0.1',
+                      foreground: bool = False):
     """Starts a SkyPilot API server locally."""
-    # Lock to prevent multiple processes from starting the server at the
-    # same time, causing issues with database initialization.
     server_url = get_server_url(host)
     assert server_url in AVAILABLE_LOCAL_API_SERVER_URLS, (
         f'server url {server_url} is not a local url')
@@ -230,7 +179,60 @@ def _start_api_server(deploy: bool = False, host: str = '127.0.0.1'):
                     f'SkyPilot API server at {server_url}. '
                     'Starting a local server.'
                     f'{colorama.Style.RESET_ALL}')
-        start_api_server_in_background(deploy=deploy, host=host)
+        if not is_api_server_local():
+            raise RuntimeError(f'Cannot start API server: {get_server_url()} '
+                               'is not a local URL')
+
+        # Check available memory before starting the server.
+        avail_mem_size_gb: float = common_utils.get_mem_size_gb()
+        if avail_mem_size_gb <= server_constants.MIN_AVAIL_MEM_GB:
+            logger.warning(
+                f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only '
+                f'has {avail_mem_size_gb:.1f}GB memory available. '
+                f'At least {server_constants.MIN_AVAIL_MEM_GB}GB is '
+                'recommended to support higher load with better performance.'
+                f'{colorama.Style.RESET_ALL}')
+
+        args = [sys.executable, *API_SERVER_CMD.split()]
+        if deploy:
+            args += ['--deploy']
+        if host is not None:
+            args += [f'--host={host}']
+
+        if foreground:
+            # Replaces the current process with the API server
+            os.execvp(args[0], args)
+
+        log_path = os.path.expanduser(constants.API_SERVER_LOGS)
+        os.makedirs(os.path.dirname(log_path), exist_ok=True)
+        cmd = f'{" ".join(args)} > {log_path} 2>&1 < /dev/null'
+
+        # Start the API server process in the background and don't wait for it.
+        # If this is called from a CLI invocation, we need
+        # start_new_session=True so that SIGINT on the CLI will not also kill
+        # the API server.
+        subprocess.Popen(cmd, shell=True, start_new_session=True)
+
+        # Wait for the server to start until timeout.
+        # Conservative upper time bound for starting the server based on
+        # profiling.
+        timeout_sec = 12
+        start_time = time.time()
+        while True:
+            api_server_info = get_api_server_status()
+            assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
+                f'API server version mismatch when starting the server. '
+                f'Server version: {api_server_info.api_version} '
+                f'Client version: {server_constants.API_VERSION}')
+            if api_server_info.status == ApiServerStatus.HEALTHY:
+                break
+            elif time.time() - start_time >= timeout_sec:
+                with ux_utils.print_exception_no_traceback():
+                    raise RuntimeError(
+                        'Failed to start SkyPilot API server at '
+                        f'{get_server_url(host)}'
+                        f'\nView logs at: {constants.API_SERVER_LOGS}')
+            time.sleep(0.5)
         logger.info(ux_utils.finishing_message('SkyPilot API server started.'))
 
 
@@ -260,7 +262,8 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
 
 
 def check_server_healthy_or_start_fn(deploy: bool = False,
-                                     host: str = '127.0.0.1'):
+                                     host: str = '127.0.0.1',
+                                     foreground: bool = False):
     try:
         check_server_healthy()
     except exceptions.ApiServerConnectionError as exc:
@@ -268,13 +271,15 @@ def check_server_healthy_or_start_fn(deploy: bool = False,
         if not is_api_server_local():
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.ApiServerConnectionError(endpoint) from exc
+        # Lock to prevent multiple processes from starting the server at the
+        # same time, causing issues with database initialization.
         with filelock.FileLock(
                 os.path.expanduser(constants.API_SERVER_CREATION_LOCK_PATH)):
             # Check again if server is already running. Other processes may
             # have started the server while we were waiting for the lock.
             api_server_info = get_api_server_status(endpoint)
             if api_server_info.status == ApiServerStatus.UNHEALTHY:
-                _start_api_server(deploy, host)
+                _start_api_server(deploy, host, foreground)
 
 
 def check_server_healthy_or_start(func):