skypilot-nightly 1.0.0.dev20250411__py3-none-any.whl → 1.0.0.dev20250413__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'b4202948016cdf48a5939ea9bf1769a2d31f73bf'
+_SKYPILOT_COMMIT_SHA = '197c8dd3ea85d23323477e7d7cf69e8dc1b693c6'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250411'
+__version__ = '1.0.0.dev20250413'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/oci.py

@@ -13,7 +13,7 @@ from sky.clouds.utils import oci_utils
 # effect.
 logging.getLogger('oci.circuit_breaker').setLevel(logging.WARNING)
 
-CONFIG_PATH = '~/.oci/config'
+OCI_CONFIG_PATH = '~/.oci/config'
 ENV_VAR_OCI_CONFIG = 'OCI_CONFIG'
 
 oci = common.LazyImport(
@@ -23,7 +23,7 @@ oci = common.LazyImport(
 
 
 def get_config_file() -> str:
-    conf_file_path = CONFIG_PATH
+    conf_file_path = OCI_CONFIG_PATH
     config_path_via_env_var = os.environ.get(ENV_VAR_OCI_CONFIG)
     if config_path_via_env_var is not None:
         conf_file_path = config_path_via_env_var

sky/authentication.py

@@ -382,10 +382,10 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
         network_mode = kubernetes_enums.KubernetesNetworkingMode.from_str(
             network_mode_str)
     except ValueError as e:
-        # Add message saying "Please check: ~/.sky/config.yaml" to the error
+        # Add message saying "Please check: ~/.sky/skyconfig.yaml" to the error
         # message.
         with ux_utils.print_exception_no_traceback():
-            raise ValueError(str(e) + ' Please check: ~/.sky/config.yaml.') \
+            raise ValueError(str(e) + ' Please check: ~/.sky/skyconfig.yaml.') \
                 from None
     _, public_key_path = get_or_generate_keys()
 

sky/backends/backend_utils.py

@@ -682,7 +682,7 @@ def write_cluster_config(
             ssh_proxy_command = ssh_proxy_command_config[region_name]
     logger.debug(f'Using ssh_proxy_command: {ssh_proxy_command!r}')
 
-    # User-supplied global instance tags from ~/.sky/config.yaml.
+    # User-supplied global instance tags from ~/.sky/skyconfig.yaml.
     labels = skypilot_config.get_nested((str(cloud).lower(), 'labels'), {})
     # labels is a dict, which is guaranteed by the type check in
     # schemas.py

sky/backends/cloud_vm_ray_backend.py

@@ -1473,7 +1473,7 @@ class RetryingVmProvisioner(object):
                     f'invalid cloud credentials: '
                     f'{common_utils.format_exception(e)}')
             except exceptions.InvalidCloudConfigs as e:
-                # Failed due to invalid user configs in ~/.sky/config.yaml.
+                # Failed due to invalid user configs in ~/.sky/skyconfig.yaml.
                 logger.warning(f'{common_utils.format_exception(e)}')
                 # We should block the entire cloud if the user config is
                 # invalid.
@@ -2065,10 +2065,10 @@ class RetryingVmProvisioner(object):
                                (clouds.Kubernetes, clouds.RunPod)) and
                         controller_utils.Controllers.from_name(cluster_name)
                         is not None):
-                    assert (clouds.CloudImplementationFeatures.STOP
+                    assert (clouds.CloudImplementationFeatures.AUTOSTOP
                             in requested_features), requested_features
                     requested_features.remove(
-                        clouds.CloudImplementationFeatures.STOP)
+                        clouds.CloudImplementationFeatures.AUTOSTOP)
 
                 # Skip if to_provision.cloud does not support requested features
                 to_provision.cloud.check_features_are_supported(

sky/check.py

@@ -142,7 +142,7 @@ def check_capabilities(
     if disallowed_cloud_names:
         disallowed_clouds_hint = (
             '\nNote: The following clouds were disabled because they were not '
-            'included in allowed_clouds in ~/.sky/config.yaml: '
+            'included in allowed_clouds in ~/.sky/skyconfig.yaml: '
             f'{", ".join([c for c in disallowed_cloud_names])}')
     if not all_enabled_clouds:
         echo(

sky/cli.py

@@ -35,7 +35,8 @@ import sys
 import textwrap
 import traceback
 import typing
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
+from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Tuple,
+                    Union)
 
 import click
 import colorama
@@ -134,49 +135,51 @@ def _get_cluster_records_and_set_ssh_config(
     # Update the SSH config for all clusters
     for record in cluster_records:
         handle = record['handle']
-        # During the failover, even though a cluster does not exist, the handle
-        # can still exist in the record, and we check for credentials to avoid
-        # updating the SSH config for non-existent clusters.
-        if (handle is not None and handle.cached_external_ips is not None and
-                'credentials' in record):
-            credentials = record['credentials']
-            if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
-                # Replace the proxy command to proxy through the SkyPilot API
-                # server with websocket.
-                key_path = (
-                    cluster_utils.SSHConfigHelper.generate_local_key_file(
-                        handle.cluster_name, credentials))
-                # Instead of directly use websocket_proxy.py, we add an
-                # additional proxy, so that ssh can use the head pod in the
-                # cluster to jump to worker pods.
-                proxy_command = (
-                    f'ssh -tt -i {key_path} '
-                    '-o StrictHostKeyChecking=no '
-                    '-o UserKnownHostsFile=/dev/null '
-                    '-o IdentitiesOnly=yes '
-                    '-W %h:%p '
-                    f'{handle.ssh_user}@127.0.0.1 '
-                    '-o ProxyCommand='
-                    # TODO(zhwu): write the template to a temp file, don't use
-                    # the one in skypilot repo, to avoid changing the file when
-                    # updating skypilot.
-                    f'\'{sys.executable} {sky.__root_dir__}/templates/'
-                    f'websocket_proxy.py '
-                    f'{server_common.get_server_url().split("://")[1]} '
-                    f'{handle.cluster_name}\'')
-                credentials['ssh_proxy_command'] = proxy_command
-            cluster_utils.SSHConfigHelper.add_cluster(
-                handle.cluster_name,
-                handle.cached_external_ips,
-                credentials,
-                handle.cached_external_ssh_ports,
-                handle.docker_user,
-                handle.ssh_user,
-            )
-        else:
+
+        if not (handle is not None and handle.cached_external_ips is not None
+                and 'credentials' in record):
             # If the cluster is not UP or does not have credentials available,
             # we need to remove the cluster from the SSH config.
             cluster_utils.SSHConfigHelper.remove_cluster(record['name'])
+            continue
+
+        # During the failover, even though a cluster does not exist, the handle
+        # can still exist in the record, and we check for credentials to avoid
+        # updating the SSH config for non-existent clusters.
+        credentials = record['credentials']
+        if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
+            # Replace the proxy command to proxy through the SkyPilot API
+            # server with websocket.
+            key_path = (cluster_utils.SSHConfigHelper.generate_local_key_file(
+                handle.cluster_name, credentials))
+            # Instead of directly use websocket_proxy.py, we add an
+            # additional proxy, so that ssh can use the head pod in the
+            # cluster to jump to worker pods.
+            proxy_command = (
+                f'ssh -tt -i {key_path} '
+                '-o StrictHostKeyChecking=no '
+                '-o UserKnownHostsFile=/dev/null '
+                '-o IdentitiesOnly=yes '
+                '-W %h:%p '
+                f'{handle.ssh_user}@127.0.0.1 '
+                '-o ProxyCommand='
+                # TODO(zhwu): write the template to a temp file, don't use
+                # the one in skypilot repo, to avoid changing the file when
+                # updating skypilot.
+                f'\'{sys.executable} {sky.__root_dir__}/templates/'
+                f'websocket_proxy.py '
+                f'{server_common.get_server_url()} '
+                f'{handle.cluster_name}\'')
+            credentials['ssh_proxy_command'] = proxy_command
+
+        cluster_utils.SSHConfigHelper.add_cluster(
+            handle.cluster_name,
+            handle.cached_external_ips,
+            credentials,
+            handle.cached_external_ssh_ports,
+            handle.docker_user,
+            handle.ssh_user,
+        )
 
     # Clean up SSH configs for clusters that do not exist.
     #
@@ -186,14 +189,15 @@ def _get_cluster_records_and_set_ssh_config(
     # removing clusters, because SkyPilot has no idea whether to remove
     # ssh config of a cluster from another user.
     clusters_exists = set(record['name'] for record in cluster_records)
+    clusters_to_remove: Set[str] = set()
     if clusters is not None:
-        for cluster in clusters:
-            if cluster not in clusters_exists:
-                cluster_utils.SSHConfigHelper.remove_cluster(cluster)
+        clusters_to_remove = set(clusters) - clusters_exists
     elif all_users:
-        for cluster_name in cluster_utils.SSHConfigHelper.list_cluster_names():
-            if cluster_name not in clusters_exists:
-                cluster_utils.SSHConfigHelper.remove_cluster(cluster_name)
+        clusters_to_remove = set(cluster_utils.SSHConfigHelper.
+                                 list_cluster_names()) - clusters_exists
+
+    for cluster_name in clusters_to_remove:
+        cluster_utils.SSHConfigHelper.remove_cluster(cluster_name)
 
     return cluster_records
 

sky/client/cli.py

@@ -35,7 +35,8 @@ import sys
 import textwrap
 import traceback
 import typing
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
+from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Tuple,
+                    Union)
 
 import click
 import colorama
@@ -134,49 +135,51 @@ def _get_cluster_records_and_set_ssh_config(
     # Update the SSH config for all clusters
     for record in cluster_records:
         handle = record['handle']
-        # During the failover, even though a cluster does not exist, the handle
-        # can still exist in the record, and we check for credentials to avoid
-        # updating the SSH config for non-existent clusters.
-        if (handle is not None and handle.cached_external_ips is not None and
-                'credentials' in record):
-            credentials = record['credentials']
-            if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
-                # Replace the proxy command to proxy through the SkyPilot API
-                # server with websocket.
-                key_path = (
-                    cluster_utils.SSHConfigHelper.generate_local_key_file(
-                        handle.cluster_name, credentials))
-                # Instead of directly use websocket_proxy.py, we add an
-                # additional proxy, so that ssh can use the head pod in the
-                # cluster to jump to worker pods.
-                proxy_command = (
-                    f'ssh -tt -i {key_path} '
-                    '-o StrictHostKeyChecking=no '
-                    '-o UserKnownHostsFile=/dev/null '
-                    '-o IdentitiesOnly=yes '
-                    '-W %h:%p '
-                    f'{handle.ssh_user}@127.0.0.1 '
-                    '-o ProxyCommand='
-                    # TODO(zhwu): write the template to a temp file, don't use
-                    # the one in skypilot repo, to avoid changing the file when
-                    # updating skypilot.
-                    f'\'{sys.executable} {sky.__root_dir__}/templates/'
-                    f'websocket_proxy.py '
-                    f'{server_common.get_server_url().split("://")[1]} '
-                    f'{handle.cluster_name}\'')
-                credentials['ssh_proxy_command'] = proxy_command
-            cluster_utils.SSHConfigHelper.add_cluster(
-                handle.cluster_name,
-                handle.cached_external_ips,
-                credentials,
-                handle.cached_external_ssh_ports,
-                handle.docker_user,
-                handle.ssh_user,
-            )
-        else:
+
+        if not (handle is not None and handle.cached_external_ips is not None
+                and 'credentials' in record):
             # If the cluster is not UP or does not have credentials available,
             # we need to remove the cluster from the SSH config.
             cluster_utils.SSHConfigHelper.remove_cluster(record['name'])
+            continue
+
+        # During the failover, even though a cluster does not exist, the handle
+        # can still exist in the record, and we check for credentials to avoid
+        # updating the SSH config for non-existent clusters.
+        credentials = record['credentials']
+        if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
+            # Replace the proxy command to proxy through the SkyPilot API
+            # server with websocket.
+            key_path = (cluster_utils.SSHConfigHelper.generate_local_key_file(
+                handle.cluster_name, credentials))
+            # Instead of directly use websocket_proxy.py, we add an
+            # additional proxy, so that ssh can use the head pod in the
+            # cluster to jump to worker pods.
+            proxy_command = (
+                f'ssh -tt -i {key_path} '
+                '-o StrictHostKeyChecking=no '
+                '-o UserKnownHostsFile=/dev/null '
+                '-o IdentitiesOnly=yes '
+                '-W %h:%p '
+                f'{handle.ssh_user}@127.0.0.1 '
+                '-o ProxyCommand='
+                # TODO(zhwu): write the template to a temp file, don't use
+                # the one in skypilot repo, to avoid changing the file when
+                # updating skypilot.
+                f'\'{sys.executable} {sky.__root_dir__}/templates/'
+                f'websocket_proxy.py '
+                f'{server_common.get_server_url()} '
+                f'{handle.cluster_name}\'')
+            credentials['ssh_proxy_command'] = proxy_command
+
+        cluster_utils.SSHConfigHelper.add_cluster(
+            handle.cluster_name,
+            handle.cached_external_ips,
+            credentials,
+            handle.cached_external_ssh_ports,
+            handle.docker_user,
+            handle.ssh_user,
+        )
 
     # Clean up SSH configs for clusters that do not exist.
     #
@@ -186,14 +189,15 @@ def _get_cluster_records_and_set_ssh_config(
     # removing clusters, because SkyPilot has no idea whether to remove
     # ssh config of a cluster from another user.
     clusters_exists = set(record['name'] for record in cluster_records)
+    clusters_to_remove: Set[str] = set()
     if clusters is not None:
-        for cluster in clusters:
-            if cluster not in clusters_exists:
-                cluster_utils.SSHConfigHelper.remove_cluster(cluster)
+        clusters_to_remove = set(clusters) - clusters_exists
     elif all_users:
-        for cluster_name in cluster_utils.SSHConfigHelper.list_cluster_names():
-            if cluster_name not in clusters_exists:
-                cluster_utils.SSHConfigHelper.remove_cluster(cluster_name)
+        clusters_to_remove = set(cluster_utils.SSHConfigHelper.
+                                 list_cluster_names()) - clusters_exists
+
+    for cluster_name in clusters_to_remove:
+        cluster_utils.SSHConfigHelper.remove_cluster(cluster_name)
 
     return cluster_records
 

sky/client/sdk.py

@@ -1812,7 +1812,8 @@ def api_login(endpoint: Optional[str] = None) -> None:
     server_common.check_server_healthy(endpoint)
 
     # Set the endpoint in the config file
-    config_path = pathlib.Path(skypilot_config.CONFIG_PATH).expanduser()
+    config_path = pathlib.Path(
+        skypilot_config.get_user_config_path()).expanduser()
     with filelock.FileLock(config_path.with_suffix('.lock')):
         if not skypilot_config.loaded():
             config_path.touch()

sky/clouds/aws.py

@@ -472,10 +472,10 @@ class AWS(clouds.Cloud):
             with ux_utils.print_exception_no_traceback():
                 logger.warning(
                     f'Skip opening ports {resources.ports} for cluster {cluster_name!r}, '
-                    'as `aws.security_group_name` in `~/.sky/config.yaml` is specified as '
+                    'as `aws.security_group_name` in `~/.sky/skyconfig.yaml` is specified as '
                     f' {security_group!r}. Please make sure the specified security group '
                     'has requested ports setup; or, leave out `aws.security_group_name` '
-                    'in `~/.sky/config.yaml`.')
+                    'in `~/.sky/skyconfig.yaml`.')
 
         return {
             'instance_type': r.instance_type,

sky/clouds/cloud.py

@@ -37,7 +37,7 @@ class CloudImplementationFeatures(enum.Enum):
     _cloud_unsupported_features in all clouds to make sure the
     check_features_are_supported() works as expected.
     """
-    STOP = 'stop'  # Includes both stop and autostop.
+    STOP = 'stop'
     MULTI_NODE = 'multi-node'
     CLONE_DISK_FROM_CLUSTER = 'clone_disk_from_cluster'
     IMAGE_ID = 'image_id'
@@ -47,7 +47,8 @@ class CloudImplementationFeatures(enum.Enum):
     OPEN_PORTS = 'open_ports'
     STORAGE_MOUNTING = 'storage_mounting'
     HOST_CONTROLLERS = 'host_controllers'  # Can run jobs/serve controllers
-    AUTO_TERMINATE = 'auto_terminate'  # Pod/VM can stop or down itself
+    AUTOSTOP = 'autostop'  # Pod/VM can stop itself
+    AUTODOWN = 'autodown'  # Pod/VM can down itself
 
 
 # Use str, enum.Enum to allow CloudCapability to be used as a string.

sky/clouds/kubernetes.py

@@ -35,6 +35,10 @@ CREDENTIAL_PATH = os.environ.get('KUBECONFIG', DEFAULT_KUBECONFIG_PATH)
 # E.g., FUSE device manager daemonset is run in this namespace.
 _SKYPILOT_SYSTEM_NAMESPACE = 'skypilot-system'
 
+# Shared directory to communicate with fusermount-server, refer to
+# addons/fuse-proxy/README.md for more details.
+_FUSERMOUNT_SHARED_DIR = '/var/run/fusermount'
+
 
 @registry.CLOUD_REGISTRY.register(aliases=['k8s'])
 class Kubernetes(clouds.Cloud):
@@ -110,9 +114,13 @@ class Kubernetes(clouds.Cloud):
             # Controllers cannot spin up new pods with exec auth.
             unsupported_features[
                 clouds.CloudImplementationFeatures.HOST_CONTROLLERS] = message
-            # Pod does not have permissions to terminate itself with exec auth.
+            # Pod does not have permissions to down itself with exec auth.
             unsupported_features[
-                clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
+                clouds.CloudImplementationFeatures.AUTODOWN] = message
+        unsupported_features[clouds.CloudImplementationFeatures.STOP] = (
+            'Stopping clusters is not supported on Kubernetes.')
+        unsupported_features[clouds.CloudImplementationFeatures.AUTOSTOP] = (
+            'Auto-stop is not supported on Kubernetes.')
         # Allow spot instances if supported by the cluster
         try:
             spot_label_key, _ = kubernetes_utils.get_spot_label(context)
@@ -551,8 +559,9 @@ class Kubernetes(clouds.Cloud):
             'k8s_service_account_name': k8s_service_account_name,
             'k8s_automount_sa_token': k8s_automount_sa_token,
             'k8s_fuse_device_required': fuse_device_required,
-            # Namespace to run the FUSE device manager in
+            # Namespace to run the fusermount-server daemonset in
             'k8s_skypilot_system_namespace': _SKYPILOT_SYSTEM_NAMESPACE,
+            'k8s_fusermount_shared_dir': _FUSERMOUNT_SHARED_DIR,
             'k8s_spot_label_key': spot_label_key,
             'k8s_spot_label_value': spot_label_value,
             'tpu_requested': tpu_requested,
@@ -658,6 +667,14 @@ class Kubernetes(clouds.Cloud):
     def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to
         Kubernetes."""
+        # Check for port forward dependencies
+        reasons = kubernetes_utils.check_port_forward_mode_dependencies(False)
+        if reasons is not None:
+            formatted = '\n'.join(
+                [reasons[0]] +
+                [f'{cls._INDENT_PREFIX}' + r for r in reasons[1:]])
+            return (False, formatted)
+
         # Test using python API
         try:
             existing_allowed_contexts = cls.existing_allowed_contexts()

sky/clouds/nebius.py

@@ -53,10 +53,8 @@ class Nebius(clouds.Cloud):
     """Nebius GPU Cloud"""
     _REPR = 'Nebius'
     _CLOUD_UNSUPPORTED_FEATURES = {
-        clouds.CloudImplementationFeatures.AUTO_TERMINATE:
-            ('Autodown and Autostop not supported. Can\'t delete disk.'),
-        # Autostop functionality can be implemented, but currently,
-        # there is only a single flag for both autostop and autodown.
+        clouds.CloudImplementationFeatures.AUTODOWN:
+            ('Autodown not supported. Can\'t delete OS disk.'),
         clouds.CloudImplementationFeatures.SPOT_INSTANCE:
             ('Spot is not supported, as Nebius API does not implement spot.'),
         clouds.CloudImplementationFeatures.CLONE_DISK_FROM_CLUSTER:

sky/clouds/oci.py

@@ -9,8 +9,8 @@ History:
    file path resolution (by os.path.expanduser) when construct the file
    mounts. This bug will cause the created workder nodes located in different
    compartment and VCN than the header node if user specifies compartment_id
-   in the sky config file, because the ~/.sky/config.yaml is not sync-ed to the
-   remote machine.
+   in the sky config file, because the ~/.sky/skyconfig.yaml is not
+   sync-ed to the remote machine.
    The workaround is set the sky config file path using ENV before running
    the sky launch: export SKYPILOT_CONFIG=/home/ubuntu/.sky/config.yaml
  - Hysun He (hysun.he@oracle.com) @ Oct 12, 2024:

sky/clouds/utils/oci_utils.py

@@ -147,7 +147,7 @@ class OCIConfig:
         if config_path_via_env_var is not None:
             config_path = config_path_via_env_var
         else:
-            config_path = skypilot_config.CONFIG_PATH
+            config_path = skypilot_config.get_user_config_path()
         return config_path
 
     @classmethod

sky/core.py

@@ -629,26 +629,21 @@ def autostop(
         raise exceptions.NotSupportedError(
             f'{operation} cluster {cluster_name!r} with backend '
             f'{backend.__class__.__name__!r} is not supported.')
-    # Check autostop is implemented for cloud
     cloud = handle.launched_resources.cloud
-    if not down and not is_cancel:
-        try:
-            cloud.check_features_are_supported(
-                handle.launched_resources,
-                {clouds.CloudImplementationFeatures.STOP})
-        except exceptions.NotSupportedError as e:
-            raise exceptions.NotSupportedError(
-                f'{colorama.Fore.YELLOW}Scheduling autostop on cluster '
-                f'{cluster_name!r}...skipped.{colorama.Style.RESET_ALL}\n'
-                f'  {_stop_not_supported_message(handle.launched_resources)}.'
-            ) from e
-
-    # Check if autodown is required and supported
+    # Check if autostop/autodown is required and supported
     if not is_cancel:
         try:
-            cloud.check_features_are_supported(
-                handle.launched_resources,
-                {clouds.CloudImplementationFeatures.AUTO_TERMINATE})
+            if down:
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.AUTODOWN})
+            else:
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.STOP})
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.AUTOSTOP})
         except exceptions.NotSupportedError as e:
             raise exceptions.NotSupportedError(
                 f'{colorama.Fore.YELLOW}{operation} on cluster '

sky/data/mounting_utils.py

@@ -30,9 +30,17 @@ _BLOBFUSE_CACHE_DIR = ('~/.sky/blobfuse2_cache/'
 # https://github.com/rclone/rclone/releases
 RCLONE_VERSION = 'v1.68.2'
 
+# A wrapper for goofys to choose the logging mechanism based on environment.
+_GOOFYS_WRAPPER = ('$(if [ -S /dev/log ] ; then '
+                   'echo "goofys"; '
+                   'else '
+                   'echo "goofys --log-file $(mktemp -t goofys.XXXX.log)"; '
+                   'fi)')
+
 
 def get_s3_mount_install_cmd() -> str:
     """Returns a command to install S3 mount utility goofys."""
+    # TODO(aylei): maintain our goofys fork under skypilot-org
     install_cmd = ('ARCH=$(uname -m) && '
                    'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
                    '  echo "goofys is not supported on $ARCH" && '
@@ -40,8 +48,8 @@ def get_s3_mount_install_cmd() -> str:
                    'else '
                    '  ARCH_SUFFIX="amd64"; '
                    'fi && '
-                   'sudo wget -nc https://github.com/romilbhardwaj/goofys/'
-                   'releases/download/0.24.0-romilb-upstream/goofys '
+                   'sudo wget -nc https://github.com/aylei/goofys/'
+                   'releases/download/0.24.0-aylei-upstream/goofys '
                    '-O /usr/local/bin/goofys && '
                    'sudo chmod 755 /usr/local/bin/goofys')
     return install_cmd
@@ -56,7 +64,7 @@ def get_s3_mount_cmd(bucket_name: str,
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = ('goofys -o allow_other '
+    mount_cmd = (f'{_GOOFYS_WRAPPER} -o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'{bucket_name}{_bucket_sub_path} {mount_path}')
@@ -73,7 +81,8 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} goofys -o allow_other '
+    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} {_GOOFYS_WRAPPER} '
+                 '-o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'--endpoint {endpoint_url} '
@@ -185,14 +194,28 @@ def get_az_mount_cmd(container_name: str,
         bucket_sub_path_arg = ''
     else:
         bucket_sub_path_arg = f'--subdirectory={_bucket_sub_path}/ '
+    mount_options = '-o allow_other -o default_permissions'
     # TODO(zpoint): clear old cache that has been created in the previous boot.
+    blobfuse2_cmd = ('blobfuse2 --no-symlinks -o umask=022 '
+                     f'--tmp-path {cache_path}_$({remote_boot_time_cmd}) '
+                     f'{bucket_sub_path_arg}'
+                     f'--container-name {container_name}')
+    # 1. Set -o nonempty to bypass empty directory check of blobfuse2 when using
+    # fusermount-wrapper, since the mount is delegated to fusermount and
+    # blobfuse2 only get the mounted fd.
+    # 2. {} is the mount point placeholder that will be replaced with the
+    # mounted fd by fusermount-wrapper.
+    wrapped = (f'fusermount-wrapper -m {mount_path} {mount_options} '
+               f'-- {blobfuse2_cmd} -o nonempty {{}}')
+    original = f'{blobfuse2_cmd} {mount_options} {mount_path}'
+    # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
+    # to avoid requiring root privilege.
+    # TODO(aylei): feeling hacky, refactor this.
+    get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
+                     f'echo "{wrapped}" || echo "{original}"')
     mount_cmd = (f'AZURE_STORAGE_ACCOUNT={storage_account_name} '
                  f'{key_env_var} '
-                 f'blobfuse2 {mount_path} --allow-other --no-symlinks '
-                 '-o umask=022 -o default_permissions '
-                 f'--tmp-path {cache_path}_$({remote_boot_time_cmd}) '
-                 f'{bucket_sub_path_arg}'
-                 f'--container-name {container_name}')
+                 f'$({get_mount_cmd})')
     return mount_cmd
 
 
@@ -209,7 +232,8 @@ def get_r2_mount_cmd(r2_credentials_path: str,
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
     mount_cmd = (f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
-                 f'AWS_PROFILE={r2_profile_name} goofys -o allow_other '
+                 f'AWS_PROFILE={r2_profile_name} {_GOOFYS_WRAPPER} '
+                 '-o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'--endpoint {endpoint_url} '

sky/exceptions.py

@@ -265,7 +265,7 @@ class CommandError(SkyPilotExcludeArgsBaseException):
                 # Chunk the command to avoid overflow.
                 command = command[:100] + '...'
             message = (f'Command {command} failed with return code '
-                       f'{returncode}.\n{error_msg}')
+                       f'{returncode}.\n{error_msg}\n{detailed_reason}')
         super().__init__(message)
 
 

sky/execution.py

@@ -237,11 +237,12 @@ def _execute(
             if Stage.DOWN in stages:
                 stages.remove(Stage.DOWN)
             if idle_minutes_to_autostop >= 0:
-                requested_features.add(
-                    clouds.CloudImplementationFeatures.AUTO_TERMINATE)
-                if not down:
+                if down:
                     requested_features.add(
-                        clouds.CloudImplementationFeatures.STOP)
+                        clouds.CloudImplementationFeatures.AUTODOWN)
+                else:
+                    requested_features.add(
+                        clouds.CloudImplementationFeatures.AUTOSTOP)
         # NOTE: in general we may not have sufficiently specified info
         # (cloud/resource) to check STOP_SPOT_INSTANCE here. This is checked in
         # the backend.

sky/provision/instance_setup.py

@@ -44,7 +44,9 @@ _DUMP_RAY_PORTS = (
 
 _RAY_PORT_COMMAND = (
     f'RAY_PORT=$({constants.SKY_PYTHON_CMD} -c '
-    '"from sky.skylet import job_lib; print(job_lib.get_ray_port())" '
+    '"from sky import sky_logging\n'
+    'with sky_logging.silent(): '
+    'from sky.skylet import job_lib; print(job_lib.get_ray_port())" '
     '2> /dev/null || echo 6379);'
     f'{constants.SKY_PYTHON_CMD} -c "from sky.utils import message_utils; '
     'print(message_utils.encode_payload({\'ray_port\': $RAY_PORT}))"')