skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/data/mounting_utils.py

@@ -17,6 +17,16 @@ _TYPE_CACHE_TTL = '5s'
 _RENAME_DIR_LIMIT = 10000
 # https://github.com/GoogleCloudPlatform/gcsfuse/releases
 GCSFUSE_VERSION = '2.2.0'
+
+# Some machines do not have fuse/fuse3 installed by default
+# hence rclone will fail on these machines
+FUSE3_INSTALL_CMD = ('(command -v fusermount3 > /dev/null 2>&1 || '
+                     '((which apt-get > /dev/null 2>&1 && '
+                     'sudo apt-get update && sudo apt-get install -y fuse3) || '
+                     '(which yum > /dev/null 2>&1 && '
+                     'sudo yum install -y fuse3) || '
+                     'true)) || true')
+
 # Creates a fusermount3 soft link on older (<22) Ubuntu systems to utilize
 # Rclone's mounting utility.
 FUSERMOUNT3_SOFT_LINK_CMD = ('[ ! -f /bin/fusermount3 ] && '
@@ -38,20 +48,45 @@ _GOOFYS_WRAPPER = ('$(if [ -S /dev/log ] ; then '
                    'fi)')
 
 
+def get_rclone_install_cmd() -> str:
+    """ RClone installation for both apt-get and rpm.
+    This would be common command.
+    """
+    # pylint: disable=line-too-long
+    install_cmd = (
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        '  ARCH_SUFFIX="arm64"; '
+        'else '
+        '  ARCH_SUFFIX="amd64"; '
+        'fi && '
+        f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
+        f' || (which yum > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
+        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
+        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
+        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm)))')
+    return install_cmd
+
+
 def get_s3_mount_install_cmd() -> str:
-    """Returns a command to install S3 mount utility goofys."""
+    """Returns command for basic S3 mounting (goofys by default, rclone for
+    ARM64)."""
     # TODO(aylei): maintain our goofys fork under skypilot-org
-    install_cmd = ('ARCH=$(uname -m) && '
-                   'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
-                   '  echo "goofys is not supported on $ARCH" && '
-                   f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
-                   'else '
-                   '  ARCH_SUFFIX="amd64"; '
-                   'fi && '
-                   'sudo wget -nc https://github.com/aylei/goofys/'
-                   'releases/download/0.24.0-aylei-upstream/goofys '
-                   '-O /usr/local/bin/goofys && '
-                   'sudo chmod 755 /usr/local/bin/goofys')
+    install_cmd = (
+        'ARCH=$(uname -m) && '
+        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+        # Use rclone for ARM64 since goofys doesn't support it
+        # Extract core rclone installation logic without redundant ARCH check
+        f'  {get_rclone_install_cmd()}; '
+        'else '
+        '  sudo wget -nc https://github.com/aylei/goofys/'
+        'releases/download/0.24.0-aylei-upstream/goofys '
+        '-O /usr/local/bin/goofys && '
+        'sudo chmod 755 /usr/local/bin/goofys; '
+        'fi')
     return install_cmd
 
 
@@ -59,15 +94,33 @@ def get_s3_mount_install_cmd() -> str:
 def get_s3_mount_cmd(bucket_name: str,
                      mount_path: str,
                      _bucket_sub_path: Optional[str] = None) -> str:
-    """Returns a command to mount an S3 bucket using goofys."""
+    """Returns a command to mount an S3 bucket (goofys by default, rclone for
+    ARM64)"""
     if _bucket_sub_path is None:
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = (f'{_GOOFYS_WRAPPER} -o allow_other '
-                 f'--stat-cache-ttl {_STAT_CACHE_TTL} '
-                 f'--type-cache-ttl {_TYPE_CACHE_TTL} '
-                 f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    # Use rclone for ARM64 architectures since goofys doesn't support them
+    arch_check = 'ARCH=$(uname -m) && '
+    rclone_mount = (
+        f'{FUSE3_INSTALL_CMD} && '
+        f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
+        f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
+        # Have to add --s3-env-auth=true to allow rclone to access private
+        # buckets.
+        '--daemon --allow-other --s3-env-auth=true')
+    goofys_mount = (f'{_GOOFYS_WRAPPER} -o allow_other '
+                    f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                    f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                    f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    mount_cmd = (f'{arch_check}'
+                 f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                 f'  {rclone_mount}; '
+                 f'else '
+                 f'  {goofys_mount}; '
+                 f'fi')
     return mount_cmd
 
 
@@ -76,17 +129,74 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
                          endpoint_url: str,
                          mount_path: str,
                          _bucket_sub_path: Optional[str] = None) -> str:
-    """Returns a command to install Nebius mount utility goofys."""
+    """Returns a command to mount Nebius bucket (goofys by default, rclone for
+    ARM64)."""
     if _bucket_sub_path is None:
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} {_GOOFYS_WRAPPER} '
-                 '-o allow_other '
-                 f'--stat-cache-ttl {_STAT_CACHE_TTL} '
-                 f'--type-cache-ttl {_TYPE_CACHE_TTL} '
-                 f'--endpoint {endpoint_url} '
-                 f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    # Use rclone for ARM64 architectures since goofys doesn't support them
+    arch_check = 'ARCH=$(uname -m) && '
+    rclone_mount = (
+        f'{FUSE3_INSTALL_CMD} && '
+        f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
+        f'AWS_PROFILE={nebius_profile_name} '
+        f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
+        f'--s3-endpoint {endpoint_url} --daemon --allow-other')
+    goofys_mount = (f'AWS_PROFILE={nebius_profile_name} {_GOOFYS_WRAPPER} '
+                    '-o allow_other '
+                    f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                    f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                    f'--endpoint {endpoint_url} '
+                    f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    mount_cmd = (f'{arch_check}'
+                 f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                 f'  {rclone_mount}; '
+                 f'else '
+                 f'  {goofys_mount}; '
+                 f'fi')
+    return mount_cmd
+
+
+def get_coreweave_mount_cmd(cw_credentials_path: str,
+                            coreweave_profile_name: str,
+                            bucket_name: str,
+                            endpoint_url: str,
+                            mount_path: str,
+                            _bucket_sub_path: Optional[str] = None) -> str:
+    """Returns a command to mount CoreWeave bucket"""
+    if _bucket_sub_path is None:
+        _bucket_sub_path = ''
+    else:
+        _bucket_sub_path = f':{_bucket_sub_path}'
+
+    # Use rclone for ARM64 architectures since goofys doesn't support them
+    arch_check = 'ARCH=$(uname -m) && '
+    rclone_mount = (
+        f'{FUSE3_INSTALL_CMD} && '
+        f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
+        f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
+        f'AWS_PROFILE={coreweave_profile_name} '
+        f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
+        f'--s3-force-path-style=false '
+        f'--s3-endpoint {endpoint_url} --daemon --allow-other')
+    goofys_mount = (f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
+                    f'AWS_PROFILE={coreweave_profile_name} {_GOOFYS_WRAPPER} '
+                    '-o allow_other '
+                    f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                    f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                    f'--subdomain '
+                    f'--endpoint {endpoint_url} '
+                    f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    mount_cmd = (f'{arch_check}'
+                 f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                 f'  {rclone_mount}; '
+                 f'else '
+                 f'  {goofys_mount}; '
+                 f'fi')
     return mount_cmd
 
 
@@ -127,27 +237,72 @@ def get_gcs_mount_cmd(bucket_name: str,
 def get_az_mount_install_cmd() -> str:
     """Returns a command to install AZ Container mount utility blobfuse2."""
     install_cmd = (
-        'sudo apt-get update; '
-        'sudo apt-get install -y '
-        '-o Dpkg::Options::="--force-confdef" '
-        'fuse3 libfuse3-dev || { '
-        '  echo "fuse3 not available, falling back to fuse"; '
-        '  sudo apt-get install -y '
-        '  -o Dpkg::Options::="--force-confdef" '
-        '  fuse libfuse-dev; '
-        '} && '
+        # Check architecture first - blobfuse2 only supports x86_64
         'ARCH=$(uname -m) && '
         'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
         '  echo "blobfuse2 is not supported on $ARCH" && '
         f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
+        # Try to install fuse3 from default repos
+        'sudo apt-get update && '
+        'FUSE3_INSTALLED=0 && '
+        # On Kubernetes, if FUSERMOUNT_SHARED_DIR is set, it means
+        # fusermount and fusermount3 is symlinked to fusermount-shim.
+        # If we reinstall fuse3, it may overwrite the symlink, so
+        # just install libfuse3, which is needed by blobfuse2.
+        'if [ -n "${FUSERMOUNT_SHARED_DIR:-}" ]; then '
+        '  PACKAGES="libfuse3-3 libfuse3-dev"; '
+        'else '
+        '  PACKAGES="fuse3 libfuse3-3 libfuse3-dev"; '
+        'fi && '
+        'if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        '$PACKAGES; then '
+        '  FUSE3_INSTALLED=1; '
+        '  echo "fuse3 installed from default repos"; '
         'else '
-        '  ARCH_SUFFIX="x86_64"; '
+        # If fuse3 not available, try focal for Ubuntu <= 20.04
+        '  DISTRO=$(grep "^ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\' | tr "[:upper:]" "[:lower:]") && '
+        '  VERSION=$(grep "^VERSION_ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\') && '
+        '  if [ "$DISTRO" = "ubuntu" ] && '
+        '[ "$(echo "$VERSION 20.04" | '
+        'awk \'{ print ($1 <= $2) }\')" = "1" ]; then '
+        '    echo "Trying to install fuse3 from focal for '
+        'Ubuntu $VERSION"; '
+        '    echo "deb http://archive.ubuntu.com/ubuntu '
+        'focal main universe" | '
+        'sudo tee /etc/apt/sources.list.d/focal-fuse3.list && '
+        '    sudo apt-get update && '
+        '    if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        '-o Dpkg::Options::="--force-confold" '
+        '$PACKAGES; then '
+        '      FUSE3_INSTALLED=1; '
+        '      echo "fuse3 installed from focal"; '
+        '      sudo rm /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    else '
+        '      sudo rm -f /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    fi; '
+        '  fi; '
         'fi && '
-        'wget -nc https://github.com/Azure/azure-storage-fuse'
-        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
-        f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
+        # Install blobfuse2 only if fuse3 is available
+        'if [ "$FUSE3_INSTALLED" = "1" ]; then '
+        '  echo "Installing blobfuse2 with libfuse3 support"; '
+        '  wget -nc https://github.com/Azure/azure-storage-fuse'
+        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}/'
+        f'blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
         '-O /tmp/blobfuse2.deb && '
-        'sudo dpkg --install /tmp/blobfuse2.deb && '
+        '  sudo dpkg --install /tmp/blobfuse2.deb; '
+        'else '
+        '  echo "Error: libfuse3 is required for Azure storage '
+        'mounting with fusermount-wrapper."; '
+        '  echo "libfuse3 could not be installed on this system."; '
+        f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
         f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
 
     return install_cmd
@@ -219,7 +374,10 @@ def get_az_mount_cmd(container_name: str,
                f'-- {blobfuse2_cmd} -o nonempty --foreground {{}}')
     original = f'{blobfuse2_cmd} {blobfuse2_options} {mount_path}'
     # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
-    # to avoid requiring root privilege.
+    # to avoid requiring privileged containers.
+    # fusermount-wrapper requires libfuse3;
+    # we install libfuse3 even on older distros like Ubuntu 18.04 by using
+    # Ubuntu 20.04 (focal) repositories.
     # TODO(aylei): feeling hacky, refactor this.
     get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
                      f'echo "{wrapped}" || echo "{original}"')
@@ -236,18 +394,36 @@ def get_r2_mount_cmd(r2_credentials_path: str,
                      bucket_name: str,
                      mount_path: str,
                      _bucket_sub_path: Optional[str] = None) -> str:
-    """Returns a command to install R2 mount utility goofys."""
+    """Returns a command to mount R2 bucket (goofys by default, rclone for
+    ARM64)."""
     if _bucket_sub_path is None:
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = (f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
-                 f'AWS_PROFILE={r2_profile_name} {_GOOFYS_WRAPPER} '
-                 '-o allow_other '
-                 f'--stat-cache-ttl {_STAT_CACHE_TTL} '
-                 f'--type-cache-ttl {_TYPE_CACHE_TTL} '
-                 f'--endpoint {endpoint_url} '
-                 f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    # Use rclone for ARM64 architectures since goofys doesn't support them
+    arch_check = 'ARCH=$(uname -m) && '
+    rclone_mount = (
+        f'{FUSE3_INSTALL_CMD} && '
+        f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
+        f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
+        f'AWS_PROFILE={r2_profile_name} '
+        f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
+        f'--s3-endpoint {endpoint_url} --daemon --allow-other')
+    goofys_mount = (f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
+                    f'AWS_PROFILE={r2_profile_name} {_GOOFYS_WRAPPER} '
+                    '-o allow_other '
+                    f'--stat-cache-ttl {_STAT_CACHE_TTL} '
+                    f'--type-cache-ttl {_TYPE_CACHE_TTL} '
+                    f'--endpoint {endpoint_url} '
+                    f'{bucket_name}{_bucket_sub_path} {mount_path}')
+
+    mount_cmd = (f'{arch_check}'
+                 f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
+                 f'  {rclone_mount}; '
+                 f'else '
+                 f'  {goofys_mount}; '
+                 f'fi')
     return mount_cmd
 
 
@@ -258,7 +434,8 @@ def get_cos_mount_cmd(rclone_config: str,
                       _bucket_sub_path: Optional[str] = None) -> str:
     """Returns a command to mount an IBM COS bucket using rclone."""
     # stores bucket profile in rclone config file at the cluster's nodes.
-    configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
+    configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
+                                f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
                                 f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
                                 f'echo "{rclone_config}" >> '
                                 f'{constants.RCLONE_CONFIG_PATH}')
@@ -278,7 +455,8 @@ def get_mount_cached_cmd(rclone_config: str, rclone_profile_name: str,
                          bucket_name: str, mount_path: str) -> str:
     """Returns a command to mount a bucket using rclone with vfs cache."""
     # stores bucket profile in rclone config file at the remote nodes.
-    configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
+    configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
+                                f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
                                 f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
                                 f'echo {shlex.quote(rclone_config)} >> '
                                 f'{constants.RCLONE_CONFIG_PATH}')
@@ -288,9 +466,9 @@ def get_mount_cached_cmd(rclone_config: str, rclone_profile_name: str,
     # the filename length limit.
     # The hash is a non-negative integer in string form.
     hashed_mount_path = hashlib.md5(mount_path.encode()).hexdigest()
-    log_file_path = os.path.join(constants.RCLONE_LOG_DIR,
+    log_file_path = os.path.join(constants.RCLONE_MOUNT_CACHED_LOG_DIR,
                                  f'{hashed_mount_path}.log')
-    create_log_cmd = (f'mkdir -p {constants.RCLONE_LOG_DIR} && '
+    create_log_cmd = (f'mkdir -p {constants.RCLONE_MOUNT_CACHED_LOG_DIR} && '
                       f'touch {log_file_path}')
     # when mounting multiple directories with vfs cache mode, it's handled by
     # rclone to create separate cache directories at ~/.cache/rclone/vfs. It is
@@ -331,29 +509,6 @@ def get_mount_cached_cmd(rclone_config: str, rclone_profile_name: str,
     return mount_cmd
 
 
-def get_rclone_install_cmd() -> str:
-    """ RClone installation for both apt-get and rpm.
-    This would be common command.
-    """
-    # pylint: disable=line-too-long
-    install_cmd = (
-        'ARCH=$(uname -m) && '
-        'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
-        '  ARCH_SUFFIX="arm"; '
-        'else '
-        '  ARCH_SUFFIX="amd64"; '
-        'fi && '
-        f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
-        f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
-        f' || (which rclone > /dev/null || (cd ~ > /dev/null'
-        f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
-        f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
-        f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm))')
-    return install_cmd
-
-
 def get_oci_mount_cmd(mount_path: str, store_name: str, region: str,
                       namespace: str, compartment: str, config_file: str,
                       config_profile: str) -> str:
@@ -433,13 +588,20 @@ def get_mounting_script(
 
         {command_runner.ALIAS_SUDO_TO_EMPTY_FOR_ROOT_CMD}
 
-        MOUNT_PATH={mount_path}
+        MOUNT_PATH=$(eval echo {mount_path})
         MOUNT_BINARY={mount_binary}
 
         # Check if path is already mounted
-        if grep -q $MOUNT_PATH /proc/mounts ; then
+        if findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then
             echo "Path already mounted - unmounting..."
-            fusermount -uz "$MOUNT_PATH"
+            (command -v fusermount >/dev/null 2>&1 && fusermount -uz "$MOUNT_PATH") \
+            || (command -v fusermount3 >/dev/null 2>&1 && fusermount3 -uz "$MOUNT_PATH") \
+            || sudo umount -l "$MOUNT_PATH" || true
+            # Ensure it's really gone (avoids races)
+            for i in $(seq 1 20); do
+                if ! findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then break; fi
+                sleep 0.2
+            done
             echo "Successfully unmounted $MOUNT_PATH."
         fi
 
@@ -454,17 +616,40 @@ def get_mounting_script(
         # Check if mount path exists
         if [ ! -d "$MOUNT_PATH" ]; then
           echo "Mount path $MOUNT_PATH does not exist. Creating..."
-          sudo mkdir -p $MOUNT_PATH
-          sudo chmod 777 $MOUNT_PATH
+          sudo mkdir -p "$MOUNT_PATH"
+          sudo chmod 777 "$MOUNT_PATH"
         else
-          # Check if mount path contains files
-          if [ "$(ls -A $MOUNT_PATH)" ]; then
-            echo "Mount path $MOUNT_PATH is not empty. Please mount to another path or remove it first."
-            exit {exceptions.MOUNT_PATH_NON_EMPTY_CODE}
-          fi
+            # If not a mountpoint and contains files, clean it to satisfy SkyPilot check
+            if ! findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then
+                if [ -n "$(ls -A "$MOUNT_PATH" 2>/dev/null)" ]; then
+                  echo "Cleaning non-empty mount path before mount..."
+                  sudo bash -lc 'shopt -s dotglob nullglob; rm -rf --one-file-system -- '"$MOUNT_PATH"'/*' 2>/dev/null || true
+                fi
+            fi
         fi
         echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..."
+        set +e
         {mount_cmd}
+        MOUNT_EXIT_CODE=$?
+        set -e
+        if [ $MOUNT_EXIT_CODE -ne 0 ]; then
+            echo "Mount failed with exit code $MOUNT_EXIT_CODE."
+            if [ "$MOUNT_BINARY" = "goofys" ]; then
+                echo "Looking for goofys log files..."
+                # Find goofys log files in /tmp (created by mktemp -t goofys.XXXX.log)
+                # Note: if /dev/log exists, goofys logs to syslog instead of a file
+                GOOFYS_LOGS=$(ls -t /tmp/goofys.*.log 2>/dev/null | head -1)
+                if [ -n "$GOOFYS_LOGS" ]; then
+                    echo "=== Goofys log file contents ==="
+                    cat "$GOOFYS_LOGS"
+                    echo "=== End of goofys log file ==="
+                else
+                    echo "No goofys log file found in /tmp"
+                fi
+            fi
+            # TODO(kevin): Print logs from rclone, etc too for observability.
+            exit $MOUNT_EXIT_CODE
+        fi
         echo "Mounting done."
     """)