skypilot-nightly 1.0.0.dev20250410__py3-none-any.whl → 1.0.0.dev20250412__py3-none-any.whl

sky/clouds/aws.py

@@ -472,10 +472,10 @@ class AWS(clouds.Cloud):
             with ux_utils.print_exception_no_traceback():
                 logger.warning(
                     f'Skip opening ports {resources.ports} for cluster {cluster_name!r}, '
-                    'as `aws.security_group_name` in `~/.sky/config.yaml` is specified as '
+                    'as `aws.security_group_name` in `~/.sky/skyconfig.yaml` is specified as '
                     f' {security_group!r}. Please make sure the specified security group '
                     'has requested ports setup; or, leave out `aws.security_group_name` '
-                    'in `~/.sky/config.yaml`.')
+                    'in `~/.sky/skyconfig.yaml`.')
 
         return {
             'instance_type': r.instance_type,

sky/clouds/cloud.py

@@ -37,7 +37,7 @@ class CloudImplementationFeatures(enum.Enum):
     _cloud_unsupported_features in all clouds to make sure the
     check_features_are_supported() works as expected.
     """
-    STOP = 'stop'  # Includes both stop and autostop.
+    STOP = 'stop'
     MULTI_NODE = 'multi-node'
     CLONE_DISK_FROM_CLUSTER = 'clone_disk_from_cluster'
     IMAGE_ID = 'image_id'
@@ -47,7 +47,8 @@ class CloudImplementationFeatures(enum.Enum):
     OPEN_PORTS = 'open_ports'
     STORAGE_MOUNTING = 'storage_mounting'
     HOST_CONTROLLERS = 'host_controllers'  # Can run jobs/serve controllers
-    AUTO_TERMINATE = 'auto_terminate'  # Pod/VM can stop or down itself
+    AUTOSTOP = 'autostop'  # Pod/VM can stop itself
+    AUTODOWN = 'autodown'  # Pod/VM can down itself
 
 
 # Use str, enum.Enum to allow CloudCapability to be used as a string.

sky/clouds/kubernetes.py

@@ -35,6 +35,10 @@ CREDENTIAL_PATH = os.environ.get('KUBECONFIG', DEFAULT_KUBECONFIG_PATH)
 # E.g., FUSE device manager daemonset is run in this namespace.
 _SKYPILOT_SYSTEM_NAMESPACE = 'skypilot-system'
 
+# Shared directory to communicate with fusermount-server, refer to
+# addons/fuse-proxy/README.md for more details.
+_FUSERMOUNT_SHARED_DIR = '/var/run/fusermount'
+
 
 @registry.CLOUD_REGISTRY.register(aliases=['k8s'])
 class Kubernetes(clouds.Cloud):
@@ -110,9 +114,13 @@ class Kubernetes(clouds.Cloud):
             # Controllers cannot spin up new pods with exec auth.
             unsupported_features[
                 clouds.CloudImplementationFeatures.HOST_CONTROLLERS] = message
-            # Pod does not have permissions to terminate itself with exec auth.
+            # Pod does not have permissions to down itself with exec auth.
             unsupported_features[
-                clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
+                clouds.CloudImplementationFeatures.AUTODOWN] = message
+        unsupported_features[clouds.CloudImplementationFeatures.STOP] = (
+            'Stopping clusters is not supported on Kubernetes.')
+        unsupported_features[clouds.CloudImplementationFeatures.AUTOSTOP] = (
+            'Auto-stop is not supported on Kubernetes.')
         # Allow spot instances if supported by the cluster
         try:
             spot_label_key, _ = kubernetes_utils.get_spot_label(context)
@@ -551,8 +559,9 @@ class Kubernetes(clouds.Cloud):
             'k8s_service_account_name': k8s_service_account_name,
             'k8s_automount_sa_token': k8s_automount_sa_token,
             'k8s_fuse_device_required': fuse_device_required,
-            # Namespace to run the FUSE device manager in
+            # Namespace to run the fusermount-server daemonset in
             'k8s_skypilot_system_namespace': _SKYPILOT_SYSTEM_NAMESPACE,
+            'k8s_fusermount_shared_dir': _FUSERMOUNT_SHARED_DIR,
             'k8s_spot_label_key': spot_label_key,
             'k8s_spot_label_value': spot_label_value,
             'tpu_requested': tpu_requested,
@@ -658,6 +667,14 @@ class Kubernetes(clouds.Cloud):
     def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to
         Kubernetes."""
+        # Check for port forward dependencies
+        reasons = kubernetes_utils.check_port_forward_mode_dependencies(False)
+        if reasons is not None:
+            formatted = '\n'.join(
+                [reasons[0]] +
+                [f'{cls._INDENT_PREFIX}' + r for r in reasons[1:]])
+            return (False, formatted)
+
         # Test using python API
         try:
             existing_allowed_contexts = cls.existing_allowed_contexts()

sky/clouds/nebius.py

@@ -53,10 +53,8 @@ class Nebius(clouds.Cloud):
     """Nebius GPU Cloud"""
     _REPR = 'Nebius'
     _CLOUD_UNSUPPORTED_FEATURES = {
-        clouds.CloudImplementationFeatures.AUTO_TERMINATE:
-            ('Autodown and Autostop not supported. Can\'t delete disk.'),
-        # Autostop functionality can be implemented, but currently,
-        # there is only a single flag for both autostop and autodown.
+        clouds.CloudImplementationFeatures.AUTODOWN:
+            ('Autodown not supported. Can\'t delete OS disk.'),
         clouds.CloudImplementationFeatures.SPOT_INSTANCE:
             ('Spot is not supported, as Nebius API does not implement spot.'),
         clouds.CloudImplementationFeatures.CLONE_DISK_FROM_CLUSTER:

sky/clouds/oci.py

@@ -9,8 +9,8 @@ History:
    file path resolution (by os.path.expanduser) when construct the file
    mounts. This bug will cause the created workder nodes located in different
    compartment and VCN than the header node if user specifies compartment_id
-   in the sky config file, because the ~/.sky/config.yaml is not sync-ed to the
-   remote machine.
+   in the sky config file, because the ~/.sky/skyconfig.yaml is not
+   sync-ed to the remote machine.
    The workaround is set the sky config file path using ENV before running
    the sky launch: export SKYPILOT_CONFIG=/home/ubuntu/.sky/config.yaml
  - Hysun He (hysun.he@oracle.com) @ Oct 12, 2024:

sky/clouds/utils/oci_utils.py

@@ -147,7 +147,7 @@ class OCIConfig:
         if config_path_via_env_var is not None:
             config_path = config_path_via_env_var
         else:
-            config_path = skypilot_config.CONFIG_PATH
+            config_path = skypilot_config.get_user_config_path()
         return config_path
 
     @classmethod

sky/core.py

@@ -629,26 +629,21 @@ def autostop(
         raise exceptions.NotSupportedError(
             f'{operation} cluster {cluster_name!r} with backend '
             f'{backend.__class__.__name__!r} is not supported.')
-    # Check autostop is implemented for cloud
     cloud = handle.launched_resources.cloud
-    if not down and not is_cancel:
-        try:
-            cloud.check_features_are_supported(
-                handle.launched_resources,
-                {clouds.CloudImplementationFeatures.STOP})
-        except exceptions.NotSupportedError as e:
-            raise exceptions.NotSupportedError(
-                f'{colorama.Fore.YELLOW}Scheduling autostop on cluster '
-                f'{cluster_name!r}...skipped.{colorama.Style.RESET_ALL}\n'
-                f'  {_stop_not_supported_message(handle.launched_resources)}.'
-            ) from e
-
-    # Check if autodown is required and supported
+    # Check if autostop/autodown is required and supported
     if not is_cancel:
         try:
-            cloud.check_features_are_supported(
-                handle.launched_resources,
-                {clouds.CloudImplementationFeatures.AUTO_TERMINATE})
+            if down:
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.AUTODOWN})
+            else:
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.STOP})
+                cloud.check_features_are_supported(
+                    handle.launched_resources,
+                    {clouds.CloudImplementationFeatures.AUTOSTOP})
         except exceptions.NotSupportedError as e:
             raise exceptions.NotSupportedError(
                 f'{colorama.Fore.YELLOW}{operation} on cluster '

sky/data/mounting_utils.py

@@ -30,9 +30,17 @@ _BLOBFUSE_CACHE_DIR = ('~/.sky/blobfuse2_cache/'
 # https://github.com/rclone/rclone/releases
 RCLONE_VERSION = 'v1.68.2'
 
+# A wrapper for goofys to choose the logging mechanism based on environment.
+_GOOFYS_WRAPPER = ('$(if [ -S /dev/log ] ; then '
+                   'echo "goofys"; '
+                   'else '
+                   'echo "goofys --log-file $(mktemp -t goofys.XXXX.log)"; '
+                   'fi)')
+
 
 def get_s3_mount_install_cmd() -> str:
     """Returns a command to install S3 mount utility goofys."""
+    # TODO(aylei): maintain our goofys fork under skypilot-org
     install_cmd = ('ARCH=$(uname -m) && '
                    'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
                    '  echo "goofys is not supported on $ARCH" && '
@@ -40,8 +48,8 @@ def get_s3_mount_install_cmd() -> str:
                    'else '
                    '  ARCH_SUFFIX="amd64"; '
                    'fi && '
-                   'sudo wget -nc https://github.com/romilbhardwaj/goofys/'
-                   'releases/download/0.24.0-romilb-upstream/goofys '
+                   'sudo wget -nc https://github.com/aylei/goofys/'
+                   'releases/download/0.24.0-aylei-upstream/goofys '
                    '-O /usr/local/bin/goofys && '
                    'sudo chmod 755 /usr/local/bin/goofys')
     return install_cmd
@@ -56,7 +64,7 @@ def get_s3_mount_cmd(bucket_name: str,
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = ('goofys -o allow_other '
+    mount_cmd = (f'{_GOOFYS_WRAPPER} -o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'{bucket_name}{_bucket_sub_path} {mount_path}')
@@ -73,7 +81,8 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
         _bucket_sub_path = ''
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
-    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} goofys -o allow_other '
+    mount_cmd = (f'AWS_PROFILE={nebius_profile_name} {_GOOFYS_WRAPPER} '
+                 '-o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'--endpoint {endpoint_url} '
@@ -185,14 +194,28 @@ def get_az_mount_cmd(container_name: str,
         bucket_sub_path_arg = ''
     else:
         bucket_sub_path_arg = f'--subdirectory={_bucket_sub_path}/ '
+    mount_options = '-o allow_other -o default_permissions'
     # TODO(zpoint): clear old cache that has been created in the previous boot.
+    blobfuse2_cmd = ('blobfuse2 --no-symlinks -o umask=022 '
+                     f'--tmp-path {cache_path}_$({remote_boot_time_cmd}) '
+                     f'{bucket_sub_path_arg}'
+                     f'--container-name {container_name}')
+    # 1. Set -o nonempty to bypass empty directory check of blobfuse2 when using
+    # fusermount-wrapper, since the mount is delegated to fusermount and
+    # blobfuse2 only get the mounted fd.
+    # 2. {} is the mount point placeholder that will be replaced with the
+    # mounted fd by fusermount-wrapper.
+    wrapped = (f'fusermount-wrapper -m {mount_path} {mount_options} '
+               f'-- {blobfuse2_cmd} -o nonempty {{}}')
+    original = f'{blobfuse2_cmd} {mount_options} {mount_path}'
+    # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
+    # to avoid requiring root privilege.
+    # TODO(aylei): feeling hacky, refactor this.
+    get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
+                     f'echo "{wrapped}" || echo "{original}"')
     mount_cmd = (f'AZURE_STORAGE_ACCOUNT={storage_account_name} '
                  f'{key_env_var} '
-                 f'blobfuse2 {mount_path} --allow-other --no-symlinks '
-                 '-o umask=022 -o default_permissions '
-                 f'--tmp-path {cache_path}_$({remote_boot_time_cmd}) '
-                 f'{bucket_sub_path_arg}'
-                 f'--container-name {container_name}')
+                 f'$({get_mount_cmd})')
     return mount_cmd
 
 
@@ -209,7 +232,8 @@ def get_r2_mount_cmd(r2_credentials_path: str,
     else:
         _bucket_sub_path = f':{_bucket_sub_path}'
     mount_cmd = (f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
-                 f'AWS_PROFILE={r2_profile_name} goofys -o allow_other '
+                 f'AWS_PROFILE={r2_profile_name} {_GOOFYS_WRAPPER} '
+                 '-o allow_other '
                  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
                  f'--type-cache-ttl {_TYPE_CACHE_TTL} '
                  f'--endpoint {endpoint_url} '

sky/exceptions.py

@@ -265,7 +265,7 @@ class CommandError(SkyPilotExcludeArgsBaseException):
                 # Chunk the command to avoid overflow.
                 command = command[:100] + '...'
             message = (f'Command {command} failed with return code '
-                       f'{returncode}.\n{error_msg}')
+                       f'{returncode}.\n{error_msg}\n{detailed_reason}')
         super().__init__(message)
 
 

sky/execution.py

@@ -237,11 +237,12 @@ def _execute(
             if Stage.DOWN in stages:
                 stages.remove(Stage.DOWN)
             if idle_minutes_to_autostop >= 0:
-                requested_features.add(
-                    clouds.CloudImplementationFeatures.AUTO_TERMINATE)
-                if not down:
+                if down:
                     requested_features.add(
-                        clouds.CloudImplementationFeatures.STOP)
+                        clouds.CloudImplementationFeatures.AUTODOWN)
+                else:
+                    requested_features.add(
+                        clouds.CloudImplementationFeatures.AUTOSTOP)
         # NOTE: in general we may not have sufficiently specified info
         # (cloud/resource) to check STOP_SPOT_INSTANCE here. This is checked in
         # the backend.

sky/jobs/client/sdk.py

@@ -82,6 +82,7 @@ def launch(
         f'{server_common.get_server_url()}/jobs/launch',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -138,6 +139,7 @@ def queue(refresh: bool,
         f'{server_common.get_server_url()}/jobs/queue',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response=response)
 
@@ -177,6 +179,7 @@ def cancel(
         f'{server_common.get_server_url()}/jobs/cancel',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response=response)
 
@@ -224,6 +227,7 @@ def tail_logs(name: Optional[str] = None,
         json=json.loads(body.model_dump_json()),
         stream=True,
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     request_id = server_common.get_request_id(response)
     return sdk.stream_response(request_id, response, output_stream)
@@ -267,6 +271,7 @@ def download_logs(
         f'{server_common.get_server_url()}/jobs/download_logs',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     job_id_remote_path_dict = sdk.stream_and_get(
         server_common.get_request_id(response))

sky/optimizer.py

@@ -6,6 +6,7 @@ import typing
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import colorama
+import numpy as np
 import prettytable
 
 from sky import check as sky_check
@@ -28,12 +29,10 @@ from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     import networkx as nx
-    import numpy as np
 
     from sky import dag as dag_lib
 else:
     nx = adaptors_common.LazyImport('networkx')
-    np = adaptors_common.LazyImport('numpy')
 
 logger = sky_logging.init_logger(__name__)
 

sky/provision/instance_setup.py

@@ -44,7 +44,9 @@ _DUMP_RAY_PORTS = (
 
 _RAY_PORT_COMMAND = (
     f'RAY_PORT=$({constants.SKY_PYTHON_CMD} -c '
-    '"from sky.skylet import job_lib; print(job_lib.get_ray_port())" '
+    '"from sky import sky_logging\n'
+    'with sky_logging.silent(): '
+    'from sky.skylet import job_lib; print(job_lib.get_ray_port())" '
     '2> /dev/null || echo 6379);'
     f'{constants.SKY_PYTHON_CMD} -c "from sky.utils import message_utils; '
     'print(message_utils.encode_payload({\'ray_port\': $RAY_PORT}))"')

sky/provision/kubernetes/config.py

@@ -43,7 +43,7 @@ def bootstrap_instances(
     if (requested_service_account ==
             kubernetes_utils.DEFAULT_SERVICE_ACCOUNT_NAME):
         # If the user has requested a different service account (via pod_config
-        # in ~/.sky/config.yaml), we assume they have already set up the
+        # in ~/.sky/skyconfig.yaml), we assume they have already set up the
         # necessary roles and role bindings.
         # If not, set up the roles and bindings for skypilot-service-account
         # here.
@@ -561,69 +561,74 @@ def _configure_skypilot_system_namespace(
 
 
 def _configure_fuse_mounting(provider_config: Dict[str, Any]) -> None:
-    """Creates sidecars required for FUSE mounting.
+    """Creates the privileged daemonset required for FUSE mounting.
 
     FUSE mounting in Kubernetes without privileged containers requires us to
-    run a sidecar container with the necessary capabilities. We run a daemonset
-    which exposes the host /dev/fuse device as a Kubernetes resource. The
-    SkyPilot pod requests this resource to mount the FUSE filesystem.
+    run a privileged daemonset which accepts fusermount requests via unix
+    domain socket and perform the mount/unmount operations on the host /dev/fuse
+    device.
 
     We create this daemonset in the skypilot_system_namespace, which is
-    configurable in the provider config. This allows the FUSE mounting sidecar
-    to be shared across multiple tenants. The default namespace is
+    configurable in the provider config. This allows the daemonset to be
+    shared across multiple tenants. The default namespace is
     'skypilot-system' (populated in clouds.Kubernetes).
+
+    For legacy smarter-device-manager daemonset, we keep it as is since it may
+    still be used by other tenants.
     """
 
-    logger.info('_configure_fuse_mounting: Setting up FUSE device manager.')
+    logger.info(
+        '_configure_fuse_mounting: Setting up fusermount-server daemonset.')
 
-    fuse_device_manager_namespace = provider_config['skypilot_system_namespace']
+    fuse_proxy_namespace = provider_config['skypilot_system_namespace']
     context = kubernetes_utils.get_context_from_config(provider_config)
 
-    # Read the device manager YAMLs from the manifests directory
+    # Read the YAMLs from the manifests directory
     root_dir = os.path.dirname(os.path.dirname(__file__))
 
-    # Load and create the ConfigMap
-    logger.info('_configure_fuse_mounting: Creating configmap.')
-    config_map_path = os.path.join(
-        root_dir, 'kubernetes/manifests/smarter-device-manager-configmap.yaml')
-    with open(config_map_path, 'r', encoding='utf-8') as file:
-        config_map = yaml.safe_load(file)
-    kubernetes_utils.merge_custom_metadata(config_map['metadata'])
-    try:
-        kubernetes.core_api(context).create_namespaced_config_map(
-            fuse_device_manager_namespace, config_map)
-    except kubernetes.api_exception() as e:
-        if e.status == 409:
-            logger.info('_configure_fuse_mounting: ConfigMap already exists '
-                        f'in namespace {fuse_device_manager_namespace!r}')
-        else:
-            raise
-    else:
-        logger.info('_configure_fuse_mounting: ConfigMap created '
-                    f'in namespace {fuse_device_manager_namespace!r}')
-
     # Load and create the DaemonSet
+    # TODO(aylei): support customize and upgrade the fusermount-server image
     logger.info('_configure_fuse_mounting: Creating daemonset.')
     daemonset_path = os.path.join(
-        root_dir, 'kubernetes/manifests/smarter-device-manager-daemonset.yaml')
+        root_dir, 'kubernetes/manifests/fusermount-server-daemonset.yaml')
     with open(daemonset_path, 'r', encoding='utf-8') as file:
         daemonset = yaml.safe_load(file)
     kubernetes_utils.merge_custom_metadata(daemonset['metadata'])
     try:
         kubernetes.apps_api(context).create_namespaced_daemon_set(
-            fuse_device_manager_namespace, daemonset)
+            fuse_proxy_namespace, daemonset)
     except kubernetes.api_exception() as e:
         if e.status == 409:
             logger.info('_configure_fuse_mounting: DaemonSet already exists '
-                        f'in namespace {fuse_device_manager_namespace!r}')
+                        f'in namespace {fuse_proxy_namespace!r}')
+            existing_ds = kubernetes.apps_api(
+                context).read_namespaced_daemon_set(
+                    daemonset['metadata']['name'], fuse_proxy_namespace)
+            ds_image = daemonset['spec']['template']['spec']['containers'][0][
+                'image']
+            if existing_ds.spec.template.spec.containers[0].image != ds_image:
+                logger.info(
+                    '_configure_fuse_mounting: Updating DaemonSet image.')
+                kubernetes.apps_api(context).patch_namespaced_daemon_set(
+                    daemonset['metadata']['name'], fuse_proxy_namespace,
+                    daemonset)
+        elif e.status == 403 or e.status == 401:
+            logger.error('SkyPilot does not have permission to create '
+                         'fusermount-server DaemonSet in namespace '
+                         f'{fuse_proxy_namespace!r}, Error: {e.reason}. '
+                         'Please check the permissions of the SkyPilot service '
+                         'account or contact your cluster admin to create the '
+                         'DaemonSet manually. '
+                         'Reference: https://docs.skypilot.co/reference/kubernetes/kubernetes-setup.html#kubernetes-setup-fuse')  # pylint: disable=line-too-long
+            raise
         else:
             raise
     else:
         logger.info('_configure_fuse_mounting: DaemonSet created '
-                    f'in namespace {fuse_device_manager_namespace!r}')
+                    f'in namespace {fuse_proxy_namespace!r}')
 
-    logger.info('FUSE device manager setup complete '
-                f'in namespace {fuse_device_manager_namespace!r}')
+    logger.info('fusermount-server daemonset setup complete '
+                f'in namespace {fuse_proxy_namespace!r}')
 
 
 def _configure_services(namespace: str, context: Optional[str],

sky/provision/kubernetes/instance.py

@@ -162,12 +162,9 @@ def _raise_pod_scheduling_errors(namespace, context, new_nodes):
                     raise config_lib.KubernetesError(
                         _lack_resource_msg('memory', pod,
                                            details=event_message))
-                if 'Insufficient smarter-devices/fuse' in event_message:
-                    raise config_lib.KubernetesError(
-                        'Something went wrong with FUSE device daemonset.'
-                        ' Try restarting your FUSE pods by running '
-                        '`kubectl delete pods -n skypilot-system -l name=smarter-device-manager`.'  # pylint: disable=line-too-long
-                        f' Full error: {event_message}')
+                # TODO(aylei): after switching from smarter-device-manager to
+                # fusermount-server, we need a new way to check whether the
+                # fusermount-server daemonset is ready.
                 gpu_lf_keys = [
                     key for lf in kubernetes_utils.LABEL_FORMATTER_REGISTRY
                     for key in lf.get_label_keys()
@@ -723,7 +720,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
                        f'{common_utils.format_exception(e)}'
                        'Continuing without using nvidia RuntimeClass.\n'
                        'If you are on a K3s cluster, manually '
-                       'override runtimeClassName in ~/.sky/config.yaml. '
+                       'override runtimeClassName in ~/.sky/skyconfig.yaml. '
                        'For more details, refer to https://docs.skypilot.co/en/latest/reference/config.html')  # pylint: disable=line-too-long
 
     needs_gpus = False

sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fusermount-server
+  labels:
+    app: fusermount-server
+    role: agent
+    parent: skypilot
+spec:
+  selector:
+    matchLabels:
+      app: fusermount-server
+  template:
+    metadata:
+      labels:
+        app: fusermount-server
+    spec:
+      # Add tolerations to run on all nodes
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      containers:
+      - name: server
+        # TODO(aylei): version strategy of our addon images
+        image: berkeleyskypilot/fusermount-server:latest
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: shared-dir
+          mountPath: /var/run/fusermount
+        env:
+        - name: FUSERMOUNT_SHARED_DIR
+          value: /var/run/fusermount
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - "test -S /var/run/fusermount/server.sock"
+          initialDelaySeconds: 10
+          periodSeconds: 5
+          timeoutSeconds: 2
+          failureThreshold: 10
+      volumes:
+      - name: shared-dir
+        hostPath:
+          path: /var/run/fusermount
+          type: DirectoryOrCreate

sky/provision/kubernetes/network_utils.py

@@ -66,7 +66,7 @@ def get_networking_mode(
     except ValueError as e:
         with ux_utils.print_exception_no_traceback():
             raise ValueError(str(e) +
-                             ' Please check: ~/.sky/config.yaml.') from None
+                             ' Please check: ~/.sky/skyconfig.yaml.') from None
     return networking_mode