skypilot-nightly 1.0.0.dev20250411__py3-none-any.whl → 1.0.0.dev20250413__py3-none-any.whl

sky/provision/kubernetes/config.py

@@ -43,7 +43,7 @@ def bootstrap_instances(
     if (requested_service_account ==
             kubernetes_utils.DEFAULT_SERVICE_ACCOUNT_NAME):
         # If the user has requested a different service account (via pod_config
-        # in ~/.sky/config.yaml), we assume they have already set up the
+        # in ~/.sky/skyconfig.yaml), we assume they have already set up the
         # necessary roles and role bindings.
         # If not, set up the roles and bindings for skypilot-service-account
         # here.
@@ -561,69 +561,74 @@ def _configure_skypilot_system_namespace(
 
 
 def _configure_fuse_mounting(provider_config: Dict[str, Any]) -> None:
-    """Creates sidecars required for FUSE mounting.
+    """Creates the privileged daemonset required for FUSE mounting.
 
     FUSE mounting in Kubernetes without privileged containers requires us to
-    run a sidecar container with the necessary capabilities. We run a daemonset
-    which exposes the host /dev/fuse device as a Kubernetes resource. The
-    SkyPilot pod requests this resource to mount the FUSE filesystem.
+    run a privileged daemonset which accepts fusermount requests via unix
+    domain socket and perform the mount/unmount operations on the host /dev/fuse
+    device.
 
     We create this daemonset in the skypilot_system_namespace, which is
-    configurable in the provider config. This allows the FUSE mounting sidecar
-    to be shared across multiple tenants. The default namespace is
+    configurable in the provider config. This allows the daemonset to be
+    shared across multiple tenants. The default namespace is
     'skypilot-system' (populated in clouds.Kubernetes).
+
+    For legacy smarter-device-manager daemonset, we keep it as is since it may
+    still be used by other tenants.
     """
 
-    logger.info('_configure_fuse_mounting: Setting up FUSE device manager.')
+    logger.info(
+        '_configure_fuse_mounting: Setting up fusermount-server daemonset.')
 
-    fuse_device_manager_namespace = provider_config['skypilot_system_namespace']
+    fuse_proxy_namespace = provider_config['skypilot_system_namespace']
     context = kubernetes_utils.get_context_from_config(provider_config)
 
-    # Read the device manager YAMLs from the manifests directory
+    # Read the YAMLs from the manifests directory
     root_dir = os.path.dirname(os.path.dirname(__file__))
 
-    # Load and create the ConfigMap
-    logger.info('_configure_fuse_mounting: Creating configmap.')
-    config_map_path = os.path.join(
-        root_dir, 'kubernetes/manifests/smarter-device-manager-configmap.yaml')
-    with open(config_map_path, 'r', encoding='utf-8') as file:
-        config_map = yaml.safe_load(file)
-    kubernetes_utils.merge_custom_metadata(config_map['metadata'])
-    try:
-        kubernetes.core_api(context).create_namespaced_config_map(
-            fuse_device_manager_namespace, config_map)
-    except kubernetes.api_exception() as e:
-        if e.status == 409:
-            logger.info('_configure_fuse_mounting: ConfigMap already exists '
-                        f'in namespace {fuse_device_manager_namespace!r}')
-        else:
-            raise
-    else:
-        logger.info('_configure_fuse_mounting: ConfigMap created '
-                    f'in namespace {fuse_device_manager_namespace!r}')
-
     # Load and create the DaemonSet
+    # TODO(aylei): support customize and upgrade the fusermount-server image
     logger.info('_configure_fuse_mounting: Creating daemonset.')
     daemonset_path = os.path.join(
-        root_dir, 'kubernetes/manifests/smarter-device-manager-daemonset.yaml')
+        root_dir, 'kubernetes/manifests/fusermount-server-daemonset.yaml')
     with open(daemonset_path, 'r', encoding='utf-8') as file:
         daemonset = yaml.safe_load(file)
     kubernetes_utils.merge_custom_metadata(daemonset['metadata'])
     try:
         kubernetes.apps_api(context).create_namespaced_daemon_set(
-            fuse_device_manager_namespace, daemonset)
+            fuse_proxy_namespace, daemonset)
     except kubernetes.api_exception() as e:
         if e.status == 409:
             logger.info('_configure_fuse_mounting: DaemonSet already exists '
-                        f'in namespace {fuse_device_manager_namespace!r}')
+                        f'in namespace {fuse_proxy_namespace!r}')
+            existing_ds = kubernetes.apps_api(
+                context).read_namespaced_daemon_set(
+                    daemonset['metadata']['name'], fuse_proxy_namespace)
+            ds_image = daemonset['spec']['template']['spec']['containers'][0][
+                'image']
+            if existing_ds.spec.template.spec.containers[0].image != ds_image:
+                logger.info(
+                    '_configure_fuse_mounting: Updating DaemonSet image.')
+                kubernetes.apps_api(context).patch_namespaced_daemon_set(
+                    daemonset['metadata']['name'], fuse_proxy_namespace,
+                    daemonset)
+        elif e.status == 403 or e.status == 401:
+            logger.error('SkyPilot does not have permission to create '
+                         'fusermount-server DaemonSet in namespace '
+                         f'{fuse_proxy_namespace!r}, Error: {e.reason}. '
+                         'Please check the permissions of the SkyPilot service '
+                         'account or contact your cluster admin to create the '
+                         'DaemonSet manually. '
+                         'Reference: https://docs.skypilot.co/reference/kubernetes/kubernetes-setup.html#kubernetes-setup-fuse')  # pylint: disable=line-too-long
+            raise
         else:
             raise
     else:
         logger.info('_configure_fuse_mounting: DaemonSet created '
-                    f'in namespace {fuse_device_manager_namespace!r}')
+                    f'in namespace {fuse_proxy_namespace!r}')
 
-    logger.info('FUSE device manager setup complete '
-                f'in namespace {fuse_device_manager_namespace!r}')
+    logger.info('fusermount-server daemonset setup complete '
+                f'in namespace {fuse_proxy_namespace!r}')
 
 
 def _configure_services(namespace: str, context: Optional[str],

sky/provision/kubernetes/instance.py

@@ -162,12 +162,9 @@ def _raise_pod_scheduling_errors(namespace, context, new_nodes):
                     raise config_lib.KubernetesError(
                         _lack_resource_msg('memory', pod,
                                            details=event_message))
-                if 'Insufficient smarter-devices/fuse' in event_message:
-                    raise config_lib.KubernetesError(
-                        'Something went wrong with FUSE device daemonset.'
-                        ' Try restarting your FUSE pods by running '
-                        '`kubectl delete pods -n skypilot-system -l name=smarter-device-manager`.'  # pylint: disable=line-too-long
-                        f' Full error: {event_message}')
+                # TODO(aylei): after switching from smarter-device-manager to
+                # fusermount-server, we need a new way to check whether the
+                # fusermount-server daemonset is ready.
                 gpu_lf_keys = [
                     key for lf in kubernetes_utils.LABEL_FORMATTER_REGISTRY
                     for key in lf.get_label_keys()
@@ -723,7 +720,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
                        f'{common_utils.format_exception(e)}'
                        'Continuing without using nvidia RuntimeClass.\n'
                        'If you are on a K3s cluster, manually '
-                       'override runtimeClassName in ~/.sky/config.yaml. '
+                       'override runtimeClassName in ~/.sky/skyconfig.yaml. '
                        'For more details, refer to https://docs.skypilot.co/en/latest/reference/config.html')  # pylint: disable=line-too-long
 
     needs_gpus = False

sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fusermount-server
+  labels:
+    app: fusermount-server
+    role: agent
+    parent: skypilot
+spec:
+  selector:
+    matchLabels:
+      app: fusermount-server
+  template:
+    metadata:
+      labels:
+        app: fusermount-server
+    spec:
+      # Add tolerations to run on all nodes
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      containers:
+      - name: server
+        # TODO(aylei): version strategy of our addon images
+        image: berkeleyskypilot/fusermount-server:latest
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: shared-dir
+          mountPath: /var/run/fusermount
+        env:
+        - name: FUSERMOUNT_SHARED_DIR
+          value: /var/run/fusermount
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - "test -S /var/run/fusermount/server.sock"
+          initialDelaySeconds: 10
+          periodSeconds: 5
+          timeoutSeconds: 2
+          failureThreshold: 10
+      volumes:
+      - name: shared-dir
+        hostPath:
+          path: /var/run/fusermount
+          type: DirectoryOrCreate

sky/provision/kubernetes/network_utils.py

@@ -66,7 +66,7 @@ def get_networking_mode(
     except ValueError as e:
         with ux_utils.print_exception_no_traceback():
             raise ValueError(str(e) +
-                             ' Please check: ~/.sky/config.yaml.') from None
+                             ' Please check: ~/.sky/skyconfig.yaml.') from None
     return networking_mode
 
 

sky/provision/kubernetes/utils.py

@@ -1454,14 +1454,14 @@ def is_kubeconfig_exec_auth(
 
 
     Using exec-based authentication is problematic when used in conjunction
-    with kubernetes.remote_identity = LOCAL_CREDENTIAL in ~/.sky/config.yaml.
+    with kubernetes.remote_identity = LOCAL_CREDENTIAL in ~/.sky/skyconfig.yaml.
     This is because the exec-based authentication may not have the relevant
     dependencies installed on the remote cluster or may have hardcoded paths
     that are not available on the remote cluster.
 
     Returns:
         bool: True if exec-based authentication is used and LOCAL_CREDENTIAL
-            mode is used for remote_identity in ~/.sky/config.yaml.
+            mode is used for remote_identity in ~/.sky/skyconfig.yaml.
         str: Error message if exec-based authentication is used, None otherwise
     """
     k8s = kubernetes.kubernetes
@@ -1514,7 +1514,7 @@ def is_kubeconfig_exec_auth(
                     'Managed Jobs or SkyServe controller on Kubernetes. '
                     'To fix, configure SkyPilot to create a service account '
                     'for running pods by setting the following in '
-                    '~/.sky/config.yaml:\n'
+                    '~/.sky/skyconfig.yaml:\n'
                     '    kubernetes:\n'
                     '      remote_identity: SERVICE_ACCOUNT\n'
                     '    More: https://docs.skypilot.co/en/latest/'
@@ -2148,32 +2148,35 @@ def fill_ssh_jump_template(ssh_key_secret: str, ssh_jump_image: str,
     return content
 
 
-def check_port_forward_mode_dependencies() -> None:
-    """Checks if 'socat' and 'nc' are installed"""
+def check_port_forward_mode_dependencies(
+        raise_error: bool = True) -> Optional[List[str]]:
+    """Checks if 'socat' and 'nc' are installed
 
-    # Construct runtime errors
-    socat_default_error = RuntimeError(
-        f'`socat` is required to setup Kubernetes cloud with '
+    Args:
+        raise_error: set to true when the dependencies need to be present.
+            set to false for `sky check`, where reason strings are compiled
+            at the end.
+
+    Returns: the reasons list if there are missing dependencies.
+    """
+
+    # errors
+    socat_message = (
+        '`socat` is required to setup Kubernetes cloud with '
         f'`{kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD.value}` '  # pylint: disable=line-too-long
-        'default networking mode and it is not installed. '
-        'On Debian/Ubuntu, install it with:\n'
-        f'  $ sudo apt install socat\n'
-        f'On MacOS, install it with: \n'
-        f'  $ brew install socat')
-    netcat_default_error = RuntimeError(
-        f'`nc` is required to setup Kubernetes cloud with '
+        'default networking mode and it is not installed. ')
+    netcat_default_message = (
+        '`nc` is required to setup Kubernetes cloud with '
         f'`{kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD.value}` '  # pylint: disable=line-too-long
-        'default networking mode and it is not installed. '
-        'On Debian/Ubuntu, install it with:\n'
-        f'  $ sudo apt install netcat\n'
-        f'On MacOS, install it with: \n'
-        f'  $ brew install netcat')
-    mac_installed_error = RuntimeError(
-        f'The default MacOS `nc` is installed. However, for '
+        'default networking mode and it is not installed. ')
+    netcat_macos_message = (
+        'The default MacOS `nc` is installed. However, for '
         f'`{kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD.value}` '  # pylint: disable=line-too-long
-        'default networking mode, GNU netcat is required. '
-        f'On MacOS, install it with: \n'
-        f'  $ brew install netcat')
+        'default networking mode, GNU netcat is required. ')
+
+    # save
+    reasons = []
+    required_binaries = []
 
     # Ensure socat is installed
     try:
@@ -2182,8 +2185,8 @@ def check_port_forward_mode_dependencies() -> None:
                        stderr=subprocess.DEVNULL,
                        check=True)
     except (FileNotFoundError, subprocess.CalledProcessError):
-        with ux_utils.print_exception_no_traceback():
-            raise socat_default_error from None
+        required_binaries.append('socat')
+        reasons.append(socat_message)
 
     # Ensure netcat is installed
     #
@@ -2198,15 +2201,28 @@ def check_port_forward_mode_dependencies() -> None:
             netcat_output.stderr)
 
         if nc_mac_installed:
-            with ux_utils.print_exception_no_traceback():
-                raise mac_installed_error from None
+            required_binaries.append('netcat')
+            reasons.append(netcat_macos_message)
         elif netcat_output.returncode != 0:
-            with ux_utils.print_exception_no_traceback():
-                raise netcat_default_error from None
+            required_binaries.append('netcat')
+            reasons.append(netcat_default_message)
 
     except FileNotFoundError:
-        with ux_utils.print_exception_no_traceback():
-            raise netcat_default_error from None
+        required_binaries.append('netcat')
+        reasons.append(netcat_default_message)
+
+    if required_binaries:
+        reasons.extend([
+            'On Debian/Ubuntu, install the missing dependenc(ies) with:',
+            f'  $ sudo apt install {" ".join(required_binaries)}',
+            'On MacOS, install with: ',
+            f'  $ brew install {" ".join(required_binaries)}',
+        ])
+        if raise_error:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError('\n'.join(reasons))
+        return reasons
+    return None
 
 
 def get_endpoint_debug_message() -> str:
@@ -2236,7 +2252,7 @@ def combine_pod_config_fields(
     cluster_config_overrides: Dict[str, Any],
 ) -> None:
     """Adds or updates fields in the YAML with fields from the
-    ~/.sky/config.yaml's kubernetes.pod_spec dict.
+    ~/.sky/skyconfig.yaml's kubernetes.pod_spec dict.
     This can be used to add fields to the YAML that are not supported by
     SkyPilot yet, or require simple configuration (e.g., adding an
     imagePullSecrets field).
@@ -2296,7 +2312,7 @@ def combine_pod_config_fields(
 
 def combine_metadata_fields(cluster_yaml_path: str) -> None:
     """Updates the metadata for all Kubernetes objects created by SkyPilot with
-    fields from the ~/.sky/config.yaml's kubernetes.custom_metadata dict.
+    fields from the ~/.sky/skyconfig.yaml's kubernetes.custom_metadata dict.
 
     Obeys the same add or update semantics as combine_pod_config_fields().
     """

sky/server/requests/payloads.py

@@ -47,6 +47,8 @@ def request_body_env_vars() -> dict:
     # Remove the path to config file, as the config content is included in the
     # request body and will be merged with the config on the server side.
     env_vars.pop(skypilot_config.ENV_VAR_SKYPILOT_CONFIG, None)
+    env_vars.pop(skypilot_config.ENV_VAR_USER_CONFIG, None)
+    env_vars.pop(skypilot_config.ENV_VAR_PROJECT_CONFIG, None)
     return env_vars
 
 

sky/setup_files/dependencies.py

@@ -131,7 +131,7 @@ extras_require: Dict[str, List[str]] = {
     'scp': local_ray,
     'oci': ['oci'] + local_ray,
     # Kubernetes 32.0.0 has an authentication bug: https://github.com/kubernetes-client/python/issues/2333 # pylint: disable=line-too-long
-    'kubernetes': ['kubernetes>=20.0.0,!=32.0.0'],
+    'kubernetes': ['kubernetes>=20.0.0,!=32.0.0', 'websockets'],
     'remote': remote,
     # For the container registry auth api. Reference:
     # https://github.com/runpod/runpod-python/releases/tag/1.6.1

sky/skylet/constants.py

@@ -117,7 +117,7 @@ RUNPOD_DOCKER_USERNAME_ENV_VAR = 'SKYPILOT_RUNPOD_DOCKER_USERNAME'
 
 # Commands for disable GPU ECC, which can improve the performance of the GPU
 # for some workloads by 30%. This will only be applied when a user specify
-# `nvidia_gpus.disable_ecc: true` in ~/.sky/config.yaml.
+# `nvidia_gpus.disable_ecc: true` in ~/.sky/skyconfig.yaml.
 # Running this command will reboot the machine, introducing overhead for
 # provisioning the machine.
 # https://portal.nutanix.com/page/documents/kbs/details?targetId=kA00e000000LKjOCAW
@@ -337,7 +337,7 @@ RCLONE_LOG_DIR = '~/.sky/rclone_log'
 RCLONE_CACHE_DIR = '~/.cache/rclone'
 RCLONE_CACHE_REFRESH_INTERVAL = 10
 
-# The keys that can be overridden in the `~/.sky/config.yaml` file. The
+# The keys that can be overridden in the `~/.sky/skyconfig.yaml` file. The
 # overrides are specified in task YAMLs.
 OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
     ('docker', 'run_options'),