skypilot-nightly 1.0.0.dev20250220__py3-none-any.whl → 1.0.0.dev20250221__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '6b2b31d8358f3ff8394a7a33ec49e9985ada230f'
+_SKYPILOT_COMMIT_SHA = 'aa3c387f04fbdd4468751b7d66fcb381bd3449dc'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250220'
+__version__ = '1.0.0.dev20250221'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -143,6 +143,7 @@ RunPod = clouds.RunPod
 Vast = clouds.Vast
 Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
+Nebius = clouds.Nebius
 
 __all__ = [
     '__version__',
@@ -161,6 +162,7 @@ __all__ = [
     'SCP',
     'Vsphere',
     'Fluidstack',
+    'Nebius',
     'Optimizer',
     'OptimizeTarget',
     'backends',

sky/adaptors/nebius.py

@@ -0,0 +1,85 @@
+"""Nebius cloud adaptor."""
+import os
+
+from sky.adaptors import common
+
+NEBIUS_TENANT_ID_FILENAME = 'NEBIUS_TENANT_ID.txt'
+NEBIUS_IAM_TOKEN_FILENAME = 'NEBIUS_IAM_TOKEN.txt'
+NEBIUS_TENANT_ID_PATH = '~/.nebius/' + NEBIUS_TENANT_ID_FILENAME
+NEBIUS_IAM_TOKEN_PATH = '~/.nebius/' + NEBIUS_IAM_TOKEN_FILENAME
+
+MAX_RETRIES_TO_DISK_CREATE = 120
+MAX_RETRIES_TO_INSTANCE_STOP = 120
+MAX_RETRIES_TO_INSTANCE_START = 120
+MAX_RETRIES_TO_INSTANCE_READY = 240
+
+MAX_RETRIES_TO_DISK_DELETE = 120
+MAX_RETRIES_TO_INSTANCE_WAIT = 120  # Maximum number of retries
+
+POLL_INTERVAL = 5
+
+_iam_token = None
+_tenant_id = None
+
+nebius = common.LazyImport(
+    'nebius',
+    import_error_message='Failed to import dependencies for Nebius AI Cloud. '
+    'Try running: pip install "skypilot[nebius]"',
+    # https://github.com/grpc/grpc/issues/37642 to avoid spam in console
+    set_loggers=lambda: os.environ.update({'GRPC_VERBOSITY': 'NONE'}))
+
+
+def request_error():
+    return nebius.aio.service_error.RequestError
+
+
+def compute():
+    # pylint: disable=import-outside-toplevel
+    from nebius.api.nebius.compute import v1 as compute_v1
+    return compute_v1
+
+
+def iam():
+    # pylint: disable=import-outside-toplevel
+    from nebius.api.nebius.iam import v1 as iam_v1
+    return iam_v1
+
+
+def nebius_common():
+    # pylint: disable=import-outside-toplevel
+    from nebius.api.nebius.common import v1 as common_v1
+    return common_v1
+
+
+def vpc():
+    # pylint: disable=import-outside-toplevel
+    from nebius.api.nebius.vpc import v1 as vpc_v1
+    return vpc_v1
+
+
+def get_iam_token():
+    global _iam_token
+    if _iam_token is None:
+        try:
+            with open(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH),
+                      encoding='utf-8') as file:
+                _iam_token = file.read().strip()
+        except FileNotFoundError:
+            return None
+    return _iam_token
+
+
+def get_tenant_id():
+    global _tenant_id
+    if _tenant_id is None:
+        try:
+            with open(os.path.expanduser(NEBIUS_TENANT_ID_PATH),
+                      encoding='utf-8') as file:
+                _tenant_id = file.read().strip()
+        except FileNotFoundError:
+            return None
+    return _tenant_id
+
+
+def sdk():
+    return nebius.sdk.SDK(credentials=get_iam_token())

sky/backends/backend_utils.py

@@ -197,6 +197,9 @@ def _get_yaml_path_from_cluster_name(cluster_name: str,
     return str(output_path)
 
 
+# Add retry for the file mounts optimization, as the underlying cp command may
+# experience transient errors, #4758.
+@common_utils.retry
 def _optimize_file_mounts(yaml_path: str) -> None:
     """Optimize file mounts in the given ray yaml file.
 
@@ -206,6 +209,10 @@ def _optimize_file_mounts(yaml_path: str) -> None:
       - wheel
       - credentials
     Format is {dst: src}.
+
+    Raises:
+        subprocess.CalledProcessError: If the file mounts are failed to be
+            copied.
     """
     yaml_config = common_utils.read_yaml(yaml_path)
 
@@ -863,6 +870,7 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, cluster_config_file: str):
             clouds.Paperspace,
             clouds.Azure,
             clouds.DO,
+            clouds.Nebius,
     )):
         config = auth.configure_ssh_info(config)
     elif isinstance(cloud, clouds.GCP):

sky/backends/cloud_vm_ray_backend.py

@@ -191,7 +191,8 @@ def _get_cluster_config_template(cloud):
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',
         clouds.Vast: 'vast-ray.yml.j2',
-        clouds.Fluidstack: 'fluidstack-ray.yml.j2'
+        clouds.Fluidstack: 'fluidstack-ray.yml.j2',
+        clouds.Nebius: 'nebius-ray.yml.j2'
     }
     return cloud_to_template[type(cloud)]
 
@@ -3233,7 +3234,14 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         all_file_mounts: Optional[Dict[Path, Path]],
         storage_mounts: Optional[Dict[Path, storage_lib.Storage]],
     ) -> None:
-        """Mounts all user files to the remote nodes."""
+        """Mounts all user files to the remote nodes.
+
+        Note: This does not handle COPY storage_mounts. These should have
+        already been translated into file_mounts by task.sync_storage_mounts().
+
+        TODO: Delete COPY storage_mounts in task.sync_storage_mounts(), and
+        assert here that all storage_mounts are MOUNT mode.
+        """
         with rich_utils.safe_status(ux_utils.spinner_message('Syncing files')):
             controller_utils.replace_skypilot_config_path_in_file_mounts(
                 handle.launched_resources.cloud, all_file_mounts)

sky/client/sdk.py

@@ -1503,14 +1503,14 @@ def stream_and_get(
 
 @usage_lib.entrypoint
 @annotations.client_api
-def api_cancel(request_ids: Optional[List[str]] = None,
+def api_cancel(request_ids: Optional[Union[str, List[str]]] = None,
                all_users: bool = False,
                silent: bool = False) -> server_common.RequestId:
     """Aborts a request or all requests.
 
     Args:
-        request_id: The prefix of the request ID of the request to abort.
-        all: Whether to abort all requests.
+        request_ids: The request ID(s) to abort. Can be a single string or a
+            list of strings.
         all_users: Whether to abort all requests from all users.
         silent: Whether to suppress the output.
 
@@ -1528,6 +1528,11 @@ def api_cancel(request_ids: Optional[List[str]] = None,
     user_id = None
     if not all_users:
         user_id = common_utils.get_user_hash()
+
+    # Convert single request ID to list if needed
+    if isinstance(request_ids, str):
+        request_ids = [request_ids]
+
     body = payloads.RequestCancelBody(request_ids=request_ids, user_id=user_id)
     if all_users:
         echo('Cancelling all users\' requests...')

sky/clouds/__init__.py

@@ -20,6 +20,7 @@ from sky.clouds.gcp import GCP
 from sky.clouds.ibm import IBM
 from sky.clouds.kubernetes import Kubernetes
 from sky.clouds.lambda_cloud import Lambda
+from sky.clouds.nebius import Nebius
 from sky.clouds.oci import OCI
 from sky.clouds.paperspace import Paperspace
 from sky.clouds.runpod import RunPod
@@ -49,6 +50,7 @@ __all__ = [
     'ProvisionerVersion',
     'StatusVersion',
     'Fluidstack',
+    'Nebius',
     # Utility functions
     'cloud_in_iterable',
 ]

sky/clouds/nebius.py

@@ -0,0 +1,294 @@
+""" Nebius Cloud. """
+import logging
+import typing
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from sky import clouds
+from sky.adaptors import nebius
+from sky.clouds import service_catalog
+from sky.utils import registry
+from sky.utils import resources_utils
+
+if typing.TYPE_CHECKING:
+    from sky import resources as resources_lib
+
+_CREDENTIAL_FILES = [
+    # credential files for Nebius
+    nebius.NEBIUS_TENANT_ID_FILENAME,
+    nebius.NEBIUS_IAM_TOKEN_FILENAME
+]
+
+
+@registry.CLOUD_REGISTRY.register
+class Nebius(clouds.Cloud):
+    """Nebius GPU Cloud"""
+    _REPR = 'Nebius'
+    _CLOUD_UNSUPPORTED_FEATURES = {
+        clouds.CloudImplementationFeatures.AUTO_TERMINATE:
+            ('Autodown and Autostop not supported. Can\'t delete disk.'),
+        # Autostop functionality can be implemented, but currently,
+        # there is only a single flag for both autostop and autodown.
+        clouds.CloudImplementationFeatures.SPOT_INSTANCE:
+            ('Spot is not supported, as Nebius API does not implement spot.'),
+        clouds.CloudImplementationFeatures.CLONE_DISK_FROM_CLUSTER:
+            (f'Migrating disk is currently not supported on {_REPR}.'),
+        clouds.CloudImplementationFeatures.DOCKER_IMAGE:
+            (f'Docker image is currently not supported on {_REPR}. '
+             'You can try running docker command inside the '
+             '`run` section in task.yaml.'),
+        clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
+            (f'Custom disk tier is currently not supported on {_REPR}.'),
+    }
+    # Nebius maximum instance name length defined as <= 63 as a hostname length
+    # 63 - 8 - 5 = 50 characters since
+    # we add 4 character from UUID to make uniq `-xxxx`
+    # our provisioner adds additional `-worker`.
+    _MAX_CLUSTER_NAME_LEN_LIMIT = 50
+    _regions: List[clouds.Region] = []
+
+    # Using the latest SkyPilot provisioner API to provision and check status.
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
+    @classmethod
+    def _unsupported_features_for_resources(
+        cls, resources: 'resources_lib.Resources'
+    ) -> Dict[clouds.CloudImplementationFeatures, str]:
+        del resources  # unused
+        return cls._CLOUD_UNSUPPORTED_FEATURES
+
+    @classmethod
+    def _max_cluster_name_length(cls) -> Optional[int]:
+        return cls._MAX_CLUSTER_NAME_LEN_LIMIT
+
+    @classmethod
+    def regions_with_offering(cls, instance_type: str,
+                              accelerators: Optional[Dict[str, int]],
+                              use_spot: bool, region: Optional[str],
+                              zone: Optional[str]) -> List[clouds.Region]:
+        assert zone is None, 'Nebius does not support zones.'
+        del accelerators, zone  # unused
+        if use_spot:
+            return []
+        regions = service_catalog.get_region_zones_for_instance_type(
+            instance_type, use_spot, 'nebius')
+
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+        return regions
+
+    @classmethod
+    def get_vcpus_mem_from_instance_type(
+        cls,
+        instance_type: str,
+    ) -> Tuple[Optional[float], Optional[float]]:
+        return service_catalog.get_vcpus_mem_from_instance_type(instance_type,
+                                                                clouds='nebius')
+
+    @classmethod
+    def zones_provision_loop(
+        cls,
+        *,
+        region: str,
+        num_nodes: int,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]] = None,
+        use_spot: bool = False,
+    ) -> Iterator[None]:
+        del num_nodes  # unused
+        regions = cls.regions_with_offering(instance_type,
+                                            accelerators,
+                                            use_spot,
+                                            region=region,
+                                            zone=None)
+        for r in regions:
+            assert r.zones is None, r
+            yield r.zones
+
+    def instance_type_to_hourly_cost(self,
+                                     instance_type: str,
+                                     use_spot: bool,
+                                     region: Optional[str] = None,
+                                     zone: Optional[str] = None) -> float:
+        return service_catalog.get_hourly_cost(instance_type,
+                                               use_spot=use_spot,
+                                               region=region,
+                                               zone=zone,
+                                               clouds='nebius')
+
+    def accelerators_to_hourly_cost(self,
+                                    accelerators: Dict[str, int],
+                                    use_spot: bool,
+                                    region: Optional[str] = None,
+                                    zone: Optional[str] = None) -> float:
+        """Returns the hourly cost of the accelerators, in dollars/hour."""
+        del accelerators, use_spot, region, zone  # unused
+        return 0.0
+
+    def get_egress_cost(self, num_gigabytes: float) -> float:
+        return 0.0
+
+    def __repr__(self):
+        return self._REPR
+
+    def is_same_cloud(self, other: clouds.Cloud) -> bool:
+        # Returns true if the two clouds are the same cloud type.
+        return isinstance(other, Nebius)
+
+    @classmethod
+    def get_default_instance_type(
+            cls,
+            cpus: Optional[str] = None,
+            memory: Optional[str] = None,
+            disk_tier: Optional[resources_utils.DiskTier] = None
+    ) -> Optional[str]:
+        """Returns the default instance type for Nebius."""
+        return service_catalog.get_default_instance_type(cpus=cpus,
+                                                         memory=memory,
+                                                         disk_tier=disk_tier,
+                                                         clouds='nebius')
+
+    @classmethod
+    def get_accelerators_from_instance_type(
+        cls,
+        instance_type: str,
+    ) -> Optional[Dict[str, Union[int, float]]]:
+        return service_catalog.get_accelerators_from_instance_type(
+            instance_type, clouds='nebius')
+
+    @classmethod
+    def get_zone_shell_cmd(cls) -> Optional[str]:
+        return None
+
+    def make_deploy_resources_variables(
+            self,
+            resources: 'resources_lib.Resources',
+            cluster_name: resources_utils.ClusterName,
+            region: 'clouds.Region',
+            zones: Optional[List['clouds.Zone']],
+            num_nodes: int,
+            dryrun: bool = False) -> Dict[str, Optional[str]]:
+        del dryrun, cluster_name
+        assert zones is None, ('Nebius does not support zones', zones)
+
+        r = resources
+        acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
+        custom_resources = resources_utils.make_ray_custom_resources_str(
+            acc_dict)
+        platform, _ = resources.instance_type.split('_')
+
+        if platform in ('cpu-d3', 'cpu-e2'):
+            image_family = 'ubuntu22.04-driverless'
+        elif platform in ('gpu-h100-sxm', 'gpu-h200-sxm', 'gpu-l40s-a'):
+            image_family = 'ubuntu22.04-cuda12'
+        else:
+            raise RuntimeError('Unsupported instance type for Nebius cloud:'
+                               f' {resources.instance_type}')
+        return {
+            'instance_type': resources.instance_type,
+            'custom_resources': custom_resources,
+            'region': region.name,
+            'image_id': image_family,
+            # Nebius does not support specific zones.
+            'zones': None,
+        }
+
+    def _get_feasible_launchable_resources(
+        self, resources: 'resources_lib.Resources'
+    ) -> 'resources_utils.FeasibleResources':
+        """Returns a list of feasible resources for the given resources."""
+        if resources.instance_type is not None:
+            assert resources.is_launchable(), resources
+            resources = resources.copy(accelerators=None)
+            return resources_utils.FeasibleResources([resources], [], None)
+
+        def _make(instance_list):
+            resource_list = []
+            for instance_type in instance_list:
+                r = resources.copy(
+                    cloud=Nebius(),
+                    instance_type=instance_type,
+                    accelerators=None,
+                    cpus=None,
+                )
+                resource_list.append(r)
+            return resource_list
+
+        # Currently, handle a filter on accelerators only.
+        accelerators = resources.accelerators
+        if accelerators is None:
+            # Return a default instance type
+            default_instance_type = Nebius.get_default_instance_type(
+                cpus=resources.cpus,
+                memory=resources.memory,
+                disk_tier=resources.disk_tier)
+            if default_instance_type is None:
+                # TODO: Add hints to all return values in this method to help
+                #  users understand why the resources are not launchable.
+                return resources_utils.FeasibleResources([], [], None)
+            else:
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
+
+        assert len(accelerators) == 1, resources
+        acc, acc_count = list(accelerators.items())[0]
+        (instance_list, fuzzy_candidate_list
+        ) = service_catalog.get_instance_type_for_accelerator(
+            acc,
+            acc_count,
+            use_spot=resources.use_spot,
+            cpus=resources.cpus,
+            region=resources.region,
+            zone=resources.zone,
+            clouds='nebius')
+        if instance_list is None:
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
+
+    @classmethod
+    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """ Verify that the user has valid credentials for Nebius. """
+        logging.debug('Nebius cloud check credentials')
+        token = nebius.get_iam_token()
+        token_msg = ('    Credentials can be set up by running: \n'\
+                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n')  # pylint: disable=line-too-long
+        tenant_msg = ('   Copy your tenat ID from the web console and save it to file \n'  # pylint: disable=line-too-long
+                      f'        $ nebius --format json iam whoami|jq -r \'.user_profile.tenants[0].tenant_id\' > {nebius.NEBIUS_TENANT_ID_PATH} \n')  # pylint: disable=line-too-long
+        if token is None:
+            return False, f'{token_msg}'
+        sdk = nebius.sdk()
+        tenant_id = nebius.get_tenant_id()
+        if tenant_id is None:
+            return False, f'{tenant_msg}'
+        try:
+            service = nebius.iam().ProjectServiceClient(sdk)
+            service.list(
+                nebius.iam().ListProjectsRequest(parent_id=tenant_id)).wait()
+        except nebius.request_error() as e:
+            return False, (
+                f'{e.status} \n'  # First line is indented by 4 spaces
+                f'{token_msg}'
+                f'{tenant_msg}')
+        return True, None
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        return {
+            f'~/.nebius/{filename}': f'~/.nebius/{filename}'
+            for filename in _CREDENTIAL_FILES
+        }
+
+    @classmethod
+    def get_current_user_identity(cls) -> Optional[List[str]]:
+        # NOTE: used for very advanced SkyPilot functionality
+        # Can implement later if desired
+        return None
+
+    def instance_type_exists(self, instance_type: str) -> bool:
+        return service_catalog.instance_type_exists(instance_type, 'nebius')
+
+    def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
+        return service_catalog.validate_region_zone(region,
+                                                    zone,
+                                                    clouds='nebius')

sky/clouds/service_catalog/constants.py

@@ -4,4 +4,4 @@ CATALOG_SCHEMA_VERSION = 'v6'
 CATALOG_DIR = '~/.sky/catalogs'
 ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vast', 'vsphere', 'cudo', 'fluidstack',
-              'paperspace', 'do')
+              'paperspace', 'do', 'nebius')

sky/clouds/service_catalog/nebius_catalog.py

@@ -0,0 +1,116 @@
+"""Nebius Catalog.
+
+This module loads the service catalog file and can be used to query
+instance types and pricing information for Nebius.
+"""
+import typing
+from typing import Dict, List, Optional, Tuple, Union
+
+from sky.clouds.service_catalog import common
+from sky.utils import resources_utils
+from sky.utils import ux_utils
+
+if typing.TYPE_CHECKING:
+    from sky.clouds import cloud
+
+# Keep it synced with the frequency in
+# skypilot-catalog/.github/workflows/update-Nebius-catalog.yml
+_PULL_FREQUENCY_HOURS = 7
+
+_df = common.read_catalog('nebius/vms.csv')
+
+
+def instance_type_exists(instance_type: str) -> bool:
+    return common.instance_type_exists_impl(_df, instance_type)
+
+
+def validate_region_zone(
+        region: Optional[str],
+        zone: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('Nebius does not support zones.')
+    return common.validate_region_zone_impl('nebius', _df, region, zone)
+
+
+def get_hourly_cost(instance_type: str,
+                    use_spot: bool = False,
+                    region: Optional[str] = None,
+                    zone: Optional[str] = None) -> float:
+    """Returns the cost, or the cheapest cost among all zones for spot."""
+    assert not use_spot, 'Nebius does not support spot.'
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('Nebius does not support zones.')
+    return common.get_hourly_cost_impl(_df, instance_type, use_spot, region,
+                                       zone)
+
+
+def get_vcpus_mem_from_instance_type(
+        instance_type: str) -> Tuple[Optional[float], Optional[float]]:
+    return common.get_vcpus_mem_from_instance_type_impl(_df, instance_type)
+
+
+def get_default_instance_type(
+        cpus: Optional[str] = None,
+        memory: Optional[str] = None,
+        disk_tier: Optional[resources_utils.DiskTier] = None) -> Optional[str]:
+    del disk_tier  # unused
+    return common.get_instance_type_for_cpus_mem_impl(_df, cpus, memory)
+
+
+def get_accelerators_from_instance_type(
+        instance_type: str) -> Optional[Dict[str, Union[int, float]]]:
+    return common.get_accelerators_from_instance_type_impl(_df, instance_type)
+
+
+def get_instance_type_for_accelerator(
+        acc_name: str,
+        acc_count: int,
+        cpus: Optional[str] = None,
+        memory: Optional[str] = None,
+        use_spot: bool = False,
+        region: Optional[str] = None,
+        zone: Optional[str] = None) -> Tuple[Optional[List[str]], List[str]]:
+    """Filter the instance types based on resource requirements.
+
+    Returns a list of instance types satisfying the required count of
+    accelerators with sorted prices and a list of candidates with fuzzy search.
+    """
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('Nebius does not support zones.')
+    return common.get_instance_type_for_accelerator_impl(df=_df,
+                                                         acc_name=acc_name,
+                                                         acc_count=acc_count,
+                                                         cpus=cpus,
+                                                         memory=memory,
+                                                         use_spot=use_spot,
+                                                         region=region,
+                                                         zone=zone)
+
+
+def regions() -> List['cloud.Region']:
+    return common.get_region_zones(_df, use_spot=False)
+
+
+def get_region_zones_for_instance_type(instance_type: str,
+                                       use_spot: bool) -> List['cloud.Region']:
+    df = _df[_df['InstanceType'] == instance_type]
+    return common.get_region_zones(df, use_spot)
+
+
+def list_accelerators(
+        gpus_only: bool,
+        name_filter: Optional[str],
+        region_filter: Optional[str],
+        quantity_filter: Optional[int],
+        case_sensitive: bool = True,
+        all_regions: bool = False,
+        require_price: bool = True) -> Dict[str, List[common.InstanceTypeInfo]]:
+    """Returns all instance types in Nebius offering GPUs."""
+
+    del require_price  # Unused.
+    return common.list_accelerators_impl('nebius', _df, gpus_only, name_filter,
+                                         region_filter, quantity_filter,
+                                         case_sensitive, all_regions)

sky/jobs/controller.py

@@ -6,6 +6,7 @@ import argparse
 import multiprocessing
 import os
 import pathlib
+import shutil
 import time
 import traceback
 import typing
@@ -17,6 +18,7 @@ from sky import exceptions
 from sky import sky_logging
 from sky.backends import backend_utils
 from sky.backends import cloud_vm_ray_backend
+from sky.data import data_utils
 from sky.jobs import recovery_strategy
 from sky.jobs import scheduler
 from sky.jobs import state as managed_job_state
@@ -488,6 +490,7 @@ def _cleanup(job_id: int, dag_yaml: str):
         cluster_name = managed_job_utils.generate_managed_job_cluster_name(
             task.name, job_id)
         managed_job_utils.terminate_cluster(cluster_name)
+
         # Clean up Storages with persistent=False.
         # TODO(zhwu): this assumes the specific backend.
         backend = cloud_vm_ray_backend.CloudVmRayBackend()
@@ -499,6 +502,20 @@ def _cleanup(job_id: int, dag_yaml: str):
             storage.construct()
         backend.teardown_ephemeral_storage(task)
 
+        # Clean up any files mounted from the local disk, such as two-hop file
+        # mounts.
+        for file_mount in (task.file_mounts or {}).values():
+            try:
+                if not data_utils.is_cloud_store_url(file_mount):
+                    path = os.path.expanduser(file_mount)
+                    if os.path.isdir(path):
+                        shutil.rmtree(path)
+                    else:
+                        os.remove(path)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(
+                    f'Failed to clean up file mount {file_mount}: {e}')
+
 
 def start(job_id, dag_yaml):
     """Start the controller."""