skypilot-nightly 1.0.0.dev20250319__py3-none-any.whl → 1.0.0.dev20250321__py3-none-any.whl

sky/data/storage_utils.py

@@ -24,6 +24,9 @@ _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG = (
     'to the cloud storage for {path!r}'
     'due to the following error: {error_msg!r}')
 
+_USE_SKYIGNORE_HINT = (
+    'To avoid using .gitignore, you can create a .skyignore file instead.')
+
 _LAST_USE_TRUNC_LENGTH = 25
 
 
@@ -109,10 +112,9 @@ def get_excluded_files_from_skyignore(src_dir_path: str) -> List[str]:
 def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
     """ Lists files and patterns ignored by git in the source directory
 
-    Runs `git status --ignored` which returns a list of excluded files and
+    Runs `git ls-files --ignored ...` which returns a list of excluded files and
     patterns read from .gitignore and .git/info/exclude using git.
-    `git init` is run if SRC_DIR_PATH is not a git repository and removed
-    after obtaining excluded list.
+    This will also be run for all submodules under the src_dir_path.
 
     Returns:
         List[str] containing files and patterns to be ignored.  Some of the
@@ -120,91 +122,107 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
     """
     expand_src_dir_path = os.path.expanduser(src_dir_path)
 
-    git_exclude_path = os.path.join(expand_src_dir_path, '.git/info/exclude')
-    gitignore_path = os.path.join(expand_src_dir_path,
-                                  constants.GIT_IGNORE_FILE)
+    # We will use `git ls-files` to list files that we should ignore, but
+    # `ls-files` will not recurse into subdirectories. So, we need to manually
+    # list the submodules and run `ls-files` within the root and each submodule.
+    # Print the submodule paths relative to expand_src_dir_path, separated by
+    # null chars.
+    submodules_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
+                      'submodule foreach -q "printf \\$displaypath\\\\\\0"')
+
+    try:
+        submodules_output = subprocess.run(submodules_cmd,
+                                           shell=True,
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.PIPE,
+                                           check=True,
+                                           text=True)
+    except subprocess.CalledProcessError as e:
+        gitignore_path = os.path.join(expand_src_dir_path,
+                                      constants.GIT_IGNORE_FILE)
+
+        if (e.returncode == exceptions.GIT_FATAL_EXIT_CODE and
+                'not a git repository' in e.stderr):
+            # If git failed because we aren't in a git repository, but there is
+            # a .gitignore, warn the user that it will be ignored.
+            if os.path.exists(gitignore_path):
+                logger.warning('Detected a .gitignore file, but '
+                               f'{src_dir_path} is not a git repository. The '
+                               '.gitignore file will be ignored. '
+                               f'{_USE_SKYIGNORE_HINT}')
+            # Otherwise, this is fine and we can exit early.
+            return []
+
+        if e.returncode == exceptions.COMMAND_NOT_FOUND_EXIT_CODE:
+            # Git is not installed. This is fine, skip the check.
+            # If .gitignore is present, warn the user.
+            if os.path.exists(gitignore_path):
+                logger.warning(f'Detected a .gitignore file in {src_dir_path}, '
+                               'but git is not installed. The .gitignore file '
+                               f'will be ignored. {_USE_SKYIGNORE_HINT}')
+            return []
+
+        # Pretty much any other error is unexpected, so re-raise.
+        raise
 
-    git_exclude_exists = os.path.isfile(git_exclude_path)
-    gitignore_exists = os.path.isfile(gitignore_path)
+    # submodules_output will contain each submodule path (relative to
+    # src_dir_path), each ending with a null character.
+    # .split will have an empty string at the end because of the final null
+    # char, so trim it.
+    submodules = submodules_output.stdout.split('\0')[:-1]
+
+    # The empty string is the relative reference to the src_dir_path.
+    all_git_repos = ['.'] + [
+        # We only care about submodules that are a subdirectory of src_dir_path.
+        submodule for submodule in submodules if not submodule.startswith('../')
+    ]
 
-    # This command outputs a list to be excluded according to .gitignore
-    # and .git/info/exclude
-    filter_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
-                  'status --ignored --porcelain=v1')
     excluded_list: List[str] = []
+    for repo in all_git_repos:
+        # repo is the path relative to src_dir_path. Get the full path.
+        repo_path = os.path.join(expand_src_dir_path, repo)
+        # This command outputs a list to be excluded according to .gitignore,
+        # .git/info/exclude, and global exclude config.
+        # -z: filenames terminated by \0 instead of \n
+        # --others: show untracked files
+        # --ignore: out of untracked files, only show ignored files
+        # --exclude-standard: use standard exclude rules (required for --ignore)
+        # --directory: if an entire directory is ignored, collapse to a single
+        #              entry rather than listing every single file
+        # Since we are using --others instead of --cached, this will not show
+        # files that are tracked but also present in .gitignore.
+        filter_cmd = (f'git -C {shlex.quote(repo_path)} ls-files -z '
+                      '--others --ignore --exclude-standard --directory')
+        output = subprocess.run(filter_cmd,
+                                shell=True,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                check=True,
+                                text=True)
+        # Don't catch any errors. We would only expect to see errors during the
+        # first git invocation - so if we see any here, crash.
+
+        output_list = output.stdout.split('\0')
+        # trim the empty string at the end
+        output_list = output_list[:-1]
+
+        for item in output_list:
+
+            if repo == '.' and item == './':
+                logger.warning(f'{src_dir_path} is within a git repo, but the '
+                               'entire directory is ignored by git. We will '
+                               'ignore all git exclusions. '
+                               f'{_USE_SKYIGNORE_HINT}')
+                return []
+
+            to_be_excluded = os.path.join(repo, item)
+            if item.endswith('/'):
+                # aws s3 sync and gsutil rsync require * to exclude
+                # files/dirs under the specified directory.
+                to_be_excluded += '*'
+
+            excluded_list.append(to_be_excluded)
 
-    if git_exclude_exists or gitignore_exists:
-        try:
-            output = subprocess.run(filter_cmd,
-                                    shell=True,
-                                    stdout=subprocess.PIPE,
-                                    stderr=subprocess.PIPE,
-                                    check=True,
-                                    text=True)
-        except subprocess.CalledProcessError as e:
-            # when the SRC_DIR_PATH is not a git repo and .git
-            # does not exist in it
-            if e.returncode == exceptions.GIT_FATAL_EXIT_CODE:
-                if 'not a git repository' in e.stderr:
-                    # Check if the user has 'write' permission to
-                    # SRC_DIR_PATH
-                    if not os.access(expand_src_dir_path, os.W_OK):
-                        error_msg = 'Write permission denial'
-                        logger.warning(
-                            _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
-                                path=src_dir_path, error_msg=error_msg))
-                        return excluded_list
-                    init_cmd = f'git -C {expand_src_dir_path} init'
-                    try:
-                        subprocess.run(init_cmd,
-                                       shell=True,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE,
-                                       check=True)
-                        output = subprocess.run(filter_cmd,
-                                                shell=True,
-                                                stdout=subprocess.PIPE,
-                                                stderr=subprocess.PIPE,
-                                                check=True,
-                                                text=True)
-                    except subprocess.CalledProcessError as init_e:
-                        logger.warning(
-                            _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
-                                path=src_dir_path, error_msg=init_e.stderr))
-                        return excluded_list
-                    if git_exclude_exists:
-                        # removes all the files/dirs created with 'git init'
-                        # under .git/ except .git/info/exclude
-                        remove_files_cmd = (f'find {expand_src_dir_path}' \
-                                            f'/.git -path {git_exclude_path}' \
-                                            ' -prune -o -type f -exec rm -f ' \
-                                            '{} +')
-                        remove_dirs_cmd = (f'find {expand_src_dir_path}' \
-                                        f'/.git -path {git_exclude_path}' \
-                                        ' -o -type d -empty -delete')
-                        subprocess.run(remove_files_cmd,
-                                       shell=True,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE,
-                                       check=True)
-                        subprocess.run(remove_dirs_cmd,
-                                       shell=True,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE,
-                                       check=True)
-
-        output_list = output.stdout.split('\n')
-        for line in output_list:
-            # FILTER_CMD outputs items preceded by '!!'
-            # to specify excluded files/dirs
-            # e.g., '!! mydir/' or '!! mydir/myfile.txt'
-            if line.startswith('!!'):
-                to_be_excluded = line[3:]
-                if line.endswith('/'):
-                    # aws s3 sync and gsutil rsync require * to exclude
-                    # files/dirs under the specified directory.
-                    to_be_excluded += '*'
-                excluded_list.append(to_be_excluded)
     return excluded_list
 
 

sky/exceptions.py

@@ -24,6 +24,8 @@ MOUNT_PATH_NON_EMPTY_CODE = 42
 INSUFFICIENT_PRIVILEGES_CODE = 52
 # Return code when git command is ran in a dir that is not git repo
 GIT_FATAL_EXIT_CODE = 128
+# Return code from bash when a command is not found
+COMMAND_NOT_FOUND_EXIT_CODE = 127
 # Architecture, such as arm64, not supported by the dependency
 ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 

sky/global_user_state.py

@@ -26,11 +26,12 @@ from sky.utils import status_lib
 if typing.TYPE_CHECKING:
     from sky import backends
     from sky import clouds
+    from sky.clouds import cloud
     from sky.data import Storage
 
 logger = sky_logging.init_logger(__name__)
 
-_ENABLED_CLOUDS_KEY = 'enabled_clouds'
+_ENABLED_CLOUDS_KEY_PREFIX = 'enabled_clouds_'
 
 _DB_PATH = os.path.expanduser('~/.sky/state.db')
 pathlib.Path(_DB_PATH).parents[0].mkdir(parents=True, exist_ok=True)
@@ -795,9 +796,11 @@ def get_cluster_names_start_with(starts_with: str) -> List[str]:
     return [row[0] for row in rows]
 
 
-def get_cached_enabled_clouds() -> List['clouds.Cloud']:
+def get_cached_enabled_clouds(
+        cloud_capability: 'cloud.CloudCapability') -> List['clouds.Cloud']:
+
     rows = _DB.cursor.execute('SELECT value FROM config WHERE key = ?',
-                              (_ENABLED_CLOUDS_KEY,))
+                              (_get_capability_key(cloud_capability),))
     ret = []
     for (value,) in rows:
         ret = json.loads(value)
@@ -817,12 +820,18 @@ def get_cached_enabled_clouds() -> List['clouds.Cloud']:
     return enabled_clouds
 
 
-def set_enabled_clouds(enabled_clouds: List[str]) -> None:
-    _DB.cursor.execute('INSERT OR REPLACE INTO config VALUES (?, ?)',
-                       (_ENABLED_CLOUDS_KEY, json.dumps(enabled_clouds)))
+def set_enabled_clouds(enabled_clouds: List[str],
+                       cloud_capability: 'cloud.CloudCapability') -> None:
+    _DB.cursor.execute(
+        'INSERT OR REPLACE INTO config VALUES (?, ?)',
+        (_get_capability_key(cloud_capability), json.dumps(enabled_clouds)))
     _DB.conn.commit()
 
 
+def _get_capability_key(cloud_capability: 'cloud.CloudCapability') -> str:
+    return _ENABLED_CLOUDS_KEY_PREFIX + cloud_capability.value
+
+
 def add_or_update_storage(storage_name: str,
                           storage_handle: 'Storage.StorageMetadata',
                           storage_status: status_lib.StorageStatus):

sky/jobs/server/core.py

@@ -105,7 +105,7 @@ def launch(
 
         local_to_controller_file_mounts = {}
 
-        if storage_lib.get_cached_enabled_storage_clouds_or_refresh():
+        if storage_lib.get_cached_enabled_storage_cloud_names_or_refresh():
             for task_ in dag.tasks:
                 controller_utils.maybe_translate_local_file_mounts_and_sync_up(
                     task_, task_type='jobs')

sky/jobs/utils.py

@@ -125,6 +125,11 @@ def get_job_status(backend: 'backends.CloudVmRayBackend',
     FAILED_SETUP or CANCELLED.
     """
     handle = global_user_state.get_handle_from_cluster_name(cluster_name)
+    if handle is None:
+        # This can happen if the cluster was preempted and background status
+        # refresh already noticed and cleaned it up.
+        logger.info(f'Cluster {cluster_name} not found.')
+        return None
     assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
     status = None
     try:

sky/optimizer.py

@@ -16,6 +16,7 @@ from sky import resources as resources_lib
 from sky import sky_logging
 from sky import task as task_lib
 from sky.adaptors import common as adaptors_common
+from sky.clouds import cloud as sky_cloud
 from sky.usage import usage_lib
 from sky.utils import common
 from sky.utils import env_options
@@ -368,7 +369,8 @@ class Optimizer:
                 # mention "kubernetes cluster" and/instead of "catalog"
                 # in the error message.
                 enabled_clouds = (
-                    sky_check.get_cached_enabled_clouds_or_refresh())
+                    sky_check.get_cached_enabled_clouds_or_refresh(
+                        sky_cloud.CloudCapability.COMPUTE))
                 if clouds.cloud_in_iterable(clouds.Kubernetes(),
                                             enabled_clouds):
                     if any(orig_resources.cloud is None
@@ -1206,6 +1208,7 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
         dag: The DAG specified by a user.
     """
     enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+        capability=sky_cloud.CloudCapability.COMPUTE,
         raise_if_no_cloud_access=True)
 
     global_disabled_clouds: Set[str] = set()
@@ -1225,8 +1228,10 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
         # Explicitly check again to update the enabled cloud list.
         sky_check.check(quiet=True,
                         clouds=list(clouds_need_recheck -
-                                    global_disabled_clouds))
+                                    global_disabled_clouds),
+                        capability=sky_cloud.CloudCapability.COMPUTE)
         enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+            capability=sky_cloud.CloudCapability.COMPUTE,
             raise_if_no_cloud_access=True)
         disabled_clouds = (clouds_need_recheck -
                            {str(c) for c in enabled_clouds})
@@ -1268,6 +1273,7 @@ def _fill_in_launchable_resources(
         a cloud that is not enabled.
     """
     enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+        capability=sky_cloud.CloudCapability.COMPUTE,
         raise_if_no_cloud_access=True)
 
     launchable: Dict[resources_lib.Resources, List[resources_lib.Resources]] = (

sky/provision/gcp/config.py

@@ -297,8 +297,8 @@ def _is_permission_satisfied(service_account, crm, iam, required_permissions,
 def _configure_iam_role(config: common.ProvisionConfig, crm, iam) -> dict:
     """Setup a gcp service account with IAM roles.
 
-    Creates a gcp service acconut and binds IAM roles which allow it to control
-    control storage/compute services. Specifically, the head node needs to have
+    Creates a gcp service account and binds IAM roles which allow it to control
+    storage/compute services. Specifically, the head node needs to have
     an IAM role that allows it to create further gce instances and store items
     in google cloud storage.
 
@@ -311,7 +311,7 @@ def _configure_iam_role(config: common.ProvisionConfig, crm, iam) -> dict:
     )
     service_account = _get_service_account(email, project_id, iam)
 
-    permissions = gcp_utils.get_minimal_permissions()
+    permissions = gcp_utils.get_minimal_compute_permissions()
     roles = constants.DEFAULT_SERVICE_ACCOUNT_ROLES
     if config.provider_config.get(constants.HAS_TPU_PROVIDER_FIELD, False):
         roles = (constants.DEFAULT_SERVICE_ACCOUNT_ROLES +

sky/provision/gcp/constants.py

@@ -141,6 +141,11 @@ FIREWALL_RULES_TEMPLATE = [
     },
 ]
 
+GCP_MINIMAL_PERMISSIONS = [
+    'serviceusage.services.enable',
+    'serviceusage.services.list',
+]
+
 # A list of permissions required to run SkyPilot on GCP.
 # Keep this in sync with https://docs.skypilot.co/en/latest/cloud-setup/cloud-permissions/gcp.html # pylint: disable=line-too-long
 VM_MINIMAL_PERMISSIONS = [
@@ -170,13 +175,22 @@ VM_MINIMAL_PERMISSIONS = [
     # Check: sky.provision.gcp.config::_is_permission_satisfied
     # 'iam.serviceAccounts.actAs',
     'iam.serviceAccounts.get',
-    'serviceusage.services.enable',
-    'serviceusage.services.list',
     'serviceusage.services.use',
     'resourcemanager.projects.get',
     'resourcemanager.projects.getIamPolicy',
 ]
 
+STORAGE_MINIMAL_PERMISSIONS = [
+    'storage.buckets.create',
+    'storage.buckets.get',
+    'storage.buckets.delete',
+    'storage.objects.create',
+    'storage.objects.update',
+    'storage.objects.delete',
+    'storage.objects.get',
+    'storage.objects.list',
+]
+
 # Permissions implied by GCP built-in roles. We hardcode these here, as we
 # cannot get the permissions of built-in role from the GCP Python API.
 # The lists are not exhaustive, but should cover the permissions listed in

sky/provision/gcp/instance.py

@@ -586,8 +586,11 @@ def open_ports(
     }
     handlers: List[Type[instance_utils.GCPInstance]] = [
         instance_utils.GCPComputeInstance,
-        instance_utils.GCPTPUVMInstance,
     ]
+    use_tpu_vms = provider_config.get('_has_tpus', False)
+    if use_tpu_vms:
+        handlers.append(instance_utils.GCPTPUVMInstance)
+
     handler_to_instances = _filter_instances(handlers, project_id, zone,
                                              label_filters, lambda _: None)
     operations = collections.defaultdict(list)

sky/provision/kubernetes/utils.py

@@ -663,18 +663,25 @@ class GKEAutoscaler(Autoscaler):
 
         # Check if any node pool with autoscaling enabled can
         # fit the instance type.
-        for node_pool in cluster['nodePools']:
-            logger.debug(f'checking if node pool {node_pool["name"]} '
+        node_pools = cluster.get('nodePools', [])
+        for node_pool in node_pools:
+            name = node_pool.get('name', '')
+            logger.debug(f'checking if node pool {name} '
                          'has autoscaling enabled.')
-            if (node_pool['autoscaling'] is not None and
-                    'enabled' in node_pool['autoscaling'] and
-                    node_pool['autoscaling']['enabled']):
-                logger.debug(
-                    f'node pool {node_pool["name"]} has autoscaling enabled. '
-                    'Checking if it can create a node '
-                    f'satisfying {instance_type}')
-                if cls._check_instance_fits_gke_autoscaler_node_pool(
-                        instance_type, node_pool):
+            autoscaling_enabled = (node_pool.get('autoscaling',
+                                                 {}).get('enabled', False))
+            if autoscaling_enabled:
+                logger.debug(f'node pool {name} has autoscaling enabled. '
+                             'Checking if it can create a node '
+                             f'satisfying {instance_type}')
+                try:
+                    if cls._check_instance_fits_gke_autoscaler_node_pool(
+                            instance_type, node_pool):
+                        return True
+                except KeyError:
+                    logger.debug('encountered KeyError while checking if '
+                                 f'node pool {name} can create a node '
+                                 f'satisfying {instance_type}.')
                     return True
         return False
 
@@ -776,9 +783,9 @@ class GKEAutoscaler(Autoscaler):
         to fit the instance type.
         """
         for accelerator in node_pool_accelerators:
-            node_accelerator_type = GKELabelFormatter. \
-                get_accelerator_from_label_value(
-                    accelerator['acceleratorType'])
+            node_accelerator_type = (
+                GKELabelFormatter.get_accelerator_from_label_value(
+                    accelerator['acceleratorType']))
             node_accelerator_count = accelerator['acceleratorCount']
             if node_accelerator_type == requested_gpu_type and int(
                     node_accelerator_count) >= requested_gpu_count:
@@ -812,24 +819,22 @@ class GKEAutoscaler(Autoscaler):
     @classmethod
     def _tpu_chip_count_from_instance_type(cls, machine_type: str) -> int:
         """Infer the number of TPU chips from the instance type."""
-        machine_type_parts = machine_type.split('-')
         # according to
         # https://cloud.google.com/kubernetes-engine/docs/concepts/tpus#machine_type
         # GKE TPU machine types have the format of
         # ct<version>-<type>-<node-chip-count>t
         logger.debug(
             f'inferring TPU chip count from machine type: {machine_type}')
-        if (len(machine_type_parts) != 3 or
-                not machine_type_parts[0].startswith('ct') or
-                not machine_type_parts[2].endswith('t') or
-                not machine_type_parts[2].strip('t').isdigit()):
+        pattern = r'ct[a-z0-9]+-[a-z]+-([0-9]+)t'
+        search = re.search(pattern, machine_type)
+        if search is None:
             logger.debug(f'machine type {machine_type} is not a '
                          'valid TPU machine type format.')
             return 0
-        num_tpu_chips = int(machine_type_parts[2].strip('t'))
+        num_tpu_chips = search.group(1)
         logger.debug(
             f'machine type {machine_type} has {num_tpu_chips} TPU chips.')
-        return num_tpu_chips
+        return int(num_tpu_chips)
 
     @classmethod
     def _is_node_multi_host_tpu(cls, resource_labels: dict) -> bool:

sky/resources.py

@@ -10,6 +10,7 @@ from sky import clouds
 from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
+from sky.clouds import cloud as sky_cloud
 from sky.clouds import service_catalog
 from sky.provision import docker_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
@@ -677,6 +678,7 @@ class Resources:
             # cloud corresponds to region/zone, errors out.
             valid_clouds = []
             enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+                sky_cloud.CloudCapability.COMPUTE,
                 raise_if_no_cloud_access=True)
             cloud_to_errors = {}
             for cloud in enabled_clouds:
@@ -796,6 +798,7 @@ class Resources:
             # If cloud not specified
             valid_clouds = []
             enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+                sky_cloud.CloudCapability.COMPUTE,
                 raise_if_no_cloud_access=True)
             for cloud in enabled_clouds:
                 if cloud.instance_type_exists(self._instance_type):
@@ -991,6 +994,7 @@ class Resources:
         else:
             at_least_one_cloud_supports_ports = False
             for cloud in sky_check.get_cached_enabled_clouds_or_refresh(
+                    sky_cloud.CloudCapability.COMPUTE,
                     raise_if_no_cloud_access=True):
                 try:
                     cloud.check_features_are_supported(
@@ -1020,7 +1024,8 @@ class Resources:
         else:
             # If no specific cloud is set, validate label against ALL clouds.
             # The label will be dropped if invalid for any one of the cloud
-            validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh()
+            validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
+                sky_cloud.CloudCapability.COMPUTE)
         invalid_table = log_utils.create_table(['Label', 'Reason'])
         for key, value in self._labels.items():
             for cloud in validated_clouds:

sky/serve/replica_managers.py

@@ -1205,7 +1205,16 @@ class SkyPilotReplicaManager(ReplicaManager):
                 for key in ['service']:
                     old_config.pop(key)
                 # Bump replica version if all fields except for service are
-                # the same. File mounts should both be empty, as update always
+                # the same.
+                # Here, we manually convert the any_of field to a set to avoid
+                # only the difference in the random order of the any_of fields.
+                old_config_any_of = old_config.get('resources',
+                                                   {}).pop('any_of', [])
+                new_config_any_of = new_config.get('resources',
+                                                   {}).pop('any_of', [])
+                if set(old_config_any_of) != set(new_config_any_of):
+                    continue
+                # File mounts should both be empty, as update always
                 # create new buckets if they are not empty.
                 if (old_config == new_config and
                         old_config.get('file_mounts', None) == {}):

sky/setup_files/dependencies.py

@@ -150,7 +150,9 @@ extras_require: Dict[str, List[str]] = {
         # docs instead.
         # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
     ],
-    'nebius': ['nebius>=0.2.0',]
+    'nebius': [
+        'nebius>=0.2.0',
+    ] + aws_dependencies
 }
 
 # Nebius needs python3.10. If python 3.9 [all] will not install nebius

sky/task.py

@@ -974,8 +974,8 @@ class Task:
         # assert len(self.resources) == 1, self.resources
         storage_cloud = None
 
-        enabled_storage_clouds = (
-            storage_lib.get_cached_enabled_storage_clouds_or_refresh(
+        enabled_storage_cloud_names = (
+            storage_lib.get_cached_enabled_storage_cloud_names_or_refresh(
                 raise_if_no_cloud_access=True))
 
         if self.best_resources is not None:
@@ -987,13 +987,13 @@ class Task:
             storage_region = resources.region
 
         if storage_cloud is not None:
-            if str(storage_cloud) not in enabled_storage_clouds:
+            if str(storage_cloud) not in enabled_storage_cloud_names:
                 storage_cloud = None
 
         storage_cloud_str = None
         if storage_cloud is None:
-            storage_cloud_str = enabled_storage_clouds[0]
-            assert storage_cloud_str is not None, enabled_storage_clouds[0]
+            storage_cloud_str = enabled_storage_cloud_names[0]
+            assert storage_cloud_str is not None, enabled_storage_cloud_names[0]
             storage_region = None  # Use default region in the Store class
         else:
             storage_cloud_str = str(storage_cloud)
@@ -1103,6 +1103,17 @@ class Task:
                     self.update_file_mounts({
                         mnt_path: blob_path,
                     })
+                elif store_type is storage_lib.StoreType.NEBIUS:
+                    if storage.source is not None and not isinstance(
+                            storage.source,
+                            list) and storage.source.startswith('nebius://'):
+                        blob_path = storage.source
+                    else:
+                        blob_path = 'nebius://' + storage.name
+                    blob_path = storage.get_bucket_sub_path_prefix(blob_path)
+                    self.update_file_mounts({
+                        mnt_path: blob_path,
+                    })
                 elif store_type is storage_lib.StoreType.IBM:
                     if isinstance(storage.source,
                                   str) and storage.source.startswith('cos://'):

sky/utils/command_runner.py

@@ -29,6 +29,8 @@ RSYNC_DISPLAY_OPTION = '-Pavz'
 # Note that "-" is mandatory for rsync and means all patterns in the ignore
 # files are treated as *exclude* patterns.  Non-exclude patterns, e.g., "!
 # do_not_exclude" doesn't work, even though git allows it.
+# TODO(cooperc): Avoid using this, and prefer utils in storage_utils instead for
+# consistency between bucket upload and rsync.
 RSYNC_FILTER_SKYIGNORE = f'--filter=\'dir-merge,- {constants.SKY_IGNORE_FILE}\''
 RSYNC_FILTER_GITIGNORE = f'--filter=\'dir-merge,- {constants.GIT_IGNORE_FILE}\''
 # The git exclude file to support.