skypilot-nightly 1.0.0.dev20250320__py3-none-any.whl → 1.0.0.dev20250321__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. sky/__init__.py +2 -2
  2. sky/adaptors/cloudflare.py +16 -4
  3. sky/adaptors/kubernetes.py +2 -1
  4. sky/adaptors/nebius.py +128 -6
  5. sky/backends/cloud_vm_ray_backend.py +3 -1
  6. sky/benchmark/benchmark_utils.py +3 -2
  7. sky/check.py +18 -87
  8. sky/cloud_stores.py +66 -0
  9. sky/clouds/aws.py +14 -7
  10. sky/clouds/azure.py +13 -6
  11. sky/clouds/cloud.py +33 -10
  12. sky/clouds/cudo.py +3 -2
  13. sky/clouds/do.py +3 -2
  14. sky/clouds/fluidstack.py +3 -2
  15. sky/clouds/gcp.py +8 -9
  16. sky/clouds/ibm.py +15 -6
  17. sky/clouds/kubernetes.py +3 -1
  18. sky/clouds/lambda_cloud.py +3 -1
  19. sky/clouds/nebius.py +7 -3
  20. sky/clouds/oci.py +15 -6
  21. sky/clouds/paperspace.py +3 -2
  22. sky/clouds/runpod.py +7 -1
  23. sky/clouds/scp.py +3 -1
  24. sky/clouds/service_catalog/kubernetes_catalog.py +3 -1
  25. sky/clouds/vast.py +3 -2
  26. sky/clouds/vsphere.py +3 -2
  27. sky/core.py +4 -2
  28. sky/data/data_transfer.py +75 -0
  29. sky/data/data_utils.py +34 -0
  30. sky/data/mounting_utils.py +18 -0
  31. sky/data/storage.py +537 -9
  32. sky/data/storage_utils.py +102 -84
  33. sky/exceptions.py +2 -0
  34. sky/global_user_state.py +12 -33
  35. sky/jobs/server/core.py +1 -1
  36. sky/jobs/utils.py +5 -0
  37. sky/optimizer.py +7 -2
  38. sky/resources.py +6 -1
  39. sky/setup_files/dependencies.py +3 -1
  40. sky/task.py +16 -5
  41. sky/utils/command_runner.py +2 -0
  42. sky/utils/controller_utils.py +8 -5
  43. sky/utils/kubernetes/kubernetes_deploy_utils.py +2 -1
  44. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/METADATA +11 -1
  45. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/RECORD +49 -49
  46. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/WHEEL +1 -1
  47. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/entry_points.txt +0 -0
  48. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/licenses/LICENSE +0 -0
  49. {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/top_level.txt +0 -0
sky/data/storage_utils.py CHANGED
@@ -24,6 +24,9 @@ _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG = (
24
24
  'to the cloud storage for {path!r}'
25
25
  'due to the following error: {error_msg!r}')
26
26
 
27
+ _USE_SKYIGNORE_HINT = (
28
+ 'To avoid using .gitignore, you can create a .skyignore file instead.')
29
+
27
30
  _LAST_USE_TRUNC_LENGTH = 25
28
31
 
29
32
 
@@ -109,10 +112,9 @@ def get_excluded_files_from_skyignore(src_dir_path: str) -> List[str]:
109
112
  def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
110
113
  """ Lists files and patterns ignored by git in the source directory
111
114
 
112
- Runs `git status --ignored` which returns a list of excluded files and
115
+ Runs `git ls-files --ignored ...` which returns a list of excluded files and
113
116
  patterns read from .gitignore and .git/info/exclude using git.
114
- `git init` is run if SRC_DIR_PATH is not a git repository and removed
115
- after obtaining excluded list.
117
+ This will also be run for all submodules under the src_dir_path.
116
118
 
117
119
  Returns:
118
120
  List[str] containing files and patterns to be ignored. Some of the
@@ -120,91 +122,107 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
120
122
  """
121
123
  expand_src_dir_path = os.path.expanduser(src_dir_path)
122
124
 
123
- git_exclude_path = os.path.join(expand_src_dir_path, '.git/info/exclude')
124
- gitignore_path = os.path.join(expand_src_dir_path,
125
- constants.GIT_IGNORE_FILE)
125
+ # We will use `git ls-files` to list files that we should ignore, but
126
+ # `ls-files` will not recurse into subdirectories. So, we need to manually
127
+ # list the submodules and run `ls-files` within the root and each submodule.
128
+ # Print the submodule paths relative to expand_src_dir_path, separated by
129
+ # null chars.
130
+ submodules_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
131
+ 'submodule foreach -q "printf \\$displaypath\\\\\\0"')
132
+
133
+ try:
134
+ submodules_output = subprocess.run(submodules_cmd,
135
+ shell=True,
136
+ stdout=subprocess.PIPE,
137
+ stderr=subprocess.PIPE,
138
+ check=True,
139
+ text=True)
140
+ except subprocess.CalledProcessError as e:
141
+ gitignore_path = os.path.join(expand_src_dir_path,
142
+ constants.GIT_IGNORE_FILE)
143
+
144
+ if (e.returncode == exceptions.GIT_FATAL_EXIT_CODE and
145
+ 'not a git repository' in e.stderr):
146
+ # If git failed because we aren't in a git repository, but there is
147
+ # a .gitignore, warn the user that it will be ignored.
148
+ if os.path.exists(gitignore_path):
149
+ logger.warning('Detected a .gitignore file, but '
150
+ f'{src_dir_path} is not a git repository. The '
151
+ '.gitignore file will be ignored. '
152
+ f'{_USE_SKYIGNORE_HINT}')
153
+ # Otherwise, this is fine and we can exit early.
154
+ return []
155
+
156
+ if e.returncode == exceptions.COMMAND_NOT_FOUND_EXIT_CODE:
157
+ # Git is not installed. This is fine, skip the check.
158
+ # If .gitignore is present, warn the user.
159
+ if os.path.exists(gitignore_path):
160
+ logger.warning(f'Detected a .gitignore file in {src_dir_path}, '
161
+ 'but git is not installed. The .gitignore file '
162
+ f'will be ignored. {_USE_SKYIGNORE_HINT}')
163
+ return []
164
+
165
+ # Pretty much any other error is unexpected, so re-raise.
166
+ raise
126
167
 
127
- git_exclude_exists = os.path.isfile(git_exclude_path)
128
- gitignore_exists = os.path.isfile(gitignore_path)
168
+ # submodules_output will contain each submodule path (relative to
169
+ # src_dir_path), each ending with a null character.
170
+ # .split will have an empty string at the end because of the final null
171
+ # char, so trim it.
172
+ submodules = submodules_output.stdout.split('\0')[:-1]
173
+
174
+ # The empty string is the relative reference to the src_dir_path.
175
+ all_git_repos = ['.'] + [
176
+ # We only care about submodules that are a subdirectory of src_dir_path.
177
+ submodule for submodule in submodules if not submodule.startswith('../')
178
+ ]
129
179
 
130
- # This command outputs a list to be excluded according to .gitignore
131
- # and .git/info/exclude
132
- filter_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
133
- 'status --ignored --porcelain=v1')
134
180
  excluded_list: List[str] = []
181
+ for repo in all_git_repos:
182
+ # repo is the path relative to src_dir_path. Get the full path.
183
+ repo_path = os.path.join(expand_src_dir_path, repo)
184
+ # This command outputs a list to be excluded according to .gitignore,
185
+ # .git/info/exclude, and global exclude config.
186
+ # -z: filenames terminated by \0 instead of \n
187
+ # --others: show untracked files
188
+ # --ignore: out of untracked files, only show ignored files
189
+ # --exclude-standard: use standard exclude rules (required for --ignore)
190
+ # --directory: if an entire directory is ignored, collapse to a single
191
+ # entry rather than listing every single file
192
+ # Since we are using --others instead of --cached, this will not show
193
+ # files that are tracked but also present in .gitignore.
194
+ filter_cmd = (f'git -C {shlex.quote(repo_path)} ls-files -z '
195
+ '--others --ignore --exclude-standard --directory')
196
+ output = subprocess.run(filter_cmd,
197
+ shell=True,
198
+ stdout=subprocess.PIPE,
199
+ stderr=subprocess.PIPE,
200
+ check=True,
201
+ text=True)
202
+ # Don't catch any errors. We would only expect to see errors during the
203
+ # first git invocation - so if we see any here, crash.
204
+
205
+ output_list = output.stdout.split('\0')
206
+ # trim the empty string at the end
207
+ output_list = output_list[:-1]
208
+
209
+ for item in output_list:
210
+
211
+ if repo == '.' and item == './':
212
+ logger.warning(f'{src_dir_path} is within a git repo, but the '
213
+ 'entire directory is ignored by git. We will '
214
+ 'ignore all git exclusions. '
215
+ f'{_USE_SKYIGNORE_HINT}')
216
+ return []
217
+
218
+ to_be_excluded = os.path.join(repo, item)
219
+ if item.endswith('/'):
220
+ # aws s3 sync and gsutil rsync require * to exclude
221
+ # files/dirs under the specified directory.
222
+ to_be_excluded += '*'
223
+
224
+ excluded_list.append(to_be_excluded)
135
225
 
136
- if git_exclude_exists or gitignore_exists:
137
- try:
138
- output = subprocess.run(filter_cmd,
139
- shell=True,
140
- stdout=subprocess.PIPE,
141
- stderr=subprocess.PIPE,
142
- check=True,
143
- text=True)
144
- except subprocess.CalledProcessError as e:
145
- # when the SRC_DIR_PATH is not a git repo and .git
146
- # does not exist in it
147
- if e.returncode == exceptions.GIT_FATAL_EXIT_CODE:
148
- if 'not a git repository' in e.stderr:
149
- # Check if the user has 'write' permission to
150
- # SRC_DIR_PATH
151
- if not os.access(expand_src_dir_path, os.W_OK):
152
- error_msg = 'Write permission denial'
153
- logger.warning(
154
- _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
155
- path=src_dir_path, error_msg=error_msg))
156
- return excluded_list
157
- init_cmd = f'git -C {expand_src_dir_path} init'
158
- try:
159
- subprocess.run(init_cmd,
160
- shell=True,
161
- stdout=subprocess.PIPE,
162
- stderr=subprocess.PIPE,
163
- check=True)
164
- output = subprocess.run(filter_cmd,
165
- shell=True,
166
- stdout=subprocess.PIPE,
167
- stderr=subprocess.PIPE,
168
- check=True,
169
- text=True)
170
- except subprocess.CalledProcessError as init_e:
171
- logger.warning(
172
- _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
173
- path=src_dir_path, error_msg=init_e.stderr))
174
- return excluded_list
175
- if git_exclude_exists:
176
- # removes all the files/dirs created with 'git init'
177
- # under .git/ except .git/info/exclude
178
- remove_files_cmd = (f'find {expand_src_dir_path}' \
179
- f'/.git -path {git_exclude_path}' \
180
- ' -prune -o -type f -exec rm -f ' \
181
- '{} +')
182
- remove_dirs_cmd = (f'find {expand_src_dir_path}' \
183
- f'/.git -path {git_exclude_path}' \
184
- ' -o -type d -empty -delete')
185
- subprocess.run(remove_files_cmd,
186
- shell=True,
187
- stdout=subprocess.PIPE,
188
- stderr=subprocess.PIPE,
189
- check=True)
190
- subprocess.run(remove_dirs_cmd,
191
- shell=True,
192
- stdout=subprocess.PIPE,
193
- stderr=subprocess.PIPE,
194
- check=True)
195
-
196
- output_list = output.stdout.split('\n')
197
- for line in output_list:
198
- # FILTER_CMD outputs items preceded by '!!'
199
- # to specify excluded files/dirs
200
- # e.g., '!! mydir/' or '!! mydir/myfile.txt'
201
- if line.startswith('!!'):
202
- to_be_excluded = line[3:]
203
- if line.endswith('/'):
204
- # aws s3 sync and gsutil rsync require * to exclude
205
- # files/dirs under the specified directory.
206
- to_be_excluded += '*'
207
- excluded_list.append(to_be_excluded)
208
226
  return excluded_list
209
227
 
210
228
 
sky/exceptions.py CHANGED
@@ -24,6 +24,8 @@ MOUNT_PATH_NON_EMPTY_CODE = 42
24
24
  INSUFFICIENT_PRIVILEGES_CODE = 52
25
25
  # Return code when git command is ran in a dir that is not git repo
26
26
  GIT_FATAL_EXIT_CODE = 128
27
+ # Return code from bash when a command is not found
28
+ COMMAND_NOT_FOUND_EXIT_CODE = 127
27
29
  # Architecture, such as arm64, not supported by the dependency
28
30
  ARCH_NOT_SUPPORTED_EXIT_CODE = 133
29
31
 
sky/global_user_state.py CHANGED
@@ -26,12 +26,12 @@ from sky.utils import status_lib
26
26
  if typing.TYPE_CHECKING:
27
27
  from sky import backends
28
28
  from sky import clouds
29
+ from sky.clouds import cloud
29
30
  from sky.data import Storage
30
31
 
31
32
  logger = sky_logging.init_logger(__name__)
32
33
 
33
- _ENABLED_CLOUDS_KEY = 'enabled_clouds'
34
- _ENABLED_STORAGE_CLOUDS_KEY = 'enabled_storage_clouds'
34
+ _ENABLED_CLOUDS_KEY_PREFIX = 'enabled_clouds_'
35
35
 
36
36
  _DB_PATH = os.path.expanduser('~/.sky/state.db')
37
37
  pathlib.Path(_DB_PATH).parents[0].mkdir(parents=True, exist_ok=True)
@@ -796,31 +796,11 @@ def get_cluster_names_start_with(starts_with: str) -> List[str]:
796
796
  return [row[0] for row in rows]
797
797
 
798
798
 
799
- def get_cached_enabled_clouds() -> List['clouds.Cloud']:
800
- rows = _DB.cursor.execute('SELECT value FROM config WHERE key = ?',
801
- (_ENABLED_CLOUDS_KEY,))
802
- ret = []
803
- for (value,) in rows:
804
- ret = json.loads(value)
805
- break
806
- enabled_clouds: List['clouds.Cloud'] = []
807
- for c in ret:
808
- try:
809
- cloud = registry.CLOUD_REGISTRY.from_str(c)
810
- except ValueError:
811
- # Handle the case for the clouds whose support has been removed from
812
- # SkyPilot, e.g., 'local' was a cloud in the past and may be stored
813
- # in the database for users before #3037. We should ignore removed
814
- # clouds and continue.
815
- continue
816
- if cloud is not None:
817
- enabled_clouds.append(cloud)
818
- return enabled_clouds
799
+ def get_cached_enabled_clouds(
800
+ cloud_capability: 'cloud.CloudCapability') -> List['clouds.Cloud']:
819
801
 
820
-
821
- def get_cached_enabled_storage_clouds() -> List['clouds.Cloud']:
822
802
  rows = _DB.cursor.execute('SELECT value FROM config WHERE key = ?',
823
- (_ENABLED_STORAGE_CLOUDS_KEY,))
803
+ (_get_capability_key(cloud_capability),))
824
804
  ret = []
825
805
  for (value,) in rows:
826
806
  ret = json.loads(value)
@@ -840,19 +820,18 @@ def get_cached_enabled_storage_clouds() -> List['clouds.Cloud']:
840
820
  return enabled_clouds
841
821
 
842
822
 
843
- def set_enabled_clouds(enabled_clouds: List[str]) -> None:
844
- _DB.cursor.execute('INSERT OR REPLACE INTO config VALUES (?, ?)',
845
- (_ENABLED_CLOUDS_KEY, json.dumps(enabled_clouds)))
846
- _DB.conn.commit()
847
-
848
-
849
- def set_enabled_storage_clouds(enabled_storage_clouds: List[str]) -> None:
823
+ def set_enabled_clouds(enabled_clouds: List[str],
824
+ cloud_capability: 'cloud.CloudCapability') -> None:
850
825
  _DB.cursor.execute(
851
826
  'INSERT OR REPLACE INTO config VALUES (?, ?)',
852
- (_ENABLED_STORAGE_CLOUDS_KEY, json.dumps(enabled_storage_clouds)))
827
+ (_get_capability_key(cloud_capability), json.dumps(enabled_clouds)))
853
828
  _DB.conn.commit()
854
829
 
855
830
 
831
+ def _get_capability_key(cloud_capability: 'cloud.CloudCapability') -> str:
832
+ return _ENABLED_CLOUDS_KEY_PREFIX + cloud_capability.value
833
+
834
+
856
835
  def add_or_update_storage(storage_name: str,
857
836
  storage_handle: 'Storage.StorageMetadata',
858
837
  storage_status: status_lib.StorageStatus):
sky/jobs/server/core.py CHANGED
@@ -105,7 +105,7 @@ def launch(
105
105
 
106
106
  local_to_controller_file_mounts = {}
107
107
 
108
- if storage_lib.get_cached_enabled_storage_clouds_or_refresh():
108
+ if storage_lib.get_cached_enabled_storage_cloud_names_or_refresh():
109
109
  for task_ in dag.tasks:
110
110
  controller_utils.maybe_translate_local_file_mounts_and_sync_up(
111
111
  task_, task_type='jobs')
sky/jobs/utils.py CHANGED
@@ -125,6 +125,11 @@ def get_job_status(backend: 'backends.CloudVmRayBackend',
125
125
  FAILED_SETUP or CANCELLED.
126
126
  """
127
127
  handle = global_user_state.get_handle_from_cluster_name(cluster_name)
128
+ if handle is None:
129
+ # This can happen if the cluster was preempted and background status
130
+ # refresh already noticed and cleaned it up.
131
+ logger.info(f'Cluster {cluster_name} not found.')
132
+ return None
128
133
  assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
129
134
  status = None
130
135
  try:
sky/optimizer.py CHANGED
@@ -16,6 +16,7 @@ from sky import resources as resources_lib
16
16
  from sky import sky_logging
17
17
  from sky import task as task_lib
18
18
  from sky.adaptors import common as adaptors_common
19
+ from sky.clouds import cloud as sky_cloud
19
20
  from sky.usage import usage_lib
20
21
  from sky.utils import common
21
22
  from sky.utils import env_options
@@ -368,7 +369,8 @@ class Optimizer:
368
369
  # mention "kubernetes cluster" and/instead of "catalog"
369
370
  # in the error message.
370
371
  enabled_clouds = (
371
- sky_check.get_cached_enabled_clouds_or_refresh())
372
+ sky_check.get_cached_enabled_clouds_or_refresh(
373
+ sky_cloud.CloudCapability.COMPUTE))
372
374
  if clouds.cloud_in_iterable(clouds.Kubernetes(),
373
375
  enabled_clouds):
374
376
  if any(orig_resources.cloud is None
@@ -1206,6 +1208,7 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
1206
1208
  dag: The DAG specified by a user.
1207
1209
  """
1208
1210
  enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
1211
+ capability=sky_cloud.CloudCapability.COMPUTE,
1209
1212
  raise_if_no_cloud_access=True)
1210
1213
 
1211
1214
  global_disabled_clouds: Set[str] = set()
@@ -1226,8 +1229,9 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
1226
1229
  sky_check.check(quiet=True,
1227
1230
  clouds=list(clouds_need_recheck -
1228
1231
  global_disabled_clouds),
1229
- capability=sky_check.CloudCapability.COMPUTE)
1232
+ capability=sky_cloud.CloudCapability.COMPUTE)
1230
1233
  enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
1234
+ capability=sky_cloud.CloudCapability.COMPUTE,
1231
1235
  raise_if_no_cloud_access=True)
1232
1236
  disabled_clouds = (clouds_need_recheck -
1233
1237
  {str(c) for c in enabled_clouds})
@@ -1269,6 +1273,7 @@ def _fill_in_launchable_resources(
1269
1273
  a cloud that is not enabled.
1270
1274
  """
1271
1275
  enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
1276
+ capability=sky_cloud.CloudCapability.COMPUTE,
1272
1277
  raise_if_no_cloud_access=True)
1273
1278
 
1274
1279
  launchable: Dict[resources_lib.Resources, List[resources_lib.Resources]] = (
sky/resources.py CHANGED
@@ -10,6 +10,7 @@ from sky import clouds
10
10
  from sky import exceptions
11
11
  from sky import sky_logging
12
12
  from sky import skypilot_config
13
+ from sky.clouds import cloud as sky_cloud
13
14
  from sky.clouds import service_catalog
14
15
  from sky.provision import docker_utils
15
16
  from sky.provision.kubernetes import utils as kubernetes_utils
@@ -677,6 +678,7 @@ class Resources:
677
678
  # cloud corresponds to region/zone, errors out.
678
679
  valid_clouds = []
679
680
  enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
681
+ sky_cloud.CloudCapability.COMPUTE,
680
682
  raise_if_no_cloud_access=True)
681
683
  cloud_to_errors = {}
682
684
  for cloud in enabled_clouds:
@@ -796,6 +798,7 @@ class Resources:
796
798
  # If cloud not specified
797
799
  valid_clouds = []
798
800
  enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
801
+ sky_cloud.CloudCapability.COMPUTE,
799
802
  raise_if_no_cloud_access=True)
800
803
  for cloud in enabled_clouds:
801
804
  if cloud.instance_type_exists(self._instance_type):
@@ -991,6 +994,7 @@ class Resources:
991
994
  else:
992
995
  at_least_one_cloud_supports_ports = False
993
996
  for cloud in sky_check.get_cached_enabled_clouds_or_refresh(
997
+ sky_cloud.CloudCapability.COMPUTE,
994
998
  raise_if_no_cloud_access=True):
995
999
  try:
996
1000
  cloud.check_features_are_supported(
@@ -1020,7 +1024,8 @@ class Resources:
1020
1024
  else:
1021
1025
  # If no specific cloud is set, validate label against ALL clouds.
1022
1026
  # The label will be dropped if invalid for any one of the cloud
1023
- validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh()
1027
+ validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
1028
+ sky_cloud.CloudCapability.COMPUTE)
1024
1029
  invalid_table = log_utils.create_table(['Label', 'Reason'])
1025
1030
  for key, value in self._labels.items():
1026
1031
  for cloud in validated_clouds:
@@ -150,7 +150,9 @@ extras_require: Dict[str, List[str]] = {
150
150
  # docs instead.
151
151
  # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
152
152
  ],
153
- 'nebius': ['nebius>=0.2.0',]
153
+ 'nebius': [
154
+ 'nebius>=0.2.0',
155
+ ] + aws_dependencies
154
156
  }
155
157
 
156
158
  # Nebius needs python3.10. If python 3.9 [all] will not install nebius
sky/task.py CHANGED
@@ -974,8 +974,8 @@ class Task:
974
974
  # assert len(self.resources) == 1, self.resources
975
975
  storage_cloud = None
976
976
 
977
- enabled_storage_clouds = (
978
- storage_lib.get_cached_enabled_storage_clouds_or_refresh(
977
+ enabled_storage_cloud_names = (
978
+ storage_lib.get_cached_enabled_storage_cloud_names_or_refresh(
979
979
  raise_if_no_cloud_access=True))
980
980
 
981
981
  if self.best_resources is not None:
@@ -987,13 +987,13 @@ class Task:
987
987
  storage_region = resources.region
988
988
 
989
989
  if storage_cloud is not None:
990
- if str(storage_cloud) not in enabled_storage_clouds:
990
+ if str(storage_cloud) not in enabled_storage_cloud_names:
991
991
  storage_cloud = None
992
992
 
993
993
  storage_cloud_str = None
994
994
  if storage_cloud is None:
995
- storage_cloud_str = enabled_storage_clouds[0]
996
- assert storage_cloud_str is not None, enabled_storage_clouds[0]
995
+ storage_cloud_str = enabled_storage_cloud_names[0]
996
+ assert storage_cloud_str is not None, enabled_storage_cloud_names[0]
997
997
  storage_region = None # Use default region in the Store class
998
998
  else:
999
999
  storage_cloud_str = str(storage_cloud)
@@ -1103,6 +1103,17 @@ class Task:
1103
1103
  self.update_file_mounts({
1104
1104
  mnt_path: blob_path,
1105
1105
  })
1106
+ elif store_type is storage_lib.StoreType.NEBIUS:
1107
+ if storage.source is not None and not isinstance(
1108
+ storage.source,
1109
+ list) and storage.source.startswith('nebius://'):
1110
+ blob_path = storage.source
1111
+ else:
1112
+ blob_path = 'nebius://' + storage.name
1113
+ blob_path = storage.get_bucket_sub_path_prefix(blob_path)
1114
+ self.update_file_mounts({
1115
+ mnt_path: blob_path,
1116
+ })
1106
1117
  elif store_type is storage_lib.StoreType.IBM:
1107
1118
  if isinstance(storage.source,
1108
1119
  str) and storage.source.startswith('cos://'):
@@ -29,6 +29,8 @@ RSYNC_DISPLAY_OPTION = '-Pavz'
29
29
  # Note that "-" is mandatory for rsync and means all patterns in the ignore
30
30
  # files are treated as *exclude* patterns. Non-exclude patterns, e.g., "!
31
31
  # do_not_exclude" doesn't work, even though git allows it.
32
+ # TODO(cooperc): Avoid using this, and prefer utils in storage_utils instead for
33
+ # consistency between bucket upload and rsync.
32
34
  RSYNC_FILTER_SKYIGNORE = f'--filter=\'dir-merge,- {constants.SKY_IGNORE_FILE}\''
33
35
  RSYNC_FILTER_GITIGNORE = f'--filter=\'dir-merge,- {constants.GIT_IGNORE_FILE}\''
34
36
  # The git exclude file to support.
@@ -19,6 +19,7 @@ from sky import resources
19
19
  from sky import sky_logging
20
20
  from sky import skypilot_config
21
21
  from sky.adaptors import cloudflare
22
+ from sky.clouds import cloud as sky_cloud
22
23
  from sky.clouds import gcp
23
24
  from sky.data import data_utils
24
25
  from sky.data import storage as storage_lib
@@ -216,9 +217,11 @@ def _get_cloud_dependencies_installation_commands(
216
217
  f'{constants.SKY_UV_INSTALL_CMD} >/dev/null 2>&1')
217
218
 
218
219
  enabled_compute_clouds = set(
219
- sky_check.get_cached_enabled_clouds_or_refresh())
220
+ sky_check.get_cached_enabled_clouds_or_refresh(
221
+ sky_cloud.CloudCapability.COMPUTE))
220
222
  enabled_storage_clouds = set(
221
- sky_check.get_cached_enabled_storage_clouds_or_refresh())
223
+ sky_check.get_cached_enabled_clouds_or_refresh(
224
+ sky_cloud.CloudCapability.STORAGE))
222
225
  enabled_clouds = enabled_compute_clouds.union(enabled_storage_clouds)
223
226
 
224
227
  for cloud in enabled_clouds:
@@ -281,7 +284,7 @@ def _get_cloud_dependencies_installation_commands(
281
284
  python_packages.update(cloud_python_dependencies)
282
285
 
283
286
  if (cloudflare.NAME
284
- in storage_lib.get_cached_enabled_storage_clouds_or_refresh()):
287
+ in storage_lib.get_cached_enabled_storage_cloud_names_or_refresh()):
285
288
  python_packages.update(dependencies.extras_require['cloudflare'])
286
289
 
287
290
  packages_string = ' '.join([f'"{package}"' for package in python_packages])
@@ -812,8 +815,8 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
812
815
  (store_type, bucket_name, sub_path, storage_account_name, region) = (
813
816
  storage_lib.StoreType.get_fields_from_store_url(bucket_wth_prefix))
814
817
  cloud_str = store_type.to_cloud()
815
- if (cloud_str not in
816
- storage_lib.get_cached_enabled_storage_clouds_or_refresh()):
818
+ if (cloud_str not in storage_lib.
819
+ get_cached_enabled_storage_cloud_names_or_refresh()):
817
820
  with ux_utils.print_exception_no_traceback():
818
821
  raise ValueError(
819
822
  f'`{task_type}.bucket` is specified in SkyPilot config '
@@ -8,6 +8,7 @@ from typing import List, Optional
8
8
  from sky import check as sky_check
9
9
  from sky import sky_logging
10
10
  from sky.backends import backend_utils
11
+ from sky.clouds import cloud as sky_cloud
11
12
  from sky.provision.kubernetes import utils as kubernetes_utils
12
13
  from sky.skylet import constants
13
14
  from sky.skylet import log_lib
@@ -169,7 +170,7 @@ def deploy_local_cluster(gpus: bool):
169
170
  with rich_utils.safe_status('[bold cyan]Running sky check...'):
170
171
  sky_check.check(clouds=['kubernetes'],
171
172
  quiet=True,
172
- capability=sky_check.CloudCapability.COMPUTE)
173
+ capability=sky_cloud.CloudCapability.COMPUTE)
173
174
  if cluster_created:
174
175
  # Prepare completion message which shows CPU and GPU count
175
176
  # Get number of CPUs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: skypilot-nightly
3
- Version: 1.0.0.dev20250320
3
+ Version: 1.0.0.dev20250321
4
4
  Summary: SkyPilot: An intercloud broker for the clouds
5
5
  Author: SkyPilot Team
6
6
  License: Apache 2.0
@@ -108,6 +108,11 @@ Provides-Extra: vsphere
108
108
  Requires-Dist: pyvmomi==8.0.1.0.2; extra == "vsphere"
109
109
  Provides-Extra: nebius
110
110
  Requires-Dist: nebius>=0.2.0; extra == "nebius"
111
+ Requires-Dist: urllib3<2; extra == "nebius"
112
+ Requires-Dist: awscli>=1.27.10; extra == "nebius"
113
+ Requires-Dist: botocore>=1.29.10; extra == "nebius"
114
+ Requires-Dist: boto3>=1.26.1; extra == "nebius"
115
+ Requires-Dist: colorama<0.4.5; extra == "nebius"
111
116
  Provides-Extra: all
112
117
  Requires-Dist: urllib3<2; extra == "all"
113
118
  Requires-Dist: awscli>=1.27.10; extra == "all"
@@ -151,6 +156,11 @@ Requires-Dist: azure-common; extra == "all"
151
156
  Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
152
157
  Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
153
158
  Requires-Dist: nebius>=0.2.0; extra == "all"
159
+ Requires-Dist: urllib3<2; extra == "all"
160
+ Requires-Dist: awscli>=1.27.10; extra == "all"
161
+ Requires-Dist: botocore>=1.29.10; extra == "all"
162
+ Requires-Dist: boto3>=1.26.1; extra == "all"
163
+ Requires-Dist: colorama<0.4.5; extra == "all"
154
164
  Dynamic: author
155
165
  Dynamic: classifier
156
166
  Dynamic: description