skypilot-nightly 1.0.0.dev20250320__py3-none-any.whl → 1.0.0.dev20250321__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sky/__init__.py +2 -2
- sky/adaptors/cloudflare.py +16 -4
- sky/adaptors/kubernetes.py +2 -1
- sky/adaptors/nebius.py +128 -6
- sky/backends/cloud_vm_ray_backend.py +3 -1
- sky/benchmark/benchmark_utils.py +3 -2
- sky/check.py +18 -87
- sky/cloud_stores.py +66 -0
- sky/clouds/aws.py +14 -7
- sky/clouds/azure.py +13 -6
- sky/clouds/cloud.py +33 -10
- sky/clouds/cudo.py +3 -2
- sky/clouds/do.py +3 -2
- sky/clouds/fluidstack.py +3 -2
- sky/clouds/gcp.py +8 -9
- sky/clouds/ibm.py +15 -6
- sky/clouds/kubernetes.py +3 -1
- sky/clouds/lambda_cloud.py +3 -1
- sky/clouds/nebius.py +7 -3
- sky/clouds/oci.py +15 -6
- sky/clouds/paperspace.py +3 -2
- sky/clouds/runpod.py +7 -1
- sky/clouds/scp.py +3 -1
- sky/clouds/service_catalog/kubernetes_catalog.py +3 -1
- sky/clouds/vast.py +3 -2
- sky/clouds/vsphere.py +3 -2
- sky/core.py +4 -2
- sky/data/data_transfer.py +75 -0
- sky/data/data_utils.py +34 -0
- sky/data/mounting_utils.py +18 -0
- sky/data/storage.py +537 -9
- sky/data/storage_utils.py +102 -84
- sky/exceptions.py +2 -0
- sky/global_user_state.py +12 -33
- sky/jobs/server/core.py +1 -1
- sky/jobs/utils.py +5 -0
- sky/optimizer.py +7 -2
- sky/resources.py +6 -1
- sky/setup_files/dependencies.py +3 -1
- sky/task.py +16 -5
- sky/utils/command_runner.py +2 -0
- sky/utils/controller_utils.py +8 -5
- sky/utils/kubernetes/kubernetes_deploy_utils.py +2 -1
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/METADATA +11 -1
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/RECORD +49 -49
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/WHEEL +1 -1
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250320.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/top_level.txt +0 -0
sky/data/storage_utils.py
CHANGED
@@ -24,6 +24,9 @@ _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG = (
|
|
24
24
|
'to the cloud storage for {path!r}'
|
25
25
|
'due to the following error: {error_msg!r}')
|
26
26
|
|
27
|
+
_USE_SKYIGNORE_HINT = (
|
28
|
+
'To avoid using .gitignore, you can create a .skyignore file instead.')
|
29
|
+
|
27
30
|
_LAST_USE_TRUNC_LENGTH = 25
|
28
31
|
|
29
32
|
|
@@ -109,10 +112,9 @@ def get_excluded_files_from_skyignore(src_dir_path: str) -> List[str]:
|
|
109
112
|
def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
|
110
113
|
""" Lists files and patterns ignored by git in the source directory
|
111
114
|
|
112
|
-
Runs `git
|
115
|
+
Runs `git ls-files --ignored ...` which returns a list of excluded files and
|
113
116
|
patterns read from .gitignore and .git/info/exclude using git.
|
114
|
-
|
115
|
-
after obtaining excluded list.
|
117
|
+
This will also be run for all submodules under the src_dir_path.
|
116
118
|
|
117
119
|
Returns:
|
118
120
|
List[str] containing files and patterns to be ignored. Some of the
|
@@ -120,91 +122,107 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
|
|
120
122
|
"""
|
121
123
|
expand_src_dir_path = os.path.expanduser(src_dir_path)
|
122
124
|
|
123
|
-
|
124
|
-
|
125
|
-
|
125
|
+
# We will use `git ls-files` to list files that we should ignore, but
|
126
|
+
# `ls-files` will not recurse into subdirectories. So, we need to manually
|
127
|
+
# list the submodules and run `ls-files` within the root and each submodule.
|
128
|
+
# Print the submodule paths relative to expand_src_dir_path, separated by
|
129
|
+
# null chars.
|
130
|
+
submodules_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
|
131
|
+
'submodule foreach -q "printf \\$displaypath\\\\\\0"')
|
132
|
+
|
133
|
+
try:
|
134
|
+
submodules_output = subprocess.run(submodules_cmd,
|
135
|
+
shell=True,
|
136
|
+
stdout=subprocess.PIPE,
|
137
|
+
stderr=subprocess.PIPE,
|
138
|
+
check=True,
|
139
|
+
text=True)
|
140
|
+
except subprocess.CalledProcessError as e:
|
141
|
+
gitignore_path = os.path.join(expand_src_dir_path,
|
142
|
+
constants.GIT_IGNORE_FILE)
|
143
|
+
|
144
|
+
if (e.returncode == exceptions.GIT_FATAL_EXIT_CODE and
|
145
|
+
'not a git repository' in e.stderr):
|
146
|
+
# If git failed because we aren't in a git repository, but there is
|
147
|
+
# a .gitignore, warn the user that it will be ignored.
|
148
|
+
if os.path.exists(gitignore_path):
|
149
|
+
logger.warning('Detected a .gitignore file, but '
|
150
|
+
f'{src_dir_path} is not a git repository. The '
|
151
|
+
'.gitignore file will be ignored. '
|
152
|
+
f'{_USE_SKYIGNORE_HINT}')
|
153
|
+
# Otherwise, this is fine and we can exit early.
|
154
|
+
return []
|
155
|
+
|
156
|
+
if e.returncode == exceptions.COMMAND_NOT_FOUND_EXIT_CODE:
|
157
|
+
# Git is not installed. This is fine, skip the check.
|
158
|
+
# If .gitignore is present, warn the user.
|
159
|
+
if os.path.exists(gitignore_path):
|
160
|
+
logger.warning(f'Detected a .gitignore file in {src_dir_path}, '
|
161
|
+
'but git is not installed. The .gitignore file '
|
162
|
+
f'will be ignored. {_USE_SKYIGNORE_HINT}')
|
163
|
+
return []
|
164
|
+
|
165
|
+
# Pretty much any other error is unexpected, so re-raise.
|
166
|
+
raise
|
126
167
|
|
127
|
-
|
128
|
-
|
168
|
+
# submodules_output will contain each submodule path (relative to
|
169
|
+
# src_dir_path), each ending with a null character.
|
170
|
+
# .split will have an empty string at the end because of the final null
|
171
|
+
# char, so trim it.
|
172
|
+
submodules = submodules_output.stdout.split('\0')[:-1]
|
173
|
+
|
174
|
+
# The empty string is the relative reference to the src_dir_path.
|
175
|
+
all_git_repos = ['.'] + [
|
176
|
+
# We only care about submodules that are a subdirectory of src_dir_path.
|
177
|
+
submodule for submodule in submodules if not submodule.startswith('../')
|
178
|
+
]
|
129
179
|
|
130
|
-
# This command outputs a list to be excluded according to .gitignore
|
131
|
-
# and .git/info/exclude
|
132
|
-
filter_cmd = (f'git -C {shlex.quote(expand_src_dir_path)} '
|
133
|
-
'status --ignored --porcelain=v1')
|
134
180
|
excluded_list: List[str] = []
|
181
|
+
for repo in all_git_repos:
|
182
|
+
# repo is the path relative to src_dir_path. Get the full path.
|
183
|
+
repo_path = os.path.join(expand_src_dir_path, repo)
|
184
|
+
# This command outputs a list to be excluded according to .gitignore,
|
185
|
+
# .git/info/exclude, and global exclude config.
|
186
|
+
# -z: filenames terminated by \0 instead of \n
|
187
|
+
# --others: show untracked files
|
188
|
+
# --ignore: out of untracked files, only show ignored files
|
189
|
+
# --exclude-standard: use standard exclude rules (required for --ignore)
|
190
|
+
# --directory: if an entire directory is ignored, collapse to a single
|
191
|
+
# entry rather than listing every single file
|
192
|
+
# Since we are using --others instead of --cached, this will not show
|
193
|
+
# files that are tracked but also present in .gitignore.
|
194
|
+
filter_cmd = (f'git -C {shlex.quote(repo_path)} ls-files -z '
|
195
|
+
'--others --ignore --exclude-standard --directory')
|
196
|
+
output = subprocess.run(filter_cmd,
|
197
|
+
shell=True,
|
198
|
+
stdout=subprocess.PIPE,
|
199
|
+
stderr=subprocess.PIPE,
|
200
|
+
check=True,
|
201
|
+
text=True)
|
202
|
+
# Don't catch any errors. We would only expect to see errors during the
|
203
|
+
# first git invocation - so if we see any here, crash.
|
204
|
+
|
205
|
+
output_list = output.stdout.split('\0')
|
206
|
+
# trim the empty string at the end
|
207
|
+
output_list = output_list[:-1]
|
208
|
+
|
209
|
+
for item in output_list:
|
210
|
+
|
211
|
+
if repo == '.' and item == './':
|
212
|
+
logger.warning(f'{src_dir_path} is within a git repo, but the '
|
213
|
+
'entire directory is ignored by git. We will '
|
214
|
+
'ignore all git exclusions. '
|
215
|
+
f'{_USE_SKYIGNORE_HINT}')
|
216
|
+
return []
|
217
|
+
|
218
|
+
to_be_excluded = os.path.join(repo, item)
|
219
|
+
if item.endswith('/'):
|
220
|
+
# aws s3 sync and gsutil rsync require * to exclude
|
221
|
+
# files/dirs under the specified directory.
|
222
|
+
to_be_excluded += '*'
|
223
|
+
|
224
|
+
excluded_list.append(to_be_excluded)
|
135
225
|
|
136
|
-
if git_exclude_exists or gitignore_exists:
|
137
|
-
try:
|
138
|
-
output = subprocess.run(filter_cmd,
|
139
|
-
shell=True,
|
140
|
-
stdout=subprocess.PIPE,
|
141
|
-
stderr=subprocess.PIPE,
|
142
|
-
check=True,
|
143
|
-
text=True)
|
144
|
-
except subprocess.CalledProcessError as e:
|
145
|
-
# when the SRC_DIR_PATH is not a git repo and .git
|
146
|
-
# does not exist in it
|
147
|
-
if e.returncode == exceptions.GIT_FATAL_EXIT_CODE:
|
148
|
-
if 'not a git repository' in e.stderr:
|
149
|
-
# Check if the user has 'write' permission to
|
150
|
-
# SRC_DIR_PATH
|
151
|
-
if not os.access(expand_src_dir_path, os.W_OK):
|
152
|
-
error_msg = 'Write permission denial'
|
153
|
-
logger.warning(
|
154
|
-
_FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
|
155
|
-
path=src_dir_path, error_msg=error_msg))
|
156
|
-
return excluded_list
|
157
|
-
init_cmd = f'git -C {expand_src_dir_path} init'
|
158
|
-
try:
|
159
|
-
subprocess.run(init_cmd,
|
160
|
-
shell=True,
|
161
|
-
stdout=subprocess.PIPE,
|
162
|
-
stderr=subprocess.PIPE,
|
163
|
-
check=True)
|
164
|
-
output = subprocess.run(filter_cmd,
|
165
|
-
shell=True,
|
166
|
-
stdout=subprocess.PIPE,
|
167
|
-
stderr=subprocess.PIPE,
|
168
|
-
check=True,
|
169
|
-
text=True)
|
170
|
-
except subprocess.CalledProcessError as init_e:
|
171
|
-
logger.warning(
|
172
|
-
_FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG.format(
|
173
|
-
path=src_dir_path, error_msg=init_e.stderr))
|
174
|
-
return excluded_list
|
175
|
-
if git_exclude_exists:
|
176
|
-
# removes all the files/dirs created with 'git init'
|
177
|
-
# under .git/ except .git/info/exclude
|
178
|
-
remove_files_cmd = (f'find {expand_src_dir_path}' \
|
179
|
-
f'/.git -path {git_exclude_path}' \
|
180
|
-
' -prune -o -type f -exec rm -f ' \
|
181
|
-
'{} +')
|
182
|
-
remove_dirs_cmd = (f'find {expand_src_dir_path}' \
|
183
|
-
f'/.git -path {git_exclude_path}' \
|
184
|
-
' -o -type d -empty -delete')
|
185
|
-
subprocess.run(remove_files_cmd,
|
186
|
-
shell=True,
|
187
|
-
stdout=subprocess.PIPE,
|
188
|
-
stderr=subprocess.PIPE,
|
189
|
-
check=True)
|
190
|
-
subprocess.run(remove_dirs_cmd,
|
191
|
-
shell=True,
|
192
|
-
stdout=subprocess.PIPE,
|
193
|
-
stderr=subprocess.PIPE,
|
194
|
-
check=True)
|
195
|
-
|
196
|
-
output_list = output.stdout.split('\n')
|
197
|
-
for line in output_list:
|
198
|
-
# FILTER_CMD outputs items preceded by '!!'
|
199
|
-
# to specify excluded files/dirs
|
200
|
-
# e.g., '!! mydir/' or '!! mydir/myfile.txt'
|
201
|
-
if line.startswith('!!'):
|
202
|
-
to_be_excluded = line[3:]
|
203
|
-
if line.endswith('/'):
|
204
|
-
# aws s3 sync and gsutil rsync require * to exclude
|
205
|
-
# files/dirs under the specified directory.
|
206
|
-
to_be_excluded += '*'
|
207
|
-
excluded_list.append(to_be_excluded)
|
208
226
|
return excluded_list
|
209
227
|
|
210
228
|
|
sky/exceptions.py
CHANGED
@@ -24,6 +24,8 @@ MOUNT_PATH_NON_EMPTY_CODE = 42
|
|
24
24
|
INSUFFICIENT_PRIVILEGES_CODE = 52
|
25
25
|
# Return code when git command is ran in a dir that is not git repo
|
26
26
|
GIT_FATAL_EXIT_CODE = 128
|
27
|
+
# Return code from bash when a command is not found
|
28
|
+
COMMAND_NOT_FOUND_EXIT_CODE = 127
|
27
29
|
# Architecture, such as arm64, not supported by the dependency
|
28
30
|
ARCH_NOT_SUPPORTED_EXIT_CODE = 133
|
29
31
|
|
sky/global_user_state.py
CHANGED
@@ -26,12 +26,12 @@ from sky.utils import status_lib
|
|
26
26
|
if typing.TYPE_CHECKING:
|
27
27
|
from sky import backends
|
28
28
|
from sky import clouds
|
29
|
+
from sky.clouds import cloud
|
29
30
|
from sky.data import Storage
|
30
31
|
|
31
32
|
logger = sky_logging.init_logger(__name__)
|
32
33
|
|
33
|
-
|
34
|
-
_ENABLED_STORAGE_CLOUDS_KEY = 'enabled_storage_clouds'
|
34
|
+
_ENABLED_CLOUDS_KEY_PREFIX = 'enabled_clouds_'
|
35
35
|
|
36
36
|
_DB_PATH = os.path.expanduser('~/.sky/state.db')
|
37
37
|
pathlib.Path(_DB_PATH).parents[0].mkdir(parents=True, exist_ok=True)
|
@@ -796,31 +796,11 @@ def get_cluster_names_start_with(starts_with: str) -> List[str]:
|
|
796
796
|
return [row[0] for row in rows]
|
797
797
|
|
798
798
|
|
799
|
-
def get_cached_enabled_clouds(
|
800
|
-
|
801
|
-
(_ENABLED_CLOUDS_KEY,))
|
802
|
-
ret = []
|
803
|
-
for (value,) in rows:
|
804
|
-
ret = json.loads(value)
|
805
|
-
break
|
806
|
-
enabled_clouds: List['clouds.Cloud'] = []
|
807
|
-
for c in ret:
|
808
|
-
try:
|
809
|
-
cloud = registry.CLOUD_REGISTRY.from_str(c)
|
810
|
-
except ValueError:
|
811
|
-
# Handle the case for the clouds whose support has been removed from
|
812
|
-
# SkyPilot, e.g., 'local' was a cloud in the past and may be stored
|
813
|
-
# in the database for users before #3037. We should ignore removed
|
814
|
-
# clouds and continue.
|
815
|
-
continue
|
816
|
-
if cloud is not None:
|
817
|
-
enabled_clouds.append(cloud)
|
818
|
-
return enabled_clouds
|
799
|
+
def get_cached_enabled_clouds(
|
800
|
+
cloud_capability: 'cloud.CloudCapability') -> List['clouds.Cloud']:
|
819
801
|
|
820
|
-
|
821
|
-
def get_cached_enabled_storage_clouds() -> List['clouds.Cloud']:
|
822
802
|
rows = _DB.cursor.execute('SELECT value FROM config WHERE key = ?',
|
823
|
-
(
|
803
|
+
(_get_capability_key(cloud_capability),))
|
824
804
|
ret = []
|
825
805
|
for (value,) in rows:
|
826
806
|
ret = json.loads(value)
|
@@ -840,19 +820,18 @@ def get_cached_enabled_storage_clouds() -> List['clouds.Cloud']:
|
|
840
820
|
return enabled_clouds
|
841
821
|
|
842
822
|
|
843
|
-
def set_enabled_clouds(enabled_clouds: List[str]
|
844
|
-
|
845
|
-
(_ENABLED_CLOUDS_KEY, json.dumps(enabled_clouds)))
|
846
|
-
_DB.conn.commit()
|
847
|
-
|
848
|
-
|
849
|
-
def set_enabled_storage_clouds(enabled_storage_clouds: List[str]) -> None:
|
823
|
+
def set_enabled_clouds(enabled_clouds: List[str],
|
824
|
+
cloud_capability: 'cloud.CloudCapability') -> None:
|
850
825
|
_DB.cursor.execute(
|
851
826
|
'INSERT OR REPLACE INTO config VALUES (?, ?)',
|
852
|
-
(
|
827
|
+
(_get_capability_key(cloud_capability), json.dumps(enabled_clouds)))
|
853
828
|
_DB.conn.commit()
|
854
829
|
|
855
830
|
|
831
|
+
def _get_capability_key(cloud_capability: 'cloud.CloudCapability') -> str:
|
832
|
+
return _ENABLED_CLOUDS_KEY_PREFIX + cloud_capability.value
|
833
|
+
|
834
|
+
|
856
835
|
def add_or_update_storage(storage_name: str,
|
857
836
|
storage_handle: 'Storage.StorageMetadata',
|
858
837
|
storage_status: status_lib.StorageStatus):
|
sky/jobs/server/core.py
CHANGED
@@ -105,7 +105,7 @@ def launch(
|
|
105
105
|
|
106
106
|
local_to_controller_file_mounts = {}
|
107
107
|
|
108
|
-
if storage_lib.
|
108
|
+
if storage_lib.get_cached_enabled_storage_cloud_names_or_refresh():
|
109
109
|
for task_ in dag.tasks:
|
110
110
|
controller_utils.maybe_translate_local_file_mounts_and_sync_up(
|
111
111
|
task_, task_type='jobs')
|
sky/jobs/utils.py
CHANGED
@@ -125,6 +125,11 @@ def get_job_status(backend: 'backends.CloudVmRayBackend',
|
|
125
125
|
FAILED_SETUP or CANCELLED.
|
126
126
|
"""
|
127
127
|
handle = global_user_state.get_handle_from_cluster_name(cluster_name)
|
128
|
+
if handle is None:
|
129
|
+
# This can happen if the cluster was preempted and background status
|
130
|
+
# refresh already noticed and cleaned it up.
|
131
|
+
logger.info(f'Cluster {cluster_name} not found.')
|
132
|
+
return None
|
128
133
|
assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
|
129
134
|
status = None
|
130
135
|
try:
|
sky/optimizer.py
CHANGED
@@ -16,6 +16,7 @@ from sky import resources as resources_lib
|
|
16
16
|
from sky import sky_logging
|
17
17
|
from sky import task as task_lib
|
18
18
|
from sky.adaptors import common as adaptors_common
|
19
|
+
from sky.clouds import cloud as sky_cloud
|
19
20
|
from sky.usage import usage_lib
|
20
21
|
from sky.utils import common
|
21
22
|
from sky.utils import env_options
|
@@ -368,7 +369,8 @@ class Optimizer:
|
|
368
369
|
# mention "kubernetes cluster" and/instead of "catalog"
|
369
370
|
# in the error message.
|
370
371
|
enabled_clouds = (
|
371
|
-
sky_check.get_cached_enabled_clouds_or_refresh(
|
372
|
+
sky_check.get_cached_enabled_clouds_or_refresh(
|
373
|
+
sky_cloud.CloudCapability.COMPUTE))
|
372
374
|
if clouds.cloud_in_iterable(clouds.Kubernetes(),
|
373
375
|
enabled_clouds):
|
374
376
|
if any(orig_resources.cloud is None
|
@@ -1206,6 +1208,7 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
|
|
1206
1208
|
dag: The DAG specified by a user.
|
1207
1209
|
"""
|
1208
1210
|
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
1211
|
+
capability=sky_cloud.CloudCapability.COMPUTE,
|
1209
1212
|
raise_if_no_cloud_access=True)
|
1210
1213
|
|
1211
1214
|
global_disabled_clouds: Set[str] = set()
|
@@ -1226,8 +1229,9 @@ def _check_specified_clouds(dag: 'dag_lib.Dag') -> None:
|
|
1226
1229
|
sky_check.check(quiet=True,
|
1227
1230
|
clouds=list(clouds_need_recheck -
|
1228
1231
|
global_disabled_clouds),
|
1229
|
-
capability=
|
1232
|
+
capability=sky_cloud.CloudCapability.COMPUTE)
|
1230
1233
|
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
1234
|
+
capability=sky_cloud.CloudCapability.COMPUTE,
|
1231
1235
|
raise_if_no_cloud_access=True)
|
1232
1236
|
disabled_clouds = (clouds_need_recheck -
|
1233
1237
|
{str(c) for c in enabled_clouds})
|
@@ -1269,6 +1273,7 @@ def _fill_in_launchable_resources(
|
|
1269
1273
|
a cloud that is not enabled.
|
1270
1274
|
"""
|
1271
1275
|
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
1276
|
+
capability=sky_cloud.CloudCapability.COMPUTE,
|
1272
1277
|
raise_if_no_cloud_access=True)
|
1273
1278
|
|
1274
1279
|
launchable: Dict[resources_lib.Resources, List[resources_lib.Resources]] = (
|
sky/resources.py
CHANGED
@@ -10,6 +10,7 @@ from sky import clouds
|
|
10
10
|
from sky import exceptions
|
11
11
|
from sky import sky_logging
|
12
12
|
from sky import skypilot_config
|
13
|
+
from sky.clouds import cloud as sky_cloud
|
13
14
|
from sky.clouds import service_catalog
|
14
15
|
from sky.provision import docker_utils
|
15
16
|
from sky.provision.kubernetes import utils as kubernetes_utils
|
@@ -677,6 +678,7 @@ class Resources:
|
|
677
678
|
# cloud corresponds to region/zone, errors out.
|
678
679
|
valid_clouds = []
|
679
680
|
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
681
|
+
sky_cloud.CloudCapability.COMPUTE,
|
680
682
|
raise_if_no_cloud_access=True)
|
681
683
|
cloud_to_errors = {}
|
682
684
|
for cloud in enabled_clouds:
|
@@ -796,6 +798,7 @@ class Resources:
|
|
796
798
|
# If cloud not specified
|
797
799
|
valid_clouds = []
|
798
800
|
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
801
|
+
sky_cloud.CloudCapability.COMPUTE,
|
799
802
|
raise_if_no_cloud_access=True)
|
800
803
|
for cloud in enabled_clouds:
|
801
804
|
if cloud.instance_type_exists(self._instance_type):
|
@@ -991,6 +994,7 @@ class Resources:
|
|
991
994
|
else:
|
992
995
|
at_least_one_cloud_supports_ports = False
|
993
996
|
for cloud in sky_check.get_cached_enabled_clouds_or_refresh(
|
997
|
+
sky_cloud.CloudCapability.COMPUTE,
|
994
998
|
raise_if_no_cloud_access=True):
|
995
999
|
try:
|
996
1000
|
cloud.check_features_are_supported(
|
@@ -1020,7 +1024,8 @@ class Resources:
|
|
1020
1024
|
else:
|
1021
1025
|
# If no specific cloud is set, validate label against ALL clouds.
|
1022
1026
|
# The label will be dropped if invalid for any one of the cloud
|
1023
|
-
validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
1027
|
+
validated_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
1028
|
+
sky_cloud.CloudCapability.COMPUTE)
|
1024
1029
|
invalid_table = log_utils.create_table(['Label', 'Reason'])
|
1025
1030
|
for key, value in self._labels.items():
|
1026
1031
|
for cloud in validated_clouds:
|
sky/setup_files/dependencies.py
CHANGED
@@ -150,7 +150,9 @@ extras_require: Dict[str, List[str]] = {
|
|
150
150
|
# docs instead.
|
151
151
|
# 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
|
152
152
|
],
|
153
|
-
'nebius': [
|
153
|
+
'nebius': [
|
154
|
+
'nebius>=0.2.0',
|
155
|
+
] + aws_dependencies
|
154
156
|
}
|
155
157
|
|
156
158
|
# Nebius needs python3.10. If python 3.9 [all] will not install nebius
|
sky/task.py
CHANGED
@@ -974,8 +974,8 @@ class Task:
|
|
974
974
|
# assert len(self.resources) == 1, self.resources
|
975
975
|
storage_cloud = None
|
976
976
|
|
977
|
-
|
978
|
-
storage_lib.
|
977
|
+
enabled_storage_cloud_names = (
|
978
|
+
storage_lib.get_cached_enabled_storage_cloud_names_or_refresh(
|
979
979
|
raise_if_no_cloud_access=True))
|
980
980
|
|
981
981
|
if self.best_resources is not None:
|
@@ -987,13 +987,13 @@ class Task:
|
|
987
987
|
storage_region = resources.region
|
988
988
|
|
989
989
|
if storage_cloud is not None:
|
990
|
-
if str(storage_cloud) not in
|
990
|
+
if str(storage_cloud) not in enabled_storage_cloud_names:
|
991
991
|
storage_cloud = None
|
992
992
|
|
993
993
|
storage_cloud_str = None
|
994
994
|
if storage_cloud is None:
|
995
|
-
storage_cloud_str =
|
996
|
-
assert storage_cloud_str is not None,
|
995
|
+
storage_cloud_str = enabled_storage_cloud_names[0]
|
996
|
+
assert storage_cloud_str is not None, enabled_storage_cloud_names[0]
|
997
997
|
storage_region = None # Use default region in the Store class
|
998
998
|
else:
|
999
999
|
storage_cloud_str = str(storage_cloud)
|
@@ -1103,6 +1103,17 @@ class Task:
|
|
1103
1103
|
self.update_file_mounts({
|
1104
1104
|
mnt_path: blob_path,
|
1105
1105
|
})
|
1106
|
+
elif store_type is storage_lib.StoreType.NEBIUS:
|
1107
|
+
if storage.source is not None and not isinstance(
|
1108
|
+
storage.source,
|
1109
|
+
list) and storage.source.startswith('nebius://'):
|
1110
|
+
blob_path = storage.source
|
1111
|
+
else:
|
1112
|
+
blob_path = 'nebius://' + storage.name
|
1113
|
+
blob_path = storage.get_bucket_sub_path_prefix(blob_path)
|
1114
|
+
self.update_file_mounts({
|
1115
|
+
mnt_path: blob_path,
|
1116
|
+
})
|
1106
1117
|
elif store_type is storage_lib.StoreType.IBM:
|
1107
1118
|
if isinstance(storage.source,
|
1108
1119
|
str) and storage.source.startswith('cos://'):
|
sky/utils/command_runner.py
CHANGED
@@ -29,6 +29,8 @@ RSYNC_DISPLAY_OPTION = '-Pavz'
|
|
29
29
|
# Note that "-" is mandatory for rsync and means all patterns in the ignore
|
30
30
|
# files are treated as *exclude* patterns. Non-exclude patterns, e.g., "!
|
31
31
|
# do_not_exclude" doesn't work, even though git allows it.
|
32
|
+
# TODO(cooperc): Avoid using this, and prefer utils in storage_utils instead for
|
33
|
+
# consistency between bucket upload and rsync.
|
32
34
|
RSYNC_FILTER_SKYIGNORE = f'--filter=\'dir-merge,- {constants.SKY_IGNORE_FILE}\''
|
33
35
|
RSYNC_FILTER_GITIGNORE = f'--filter=\'dir-merge,- {constants.GIT_IGNORE_FILE}\''
|
34
36
|
# The git exclude file to support.
|
sky/utils/controller_utils.py
CHANGED
@@ -19,6 +19,7 @@ from sky import resources
|
|
19
19
|
from sky import sky_logging
|
20
20
|
from sky import skypilot_config
|
21
21
|
from sky.adaptors import cloudflare
|
22
|
+
from sky.clouds import cloud as sky_cloud
|
22
23
|
from sky.clouds import gcp
|
23
24
|
from sky.data import data_utils
|
24
25
|
from sky.data import storage as storage_lib
|
@@ -216,9 +217,11 @@ def _get_cloud_dependencies_installation_commands(
|
|
216
217
|
f'{constants.SKY_UV_INSTALL_CMD} >/dev/null 2>&1')
|
217
218
|
|
218
219
|
enabled_compute_clouds = set(
|
219
|
-
sky_check.get_cached_enabled_clouds_or_refresh(
|
220
|
+
sky_check.get_cached_enabled_clouds_or_refresh(
|
221
|
+
sky_cloud.CloudCapability.COMPUTE))
|
220
222
|
enabled_storage_clouds = set(
|
221
|
-
sky_check.
|
223
|
+
sky_check.get_cached_enabled_clouds_or_refresh(
|
224
|
+
sky_cloud.CloudCapability.STORAGE))
|
222
225
|
enabled_clouds = enabled_compute_clouds.union(enabled_storage_clouds)
|
223
226
|
|
224
227
|
for cloud in enabled_clouds:
|
@@ -281,7 +284,7 @@ def _get_cloud_dependencies_installation_commands(
|
|
281
284
|
python_packages.update(cloud_python_dependencies)
|
282
285
|
|
283
286
|
if (cloudflare.NAME
|
284
|
-
in storage_lib.
|
287
|
+
in storage_lib.get_cached_enabled_storage_cloud_names_or_refresh()):
|
285
288
|
python_packages.update(dependencies.extras_require['cloudflare'])
|
286
289
|
|
287
290
|
packages_string = ' '.join([f'"{package}"' for package in python_packages])
|
@@ -812,8 +815,8 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
|
|
812
815
|
(store_type, bucket_name, sub_path, storage_account_name, region) = (
|
813
816
|
storage_lib.StoreType.get_fields_from_store_url(bucket_wth_prefix))
|
814
817
|
cloud_str = store_type.to_cloud()
|
815
|
-
if (cloud_str not in
|
816
|
-
|
818
|
+
if (cloud_str not in storage_lib.
|
819
|
+
get_cached_enabled_storage_cloud_names_or_refresh()):
|
817
820
|
with ux_utils.print_exception_no_traceback():
|
818
821
|
raise ValueError(
|
819
822
|
f'`{task_type}.bucket` is specified in SkyPilot config '
|
@@ -8,6 +8,7 @@ from typing import List, Optional
|
|
8
8
|
from sky import check as sky_check
|
9
9
|
from sky import sky_logging
|
10
10
|
from sky.backends import backend_utils
|
11
|
+
from sky.clouds import cloud as sky_cloud
|
11
12
|
from sky.provision.kubernetes import utils as kubernetes_utils
|
12
13
|
from sky.skylet import constants
|
13
14
|
from sky.skylet import log_lib
|
@@ -169,7 +170,7 @@ def deploy_local_cluster(gpus: bool):
|
|
169
170
|
with rich_utils.safe_status('[bold cyan]Running sky check...'):
|
170
171
|
sky_check.check(clouds=['kubernetes'],
|
171
172
|
quiet=True,
|
172
|
-
capability=
|
173
|
+
capability=sky_cloud.CloudCapability.COMPUTE)
|
173
174
|
if cluster_created:
|
174
175
|
# Prepare completion message which shows CPU and GPU count
|
175
176
|
# Get number of CPUs
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: skypilot-nightly
|
3
|
-
Version: 1.0.0.
|
3
|
+
Version: 1.0.0.dev20250321
|
4
4
|
Summary: SkyPilot: An intercloud broker for the clouds
|
5
5
|
Author: SkyPilot Team
|
6
6
|
License: Apache 2.0
|
@@ -108,6 +108,11 @@ Provides-Extra: vsphere
|
|
108
108
|
Requires-Dist: pyvmomi==8.0.1.0.2; extra == "vsphere"
|
109
109
|
Provides-Extra: nebius
|
110
110
|
Requires-Dist: nebius>=0.2.0; extra == "nebius"
|
111
|
+
Requires-Dist: urllib3<2; extra == "nebius"
|
112
|
+
Requires-Dist: awscli>=1.27.10; extra == "nebius"
|
113
|
+
Requires-Dist: botocore>=1.29.10; extra == "nebius"
|
114
|
+
Requires-Dist: boto3>=1.26.1; extra == "nebius"
|
115
|
+
Requires-Dist: colorama<0.4.5; extra == "nebius"
|
111
116
|
Provides-Extra: all
|
112
117
|
Requires-Dist: urllib3<2; extra == "all"
|
113
118
|
Requires-Dist: awscli>=1.27.10; extra == "all"
|
@@ -151,6 +156,11 @@ Requires-Dist: azure-common; extra == "all"
|
|
151
156
|
Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
|
152
157
|
Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
|
153
158
|
Requires-Dist: nebius>=0.2.0; extra == "all"
|
159
|
+
Requires-Dist: urllib3<2; extra == "all"
|
160
|
+
Requires-Dist: awscli>=1.27.10; extra == "all"
|
161
|
+
Requires-Dist: botocore>=1.29.10; extra == "all"
|
162
|
+
Requires-Dist: boto3>=1.26.1; extra == "all"
|
163
|
+
Requires-Dist: colorama<0.4.5; extra == "all"
|
154
164
|
Dynamic: author
|
155
165
|
Dynamic: classifier
|
156
166
|
Dynamic: description
|