skypilot-nightly 1.0.0.dev20250319__py3-none-any.whl → 1.0.0.dev20250321__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sky/__init__.py +2 -2
- sky/adaptors/cloudflare.py +19 -3
- sky/adaptors/kubernetes.py +2 -1
- sky/adaptors/nebius.py +128 -6
- sky/backends/cloud_vm_ray_backend.py +3 -1
- sky/benchmark/benchmark_utils.py +3 -2
- sky/check.py +89 -55
- sky/cloud_stores.py +66 -0
- sky/clouds/aws.py +14 -2
- sky/clouds/azure.py +13 -1
- sky/clouds/cloud.py +37 -2
- sky/clouds/cudo.py +3 -2
- sky/clouds/do.py +3 -2
- sky/clouds/fluidstack.py +3 -2
- sky/clouds/gcp.py +55 -34
- sky/clouds/ibm.py +15 -1
- sky/clouds/kubernetes.py +3 -1
- sky/clouds/lambda_cloud.py +3 -1
- sky/clouds/nebius.py +7 -3
- sky/clouds/oci.py +15 -1
- sky/clouds/paperspace.py +3 -2
- sky/clouds/runpod.py +7 -1
- sky/clouds/scp.py +3 -1
- sky/clouds/service_catalog/kubernetes_catalog.py +3 -1
- sky/clouds/utils/gcp_utils.py +11 -1
- sky/clouds/vast.py +3 -2
- sky/clouds/vsphere.py +3 -2
- sky/core.py +6 -2
- sky/data/data_transfer.py +75 -0
- sky/data/data_utils.py +34 -0
- sky/data/mounting_utils.py +18 -0
- sky/data/storage.py +542 -16
- sky/data/storage_utils.py +102 -84
- sky/exceptions.py +2 -0
- sky/global_user_state.py +15 -6
- sky/jobs/server/core.py +1 -1
- sky/jobs/utils.py +5 -0
- sky/optimizer.py +8 -2
- sky/provision/gcp/config.py +3 -3
- sky/provision/gcp/constants.py +16 -2
- sky/provision/gcp/instance.py +4 -1
- sky/provision/kubernetes/utils.py +26 -21
- sky/resources.py +6 -1
- sky/serve/replica_managers.py +10 -1
- sky/setup_files/dependencies.py +3 -1
- sky/task.py +16 -5
- sky/utils/command_runner.py +2 -0
- sky/utils/controller_utils.py +13 -4
- sky/utils/kubernetes/kubernetes_deploy_utils.py +4 -1
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/METADATA +13 -2
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/RECORD +55 -55
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/WHEEL +1 -1
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info/licenses}/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250319.dist-info → skypilot_nightly-1.0.0.dev20250321.dist-info}/top_level.txt +0 -0
sky/clouds/cloud.py
CHANGED
@@ -13,6 +13,8 @@ import math
|
|
13
13
|
import typing
|
14
14
|
from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
|
15
15
|
|
16
|
+
from typing_extensions import assert_never
|
17
|
+
|
16
18
|
from sky import exceptions
|
17
19
|
from sky import skypilot_config
|
18
20
|
from sky.clouds import service_catalog
|
@@ -48,6 +50,16 @@ class CloudImplementationFeatures(enum.Enum):
|
|
48
50
|
AUTO_TERMINATE = 'auto_terminate' # Pod/VM can stop or down itself
|
49
51
|
|
50
52
|
|
53
|
+
class CloudCapability(enum.Enum):
|
54
|
+
# Compute capability.
|
55
|
+
COMPUTE = 'compute'
|
56
|
+
# Storage capability.
|
57
|
+
STORAGE = 'storage'
|
58
|
+
|
59
|
+
|
60
|
+
ALL_CAPABILITIES = [CloudCapability.COMPUTE, CloudCapability.STORAGE]
|
61
|
+
|
62
|
+
|
51
63
|
class Region(collections.namedtuple('Region', ['name'])):
|
52
64
|
"""A region."""
|
53
65
|
name: str
|
@@ -435,13 +447,36 @@ class Cloud:
|
|
435
447
|
return {reservation: 0 for reservation in specific_reservations}
|
436
448
|
|
437
449
|
@classmethod
|
438
|
-
def check_credentials(
|
450
|
+
def check_credentials(
|
451
|
+
cls,
|
452
|
+
cloud_capability: CloudCapability) -> Tuple[bool, Optional[str]]:
|
439
453
|
"""Checks if the user has access credentials to this cloud.
|
440
454
|
|
441
455
|
Returns a boolean of whether the user can access this cloud, and a
|
442
456
|
string describing the reason if the user cannot access.
|
457
|
+
|
458
|
+
Raises NotSupportedError if the capability is
|
459
|
+
not supported by this cloud.
|
443
460
|
"""
|
444
|
-
|
461
|
+
if cloud_capability == CloudCapability.COMPUTE:
|
462
|
+
return cls._check_compute_credentials()
|
463
|
+
elif cloud_capability == CloudCapability.STORAGE:
|
464
|
+
return cls._check_storage_credentials()
|
465
|
+
assert_never(cloud_capability)
|
466
|
+
|
467
|
+
@classmethod
|
468
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
469
|
+
"""Checks if the user has access credentials to
|
470
|
+
this cloud's compute service."""
|
471
|
+
raise exceptions.NotSupportedError(
|
472
|
+
f'{cls._REPR} does not support {CloudCapability.COMPUTE.value}.')
|
473
|
+
|
474
|
+
@classmethod
|
475
|
+
def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
|
476
|
+
"""Checks if the user has access credentials to
|
477
|
+
this cloud's storage service."""
|
478
|
+
raise exceptions.NotSupportedError(
|
479
|
+
f'{cls._REPR} does not support {CloudCapability.STORAGE.value}.')
|
445
480
|
|
446
481
|
# TODO(zhwu): Make the return type immutable.
|
447
482
|
@classmethod
|
sky/clouds/cudo.py
CHANGED
@@ -267,8 +267,9 @@ class Cudo(clouds.Cloud):
|
|
267
267
|
fuzzy_candidate_list, None)
|
268
268
|
|
269
269
|
@classmethod
|
270
|
-
def
|
271
|
-
|
270
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
271
|
+
"""Checks if the user has access credentials to
|
272
|
+
Cudo's compute service."""
|
272
273
|
try:
|
273
274
|
# pylint: disable=import-outside-toplevel,unused-import
|
274
275
|
from cudo_compute import cudo_api
|
sky/clouds/do.py
CHANGED
@@ -259,8 +259,9 @@ class DO(clouds.Cloud):
|
|
259
259
|
fuzzy_candidate_list, None)
|
260
260
|
|
261
261
|
@classmethod
|
262
|
-
def
|
263
|
-
"""Verify that the user has valid credentials for
|
262
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
263
|
+
"""Verify that the user has valid credentials for
|
264
|
+
DO's compute service."""
|
264
265
|
|
265
266
|
try:
|
266
267
|
do.exceptions()
|
sky/clouds/fluidstack.py
CHANGED
@@ -255,8 +255,9 @@ class Fluidstack(clouds.Cloud):
|
|
255
255
|
fuzzy_candidate_list, None)
|
256
256
|
|
257
257
|
@classmethod
|
258
|
-
def
|
259
|
-
|
258
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
259
|
+
"""Checks if the user has access credentials to
|
260
|
+
FluidStack's compute service."""
|
260
261
|
try:
|
261
262
|
assert os.path.exists(
|
262
263
|
os.path.expanduser(fluidstack_utils.FLUIDSTACK_API_KEY_PATH))
|
sky/clouds/gcp.py
CHANGED
@@ -124,6 +124,7 @@ def _run_output(cmd):
|
|
124
124
|
|
125
125
|
|
126
126
|
def is_api_disabled(endpoint: str, project_id: str) -> bool:
|
127
|
+
# requires serviceusage.services.list
|
127
128
|
proc = subprocess.run((f'gcloud services list --project {project_id} '
|
128
129
|
f' | grep {endpoint}.googleapis.com'),
|
129
130
|
check=False,
|
@@ -718,7 +719,28 @@ class GCP(clouds.Cloud):
|
|
718
719
|
return DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH
|
719
720
|
|
720
721
|
@classmethod
|
721
|
-
def
|
722
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
723
|
+
"""Checks if the user has access credentials to this cloud's compute service."""
|
724
|
+
return cls._check_credentials(
|
725
|
+
[
|
726
|
+
('compute', 'Compute Engine'),
|
727
|
+
('cloudresourcemanager', 'Cloud Resource Manager'),
|
728
|
+
('iam', 'Identity and Access Management (IAM)'),
|
729
|
+
('tpu', 'Cloud TPU'), # Keep as final element.
|
730
|
+
],
|
731
|
+
gcp_utils.get_minimal_compute_permissions())
|
732
|
+
|
733
|
+
@classmethod
|
734
|
+
def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
|
735
|
+
"""Checks if the user has access credentials to this cloud's storage service."""
|
736
|
+
return cls._check_credentials(
|
737
|
+
[('storage', 'Cloud Storage')],
|
738
|
+
gcp_utils.get_minimal_storage_permissions())
|
739
|
+
|
740
|
+
@classmethod
|
741
|
+
def _check_credentials(
|
742
|
+
cls, apis: List[Tuple[str, str]],
|
743
|
+
gcp_minimal_permissions: List[str]) -> Tuple[bool, Optional[str]]:
|
722
744
|
"""Checks if the user has access credentials to this cloud."""
|
723
745
|
try:
|
724
746
|
# pylint: disable=import-outside-toplevel,unused-import
|
@@ -783,13 +805,37 @@ class GCP(clouds.Cloud):
|
|
783
805
|
f'{cls._INDENT_PREFIX}Details: '
|
784
806
|
f'{common_utils.format_exception(e, use_bracket=True)}')
|
785
807
|
|
786
|
-
#
|
787
|
-
|
788
|
-
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
808
|
+
# pylint: disable=import-outside-toplevel,unused-import
|
809
|
+
import google.auth
|
810
|
+
|
811
|
+
# This takes user's credential info from "~/.config/gcloud/application_default_credentials.json". # pylint: disable=line-too-long
|
812
|
+
credentials, project = google.auth.default()
|
813
|
+
crm = gcp.build('cloudresourcemanager',
|
814
|
+
'v1',
|
815
|
+
credentials=credentials,
|
816
|
+
cache_discovery=False)
|
817
|
+
permissions = {'permissions': gcp_minimal_permissions}
|
818
|
+
request = crm.projects().testIamPermissions(resource=project,
|
819
|
+
body=permissions)
|
820
|
+
try:
|
821
|
+
ret_permissions = request.execute().get('permissions', [])
|
822
|
+
except gcp.gcp_auth_refresh_error_exception() as e:
|
823
|
+
return False, common_utils.format_exception(e, use_bracket=True)
|
824
|
+
|
825
|
+
diffs = set(gcp_minimal_permissions).difference(set(ret_permissions))
|
826
|
+
if diffs:
|
827
|
+
identity_str = identity[0] if identity else None
|
828
|
+
return False, (
|
829
|
+
'The following permissions are not enabled for the current '
|
830
|
+
f'GCP identity ({identity_str}):\n '
|
831
|
+
f'{diffs}\n '
|
832
|
+
'For more details, visit: https://docs.skypilot.co/en/latest/cloud-setup/cloud-permissions/gcp.html') # pylint: disable=line-too-long
|
833
|
+
|
834
|
+
# This code must be executed after the iam check above,
|
835
|
+
# as the check below for api enablement itself needs:
|
836
|
+
# - serviceusage.services.enable
|
837
|
+
# - serviceusage.services.list
|
838
|
+
# iam permissions.
|
793
839
|
enabled_api = False
|
794
840
|
for endpoint, display_name in apis:
|
795
841
|
if is_api_disabled(endpoint, project_id):
|
@@ -801,6 +847,7 @@ class GCP(clouds.Cloud):
|
|
801
847
|
suffix = ' (free of charge)'
|
802
848
|
print(f'\nEnabling {display_name} API{suffix}...')
|
803
849
|
t1 = time.time()
|
850
|
+
# requires serviceusage.services.enable
|
804
851
|
proc = subprocess.run(
|
805
852
|
f'gcloud services enable {endpoint}.googleapis.com '
|
806
853
|
f'--project {project_id}',
|
@@ -830,32 +877,6 @@ class GCP(clouds.Cloud):
|
|
830
877
|
'effect. If any SkyPilot commands/calls failed, retry after '
|
831
878
|
'some time.')
|
832
879
|
|
833
|
-
# pylint: disable=import-outside-toplevel,unused-import
|
834
|
-
import google.auth
|
835
|
-
|
836
|
-
# This takes user's credential info from "~/.config/gcloud/application_default_credentials.json". # pylint: disable=line-too-long
|
837
|
-
credentials, project = google.auth.default()
|
838
|
-
crm = gcp.build('cloudresourcemanager',
|
839
|
-
'v1',
|
840
|
-
credentials=credentials,
|
841
|
-
cache_discovery=False)
|
842
|
-
gcp_minimal_permissions = gcp_utils.get_minimal_permissions()
|
843
|
-
permissions = {'permissions': gcp_minimal_permissions}
|
844
|
-
request = crm.projects().testIamPermissions(resource=project,
|
845
|
-
body=permissions)
|
846
|
-
try:
|
847
|
-
ret_permissions = request.execute().get('permissions', [])
|
848
|
-
except gcp.gcp_auth_refresh_error_exception() as e:
|
849
|
-
return False, common_utils.format_exception(e, use_bracket=True)
|
850
|
-
|
851
|
-
diffs = set(gcp_minimal_permissions).difference(set(ret_permissions))
|
852
|
-
if diffs:
|
853
|
-
identity_str = identity[0] if identity else None
|
854
|
-
return False, (
|
855
|
-
'The following permissions are not enabled for the current '
|
856
|
-
f'GCP identity ({identity_str}):\n '
|
857
|
-
f'{diffs}\n '
|
858
|
-
'For more details, visit: https://docs.skypilot.co/en/latest/cloud-setup/cloud-permissions/gcp.html') # pylint: disable=line-too-long
|
859
880
|
return True, None
|
860
881
|
|
861
882
|
def get_credential_file_mounts(self) -> Dict[str, str]:
|
sky/clouds/ibm.py
CHANGED
@@ -395,7 +395,21 @@ class IBM(clouds.Cloud):
|
|
395
395
|
return image_size
|
396
396
|
|
397
397
|
@classmethod
|
398
|
-
def
|
398
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
399
|
+
"""Checks if the user has access credentials to
|
400
|
+
IBM's compute service."""
|
401
|
+
return cls._check_credentials()
|
402
|
+
|
403
|
+
@classmethod
|
404
|
+
def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
|
405
|
+
"""Checks if the user has access credentials to
|
406
|
+
IBM's storage service."""
|
407
|
+
# TODO(seungjin): Implement separate check for
|
408
|
+
# if the user has access to IBM COS.
|
409
|
+
return cls._check_credentials()
|
410
|
+
|
411
|
+
@classmethod
|
412
|
+
def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
|
399
413
|
"""Checks if the user has access credentials to this cloud."""
|
400
414
|
|
401
415
|
required_fields = ['iam_api_key', 'resource_group_id']
|
sky/clouds/kubernetes.py
CHANGED
@@ -655,7 +655,9 @@ class Kubernetes(clouds.Cloud):
|
|
655
655
|
[], None)
|
656
656
|
|
657
657
|
@classmethod
|
658
|
-
def
|
658
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
659
|
+
"""Checks if the user has access credentials to
|
660
|
+
Kubernetes."""
|
659
661
|
# Test using python API
|
660
662
|
try:
|
661
663
|
existing_allowed_contexts = cls.existing_allowed_contexts()
|
sky/clouds/lambda_cloud.py
CHANGED
@@ -241,7 +241,9 @@ class Lambda(clouds.Cloud):
|
|
241
241
|
fuzzy_candidate_list, None)
|
242
242
|
|
243
243
|
@classmethod
|
244
|
-
def
|
244
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
245
|
+
"""Checks if the user has access credentials to
|
246
|
+
Lambda's compute service."""
|
245
247
|
try:
|
246
248
|
lambda_utils.LambdaCloudClient().list_instances()
|
247
249
|
except (AssertionError, KeyError, lambda_utils.LambdaCloudError):
|
sky/clouds/nebius.py
CHANGED
@@ -250,8 +250,9 @@ class Nebius(clouds.Cloud):
|
|
250
250
|
fuzzy_candidate_list, None)
|
251
251
|
|
252
252
|
@classmethod
|
253
|
-
def
|
254
|
-
"""
|
253
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
254
|
+
"""Checks if the user has access credentials to
|
255
|
+
Nebius's compute service."""
|
255
256
|
logging.debug('Nebius cloud check credentials')
|
256
257
|
token_cred_msg = (' Credentials can be set up by running: \n'\
|
257
258
|
f' $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n'\
|
@@ -279,10 +280,13 @@ class Nebius(clouds.Cloud):
|
|
279
280
|
return True, None
|
280
281
|
|
281
282
|
def get_credential_file_mounts(self) -> Dict[str, str]:
|
282
|
-
|
283
|
+
credential_file_mounts = {
|
283
284
|
f'~/.nebius/{filename}': f'~/.nebius/{filename}'
|
284
285
|
for filename in _CREDENTIAL_FILES
|
285
286
|
}
|
287
|
+
credential_file_mounts['~/.aws/credentials'] = '~/.aws/credentials'
|
288
|
+
|
289
|
+
return credential_file_mounts
|
286
290
|
|
287
291
|
@classmethod
|
288
292
|
def get_current_user_identity(cls) -> Optional[List[str]]:
|
sky/clouds/oci.py
CHANGED
@@ -396,7 +396,21 @@ class OCI(clouds.Cloud):
|
|
396
396
|
fuzzy_candidate_list, None)
|
397
397
|
|
398
398
|
@classmethod
|
399
|
-
def
|
399
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
400
|
+
"""Checks if the user has access credentials to
|
401
|
+
OCI's compute service."""
|
402
|
+
return cls._check_credentials()
|
403
|
+
|
404
|
+
@classmethod
|
405
|
+
def _check_storage_credentials(cls) -> Tuple[bool, Optional[str]]:
|
406
|
+
"""Checks if the user has access credentials to
|
407
|
+
OCI's storage service."""
|
408
|
+
# TODO(seungjin): Implement separate check for
|
409
|
+
# if the user has access to OCI Object Storage.
|
410
|
+
return cls._check_credentials()
|
411
|
+
|
412
|
+
@classmethod
|
413
|
+
def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
|
400
414
|
"""Checks if the user has access credentials to this cloud."""
|
401
415
|
|
402
416
|
short_credential_help_str = (
|
sky/clouds/paperspace.py
CHANGED
@@ -249,8 +249,9 @@ class Paperspace(clouds.Cloud):
|
|
249
249
|
fuzzy_candidate_list, None)
|
250
250
|
|
251
251
|
@classmethod
|
252
|
-
def
|
253
|
-
"""
|
252
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
253
|
+
"""Checks if the user has access credentials to
|
254
|
+
Paperspace's compute service."""
|
254
255
|
try:
|
255
256
|
# attempt to make a CURL request for listing instances
|
256
257
|
utils.PaperspaceCloudClient().list_instances()
|
sky/clouds/runpod.py
CHANGED
@@ -248,7 +248,13 @@ class RunPod(clouds.Cloud):
|
|
248
248
|
fuzzy_candidate_list, None)
|
249
249
|
|
250
250
|
@classmethod
|
251
|
-
def
|
251
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
252
|
+
"""Checks if the user has access credentials to
|
253
|
+
RunPod's compute service."""
|
254
|
+
return cls._check_credentials()
|
255
|
+
|
256
|
+
@classmethod
|
257
|
+
def _check_credentials(cls) -> Tuple[bool, Optional[str]]:
|
252
258
|
""" Verify that the user has valid credentials for RunPod. """
|
253
259
|
try:
|
254
260
|
import runpod # pylint: disable=import-outside-toplevel
|
sky/clouds/scp.py
CHANGED
@@ -312,7 +312,9 @@ class SCP(clouds.Cloud):
|
|
312
312
|
fuzzy_candidate_list, None)
|
313
313
|
|
314
314
|
@classmethod
|
315
|
-
def
|
315
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
316
|
+
"""Checks if the user has access credentials to
|
317
|
+
SCP's compute service."""
|
316
318
|
try:
|
317
319
|
scp_utils.SCPClient().list_instances()
|
318
320
|
except (AssertionError, KeyError, scp_utils.SCPClientError,
|
@@ -12,6 +12,7 @@ from sky import clouds as sky_clouds
|
|
12
12
|
from sky import sky_logging
|
13
13
|
from sky.adaptors import common as adaptors_common
|
14
14
|
from sky.adaptors import kubernetes
|
15
|
+
from sky.clouds import cloud
|
15
16
|
from sky.clouds.service_catalog import CloudFilter
|
16
17
|
from sky.clouds.service_catalog import common
|
17
18
|
from sky.provision.kubernetes import utils as kubernetes_utils
|
@@ -132,7 +133,8 @@ def _list_accelerators(
|
|
132
133
|
|
133
134
|
# First check if Kubernetes is enabled. This ensures k8s python client is
|
134
135
|
# installed. Do not put any k8s-specific logic before this check.
|
135
|
-
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
136
|
+
enabled_clouds = sky_check.get_cached_enabled_clouds_or_refresh(
|
137
|
+
cloud.CloudCapability.COMPUTE)
|
136
138
|
if not sky_clouds.cloud_in_iterable(sky_clouds.Kubernetes(),
|
137
139
|
enabled_clouds):
|
138
140
|
return {}, {}, {}
|
sky/clouds/utils/gcp_utils.py
CHANGED
@@ -167,7 +167,7 @@ def _list_reservations_for_instance_type(
|
|
167
167
|
return [GCPReservation.from_dict(r) for r in json.loads(stdout)]
|
168
168
|
|
169
169
|
|
170
|
-
def
|
170
|
+
def get_minimal_compute_permissions() -> List[str]:
|
171
171
|
permissions = copy.copy(constants.VM_MINIMAL_PERMISSIONS)
|
172
172
|
if skypilot_config.get_nested(('gcp', 'vpc_name'), None) is None:
|
173
173
|
# If custom VPC is not specified, permissions to modify network are
|
@@ -179,4 +179,14 @@ def get_minimal_permissions() -> List[str]:
|
|
179
179
|
skypilot_config.get_nested(('gcp', 'specific_reservations'), [])):
|
180
180
|
permissions += constants.RESERVATION_PERMISSIONS
|
181
181
|
|
182
|
+
permissions += constants.GCP_MINIMAL_PERMISSIONS
|
183
|
+
|
184
|
+
return permissions
|
185
|
+
|
186
|
+
|
187
|
+
def get_minimal_storage_permissions() -> List[str]:
|
188
|
+
permissions = copy.copy(constants.STORAGE_MINIMAL_PERMISSIONS)
|
189
|
+
|
190
|
+
permissions += constants.GCP_MINIMAL_PERMISSIONS
|
191
|
+
|
182
192
|
return permissions
|
sky/clouds/vast.py
CHANGED
@@ -234,8 +234,9 @@ class Vast(clouds.Cloud):
|
|
234
234
|
fuzzy_candidate_list, None)
|
235
235
|
|
236
236
|
@classmethod
|
237
|
-
def
|
238
|
-
"""
|
237
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
238
|
+
"""Checks if the user has valid credentials for
|
239
|
+
Vast's compute service. """
|
239
240
|
try:
|
240
241
|
import vastai_sdk as _vast # pylint: disable=import-outside-toplevel
|
241
242
|
vast = _vast.VastAI()
|
sky/clouds/vsphere.py
CHANGED
@@ -254,8 +254,9 @@ class Vsphere(clouds.Cloud):
|
|
254
254
|
fuzzy_candidate_list, None)
|
255
255
|
|
256
256
|
@classmethod
|
257
|
-
def
|
258
|
-
"""Checks if the user has access credentials to
|
257
|
+
def _check_compute_credentials(cls) -> Tuple[bool, Optional[str]]:
|
258
|
+
"""Checks if the user has access credentials to
|
259
|
+
vSphere's compute service."""
|
259
260
|
|
260
261
|
try:
|
261
262
|
# pylint: disable=import-outside-toplevel,unused-import
|
sky/core.py
CHANGED
@@ -19,6 +19,7 @@ from sky import optimizer
|
|
19
19
|
from sky import sky_logging
|
20
20
|
from sky import task as task_lib
|
21
21
|
from sky.backends import backend_utils
|
22
|
+
from sky.clouds import cloud as sky_cloud
|
22
23
|
from sky.clouds import service_catalog
|
23
24
|
from sky.jobs.server import core as managed_jobs_core
|
24
25
|
from sky.provision.kubernetes import constants as kubernetes_constants
|
@@ -1001,7 +1002,8 @@ def storage_delete(name: str) -> None:
|
|
1001
1002
|
# ===================
|
1002
1003
|
@usage_lib.entrypoint
|
1003
1004
|
def enabled_clouds() -> List[clouds.Cloud]:
|
1004
|
-
return global_user_state.get_cached_enabled_clouds(
|
1005
|
+
return global_user_state.get_cached_enabled_clouds(
|
1006
|
+
sky_cloud.CloudCapability.COMPUTE)
|
1005
1007
|
|
1006
1008
|
|
1007
1009
|
@usage_lib.entrypoint
|
@@ -1134,7 +1136,9 @@ def local_down() -> None:
|
|
1134
1136
|
# Run sky check
|
1135
1137
|
with rich_utils.safe_status(
|
1136
1138
|
ux_utils.spinner_message('Running sky check...')):
|
1137
|
-
sky_check.check(clouds=['kubernetes'],
|
1139
|
+
sky_check.check(clouds=['kubernetes'],
|
1140
|
+
quiet=True,
|
1141
|
+
capability=sky_cloud.CloudCapability.COMPUTE)
|
1138
1142
|
logger.info(
|
1139
1143
|
ux_utils.finishing_message('Local cluster removed.',
|
1140
1144
|
log_path=log_path,
|
sky/data/data_transfer.py
CHANGED
@@ -14,6 +14,7 @@ Currently implemented:
|
|
14
14
|
TODO:
|
15
15
|
- All combinations of Azure Transfer
|
16
16
|
- All combinations of R2 Transfer
|
17
|
+
- All combinations of Nebius Transfer
|
17
18
|
- GCS -> S3
|
18
19
|
"""
|
19
20
|
import json
|
@@ -135,6 +136,35 @@ def s3_to_r2(s3_bucket_name: str, r2_bucket_name: str) -> None:
|
|
135
136
|
'a local source for the storage object.')
|
136
137
|
|
137
138
|
|
139
|
+
def s3_to_nebius(s3_bucket_name: str, nebius_bucket_name: str) -> None:
|
140
|
+
"""Creates a one-time transfer from Amazon S3 to Nebius Object Storage.
|
141
|
+
|
142
|
+
it will block until the transfer is complete.
|
143
|
+
|
144
|
+
Args:
|
145
|
+
s3_bucket_name: str; Name of the Amazon S3 Bucket
|
146
|
+
nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
|
147
|
+
"""
|
148
|
+
raise NotImplementedError('Moving data directly from clouds to Nebius is '
|
149
|
+
'currently not supported. Please specify '
|
150
|
+
'a local source for the storage object.')
|
151
|
+
|
152
|
+
|
153
|
+
def r2_to_nebius(r2_bucket_name: str, nebius_bucket_name: str) -> None:
|
154
|
+
"""Creates a one-time transfer from Cloudflare R2 Bucket to
|
155
|
+
Nebius Object Storage.
|
156
|
+
|
157
|
+
it will block until the transfer is complete.
|
158
|
+
|
159
|
+
Args:
|
160
|
+
r2_bucket_name: str; Name of the Amazon S3 Bucket
|
161
|
+
nebius_bucket_name: str; Name of the Cloudflare R2 Bucket
|
162
|
+
"""
|
163
|
+
raise NotImplementedError('Moving data directly from clouds to Nebius is '
|
164
|
+
'currently not supported. Please specify '
|
165
|
+
'a local source for the storage object.')
|
166
|
+
|
167
|
+
|
138
168
|
def gcs_to_s3(gs_bucket_name: str, s3_bucket_name: str) -> None:
|
139
169
|
"""Creates a one-time transfer from Google Cloud Storage to Amazon S3.
|
140
170
|
|
@@ -160,6 +190,19 @@ def gcs_to_r2(gs_bucket_name: str, r2_bucket_name: str) -> None:
|
|
160
190
|
'a local source for the storage object.')
|
161
191
|
|
162
192
|
|
193
|
+
def gcs_to_nebius(gs_bucket_name: str, nebius_bucket_name: str) -> None:
|
194
|
+
"""Creates a one-time transfer from Google Cloud Storage to
|
195
|
+
Nebius Object Storage.
|
196
|
+
|
197
|
+
Args:
|
198
|
+
gs_bucket_name: str; Name of the Google Cloud Storage Bucket
|
199
|
+
nebius_bucket_name: str; Name of the Nebius Bucket
|
200
|
+
"""
|
201
|
+
raise NotImplementedError('Moving data directly from clouds to Nebius is '
|
202
|
+
'currently not supported. Please specify '
|
203
|
+
'a local source for the storage object.')
|
204
|
+
|
205
|
+
|
163
206
|
def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
|
164
207
|
"""Creates a one-time transfer from Cloudflare R2 to Google Cloud Storage.
|
165
208
|
|
@@ -175,6 +218,22 @@ def r2_to_gcs(r2_bucket_name: str, gs_bucket_name: str) -> None:
|
|
175
218
|
'a local source for the storage object.')
|
176
219
|
|
177
220
|
|
221
|
+
def nebius_to_gcs(nebius_bucket_name: str, gs_bucket_name: str) -> None:
|
222
|
+
"""Creates a one-time transfer from Nebius Object Storage
|
223
|
+
to Google Cloud Storage.
|
224
|
+
|
225
|
+
Can be viewed from: https://console.cloud.google.com/transfer/cloud
|
226
|
+
it will block until the transfer is complete.
|
227
|
+
|
228
|
+
Args:
|
229
|
+
nebius_bucket_name: str; Name of the Nebius Object Storage
|
230
|
+
gs_bucket_name: str; Name of the Google Cloud Storage Bucket
|
231
|
+
"""
|
232
|
+
raise NotImplementedError('Moving data directly from Nebius to clouds is '
|
233
|
+
'currently not supported. Please specify '
|
234
|
+
'a local source for the storage object.')
|
235
|
+
|
236
|
+
|
178
237
|
def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
|
179
238
|
"""Creates a one-time transfer from Amazon S3 to Google Cloud Storage.
|
180
239
|
|
@@ -190,6 +249,22 @@ def r2_to_s3(r2_bucket_name: str, s3_bucket_name: str) -> None:
|
|
190
249
|
'a local source for the storage object.')
|
191
250
|
|
192
251
|
|
252
|
+
def nebius_to_s3(nebius_bucket_name: str, s3_bucket_name: str) -> None:
|
253
|
+
"""Creates a one-time transfer from Nebius Object Storage
|
254
|
+
to Amazon S3 Bucket.
|
255
|
+
|
256
|
+
Can be viewed from: https://console.cloud.google.com/transfer/cloud
|
257
|
+
it will block until the transfer is complete.
|
258
|
+
|
259
|
+
Args:
|
260
|
+
nebius_bucket_name: str; Name of the Nebius Object Storage\
|
261
|
+
s3_bucket_name: str; Name of the Amazon S3 Bucket
|
262
|
+
"""
|
263
|
+
raise NotImplementedError('Moving data directly from R2 to clouds is '
|
264
|
+
'currently not supported. Please specify '
|
265
|
+
'a local source for the storage object.')
|
266
|
+
|
267
|
+
|
193
268
|
def _add_bucket_iam_member(bucket_name: str, role: str, member: str) -> None:
|
194
269
|
storage_client = gcp.storage_client()
|
195
270
|
bucket = storage_client.bucket(bucket_name)
|
sky/data/data_utils.py
CHANGED
@@ -20,6 +20,7 @@ from sky.adaptors import azure
|
|
20
20
|
from sky.adaptors import cloudflare
|
21
21
|
from sky.adaptors import gcp
|
22
22
|
from sky.adaptors import ibm
|
23
|
+
from sky.adaptors import nebius
|
23
24
|
from sky.skylet import log_lib
|
24
25
|
from sky.utils import common_utils
|
25
26
|
from sky.utils import ux_utils
|
@@ -94,6 +95,18 @@ def split_r2_path(r2_path: str) -> Tuple[str, str]:
|
|
94
95
|
return bucket, key
|
95
96
|
|
96
97
|
|
98
|
+
def split_nebius_path(nebius_path: str) -> Tuple[str, str]:
|
99
|
+
"""Splits Nebius Path into Bucket name and Relative Path to Bucket
|
100
|
+
|
101
|
+
Args:
|
102
|
+
nebius_path: str; Nebius Path, e.g. nebius://imagenet/train/
|
103
|
+
"""
|
104
|
+
path_parts = nebius_path.replace('nebius://', '').split('/')
|
105
|
+
bucket = path_parts.pop(0)
|
106
|
+
key = '/'.join(path_parts)
|
107
|
+
return bucket, key
|
108
|
+
|
109
|
+
|
97
110
|
def split_cos_path(s3_path: str) -> Tuple[str, str, str]:
|
98
111
|
"""returns extracted region, bucket name and bucket path to data
|
99
112
|
from the specified cos bucket's url.
|
@@ -307,6 +320,16 @@ def create_r2_client(region: str = 'auto') -> Client:
|
|
307
320
|
return cloudflare.client('s3', region)
|
308
321
|
|
309
322
|
|
323
|
+
def create_nebius_client(region: Optional[str]) -> Client:
|
324
|
+
"""Helper method that connects to Boto3 client for Nebius Object Storage
|
325
|
+
|
326
|
+
Args:
|
327
|
+
region: str; Region for Nebius Object Storage
|
328
|
+
"""
|
329
|
+
region = region if region is not None else nebius.DEFAULT_REGION
|
330
|
+
return nebius.client('s3', region)
|
331
|
+
|
332
|
+
|
310
333
|
def verify_r2_bucket(name: str) -> bool:
|
311
334
|
"""Helper method that checks if the R2 bucket exists
|
312
335
|
|
@@ -318,6 +341,17 @@ def verify_r2_bucket(name: str) -> bool:
|
|
318
341
|
return bucket in r2.buckets.all()
|
319
342
|
|
320
343
|
|
344
|
+
def verify_nebius_bucket(name: str) -> bool:
|
345
|
+
"""Helper method that checks if the Nebius bucket exists
|
346
|
+
|
347
|
+
Args:
|
348
|
+
name: str; Name of Nebius Object Storage (without nebius:// prefix)
|
349
|
+
"""
|
350
|
+
nebius_s = nebius.resource('s3')
|
351
|
+
bucket = nebius_s.Bucket(name)
|
352
|
+
return bucket in nebius_s.buckets.all()
|
353
|
+
|
354
|
+
|
321
355
|
def verify_ibm_cos_bucket(name: str) -> bool:
|
322
356
|
"""Helper method that checks if the cos bucket exists
|
323
357
|
|
sky/data/mounting_utils.py
CHANGED
@@ -54,6 +54,24 @@ def get_s3_mount_cmd(bucket_name: str,
|
|
54
54
|
return mount_cmd
|
55
55
|
|
56
56
|
|
57
|
+
def get_nebius_mount_cmd(nebius_profile_name: str,
|
58
|
+
endpoint_url: str,
|
59
|
+
bucket_name: str,
|
60
|
+
mount_path: str,
|
61
|
+
_bucket_sub_path: Optional[str] = None) -> str:
|
62
|
+
"""Returns a command to install Nebius mount utility goofys."""
|
63
|
+
if _bucket_sub_path is None:
|
64
|
+
_bucket_sub_path = ''
|
65
|
+
else:
|
66
|
+
_bucket_sub_path = f':{_bucket_sub_path}'
|
67
|
+
mount_cmd = (f'AWS_PROFILE={nebius_profile_name} goofys -o allow_other '
|
68
|
+
f'--stat-cache-ttl {_STAT_CACHE_TTL} '
|
69
|
+
f'--type-cache-ttl {_TYPE_CACHE_TTL} '
|
70
|
+
f'--endpoint {endpoint_url} '
|
71
|
+
f'{bucket_name}{_bucket_sub_path} {mount_path}')
|
72
|
+
return mount_cmd
|
73
|
+
|
74
|
+
|
57
75
|
def get_gcs_mount_install_cmd() -> str:
|
58
76
|
"""Returns a command to install GCS mount utility gcsfuse."""
|
59
77
|
install_cmd = ('ARCH=$(uname -m) && '
|