skypilot-nightly 1.0.0.dev20250101__py3-none-any.whl → 1.0.0.dev20250103__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '3b0b11011aefa882539e346629d0644fbd59cfe0'
+_SKYPILOT_COMMIT_SHA = '6a6d6671958c9fce56ffa314144daa1156a43342'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250101'
+__version__ = '1.0.0.dev20250103'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/do.py

@@ -0,0 +1,20 @@
+"""Digital Ocean cloud adaptors"""
+
+# pylint: disable=import-outside-toplevel
+
+from sky.adaptors import common
+
+_IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for DO. '
+                         'Try pip install "skypilot[do]"')
+pydo = common.LazyImport('pydo', import_error_message=_IMPORT_ERROR_MESSAGE)
+azure = common.LazyImport('azure', import_error_message=_IMPORT_ERROR_MESSAGE)
+_LAZY_MODULES = (pydo, azure)
+
+
+# `pydo`` inherits Azure exceptions. See:
+# https://github.com/digitalocean/pydo/blob/7b01498d99eb0d3a772366b642e5fab3d6fc6aa2/examples/poc_droplets_volumes_sshkeys.py#L6
+@common.load_lazy_modules(modules=_LAZY_MODULES)
+def exceptions():
+    """Azure exceptions."""
+    from azure.core import exceptions as azure_exceptions
+    return azure_exceptions

sky/backends/backend_utils.py

@@ -1000,6 +1000,7 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, cluster_config_file: str):
             clouds.Cudo,
             clouds.Paperspace,
             clouds.Azure,
+            clouds.DO,
     )):
         config = auth.configure_ssh_info(config)
     elif isinstance(cloud, clouds.GCP):

sky/backends/cloud_vm_ray_backend.py

@@ -178,6 +178,7 @@ def _get_cluster_config_template(cloud):
         clouds.SCP: 'scp-ray.yml.j2',
         clouds.OCI: 'oci-ray.yml.j2',
         clouds.Paperspace: 'paperspace-ray.yml.j2',
+        clouds.DO: 'do-ray.yml.j2',
         clouds.RunPod: 'runpod-ray.yml.j2',
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',

sky/clouds/__init__.py

@@ -15,6 +15,7 @@ from sky.clouds.cloud_registry import CLOUD_REGISTRY
 from sky.clouds.aws import AWS
 from sky.clouds.azure import Azure
 from sky.clouds.cudo import Cudo
+from sky.clouds.do import DO
 from sky.clouds.fluidstack import Fluidstack
 from sky.clouds.gcp import GCP
 from sky.clouds.ibm import IBM
@@ -34,6 +35,7 @@ __all__ = [
     'Cudo',
     'GCP',
     'Lambda',
+    'DO',
     'Paperspace',
     'SCP',
     'RunPod',

sky/clouds/do.py

@@ -0,0 +1,303 @@
+""" Digital Ocean Cloud. """
+
+import json
+import typing
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from sky import clouds
+from sky.adaptors import do
+from sky.clouds import service_catalog
+from sky.provision.do import utils as do_utils
+from sky.utils import resources_utils
+
+if typing.TYPE_CHECKING:
+    from sky import resources as resources_lib
+
+_CREDENTIAL_FILE = 'config.yaml'
+
+
+@clouds.CLOUD_REGISTRY.register(aliases=['digitalocean'])
+class DO(clouds.Cloud):
+    """Digital Ocean Cloud"""
+
+    _REPR = 'DO'
+    _CLOUD_UNSUPPORTED_FEATURES = {
+        clouds.CloudImplementationFeatures.CLONE_DISK_FROM_CLUSTER:
+            'Migrating '
+            f'disk is not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.SPOT_INSTANCE:
+            'Spot instances are '
+            f'not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
+            'Custom disk tiers'
+            f' is not supported in {_REPR}.',
+    }
+    # DO maximum node name length defined as <= 255
+    # https://docs.digitalocean.com/reference/api/api-reference/#operation/droplets_create
+    # 255 - 8 = 247 characters since
+    # our provisioner adds additional `-worker`.
+    _MAX_CLUSTER_NAME_LEN_LIMIT = 247
+    _regions: List[clouds.Region] = []
+
+    # Using the latest SkyPilot provisioner API to provision and check status.
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
+    @classmethod
+    def _unsupported_features_for_resources(
+        cls, resources: 'resources_lib.Resources'
+    ) -> Dict[clouds.CloudImplementationFeatures, str]:
+        """The features not supported based on the resources provided.
+
+        This method is used by check_features_are_supported() to check if the
+        cloud implementation supports all the requested features.
+
+        Returns:
+            A dict of {feature: reason} for the features not supported by the
+            cloud implementation.
+        """
+        del resources  # unused
+        return cls._CLOUD_UNSUPPORTED_FEATURES
+
+    @classmethod
+    def _max_cluster_name_length(cls) -> Optional[int]:
+        return cls._MAX_CLUSTER_NAME_LEN_LIMIT
+
+    @classmethod
+    def regions_with_offering(
+        cls,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]],
+        use_spot: bool,
+        region: Optional[str],
+        zone: Optional[str],
+    ) -> List[clouds.Region]:
+        assert zone is None, 'DO does not support zones.'
+        del accelerators, zone  # unused
+        if use_spot:
+            return []
+        regions = service_catalog.get_region_zones_for_instance_type(
+            instance_type, use_spot, 'DO')
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+        return regions
+
+    @classmethod
+    def get_vcpus_mem_from_instance_type(
+        cls,
+        instance_type: str,
+    ) -> Tuple[Optional[float], Optional[float]]:
+        return service_catalog.get_vcpus_mem_from_instance_type(instance_type,
+                                                                clouds='DO')
+
+    @classmethod
+    def zones_provision_loop(
+        cls,
+        *,
+        region: str,
+        num_nodes: int,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]] = None,
+        use_spot: bool = False,
+    ) -> Iterator[None]:
+        del num_nodes  # unused
+        regions = cls.regions_with_offering(instance_type,
+                                            accelerators,
+                                            use_spot,
+                                            region=region,
+                                            zone=None)
+        for r in regions:
+            assert r.zones is None, r
+            yield r.zones
+
+    def instance_type_to_hourly_cost(
+        self,
+        instance_type: str,
+        use_spot: bool,
+        region: Optional[str] = None,
+        zone: Optional[str] = None,
+    ) -> float:
+        return service_catalog.get_hourly_cost(
+            instance_type,
+            use_spot=use_spot,
+            region=region,
+            zone=zone,
+            clouds='DO',
+        )
+
+    def accelerators_to_hourly_cost(
+        self,
+        accelerators: Dict[str, int],
+        use_spot: bool,
+        region: Optional[str] = None,
+        zone: Optional[str] = None,
+    ) -> float:
+        """Returns the hourly cost of the accelerators, in dollars/hour."""
+        # the acc price is include in the instance price.
+        del accelerators, use_spot, region, zone  # unused
+        return 0.0
+
+    def get_egress_cost(self, num_gigabytes: float) -> float:
+        return 0.0
+
+    def __repr__(self):
+        return self._REPR
+
+    @classmethod
+    def get_default_instance_type(
+        cls,
+        cpus: Optional[str] = None,
+        memory: Optional[str] = None,
+        disk_tier: Optional[resources_utils.DiskTier] = None,
+    ) -> Optional[str]:
+        """Returns the default instance type for DO."""
+        return service_catalog.get_default_instance_type(cpus=cpus,
+                                                         memory=memory,
+                                                         disk_tier=disk_tier,
+                                                         clouds='DO')
+
+    @classmethod
+    def get_accelerators_from_instance_type(
+            cls, instance_type: str) -> Optional[Dict[str, Union[int, float]]]:
+        return service_catalog.get_accelerators_from_instance_type(
+            instance_type, clouds='DO')
+
+    @classmethod
+    def get_zone_shell_cmd(cls) -> Optional[str]:
+        return None
+
+    def make_deploy_resources_variables(
+            self,
+            resources: 'resources_lib.Resources',
+            cluster_name: resources_utils.ClusterName,
+            region: 'clouds.Region',
+            zones: Optional[List['clouds.Zone']],
+            num_nodes: int,
+            dryrun: bool = False) -> Dict[str, Optional[str]]:
+        del zones, dryrun, cluster_name
+
+        r = resources
+        acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
+        if acc_dict is not None:
+            custom_resources = json.dumps(acc_dict, separators=(',', ':'))
+        else:
+            custom_resources = None
+        image_id = None
+        if (resources.image_id is not None and
+                resources.extract_docker_image() is None):
+            if None in resources.image_id:
+                image_id = resources.image_id[None]
+            else:
+                assert region.name in resources.image_id
+                image_id = resources.image_id[region.name]
+        return {
+            'instance_type': resources.instance_type,
+            'custom_resources': custom_resources,
+            'region': region.name,
+            **({
+                'image_id': image_id
+            } if image_id else {})
+        }
+
+    def _get_feasible_launchable_resources(
+        self, resources: 'resources_lib.Resources'
+    ) -> resources_utils.FeasibleResources:
+        """Returns a list of feasible resources for the given resources."""
+        if resources.use_spot:
+            # TODO: Add hints to all return values in this method to help
+            #  users understand why the resources are not launchable.
+            return resources_utils.FeasibleResources([], [], None)
+        if resources.instance_type is not None:
+            assert resources.is_launchable(), resources
+            resources = resources.copy(accelerators=None)
+            return resources_utils.FeasibleResources([resources], [], None)
+
+        def _make(instance_list):
+            resource_list = []
+            for instance_type in instance_list:
+                r = resources.copy(
+                    cloud=DO(),
+                    instance_type=instance_type,
+                    accelerators=None,
+                    cpus=None,
+                )
+                resource_list.append(r)
+            return resource_list
+
+        # Currently, handle a filter on accelerators only.
+        accelerators = resources.accelerators
+        if accelerators is None:
+            # Return a default instance type
+            default_instance_type = DO.get_default_instance_type(
+                cpus=resources.cpus,
+                memory=resources.memory,
+                disk_tier=resources.disk_tier)
+            return resources_utils.FeasibleResources(
+                _make([default_instance_type]), [], None)
+
+        assert len(accelerators) == 1, resources
+        acc, acc_count = list(accelerators.items())[0]
+        (instance_list, fuzzy_candidate_list) = (
+            service_catalog.get_instance_type_for_accelerator(
+                acc,
+                acc_count,
+                use_spot=resources.use_spot,
+                cpus=resources.cpus,
+                memory=resources.memory,
+                region=resources.region,
+                zone=resources.zone,
+                clouds='DO',
+            ))
+        if instance_list is None:
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
+
+    @classmethod
+    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """Verify that the user has valid credentials for DO."""
+        try:
+            # attempt to make a CURL request for listing instances
+            do_utils.client().droplets.list()
+        except do.exceptions().HttpResponseError as err:
+            return False, str(err)
+        except do_utils.DigitalOceanError as err:
+            return False, str(err)
+
+        return True, None
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        try:
+            do_utils.client()
+            return {
+                f'~/.config/doctl/{_CREDENTIAL_FILE}': do_utils.CREDENTIALS_PATH
+            }
+        except do_utils.DigitalOceanError:
+            return {}
+
+    @classmethod
+    def get_current_user_identity(cls) -> Optional[List[str]]:
+        # NOTE: used for very advanced SkyPilot functionality
+        # Can implement later if desired
+        return None
+
+    @classmethod
+    def get_image_size(cls, image_id: str, region: Optional[str]) -> float:
+        del region
+        try:
+            response = do_utils.client().images.get(image_id=image_id)
+            return response['image']['size_gigabytes']
+        except do.exceptions().HttpResponseError as err:
+            raise do_utils.DigitalOceanError(
+                'HTTP error while retrieving size of '
+                f'image_id {response}: {err.error.message}') from err
+        except KeyError as err:
+            raise do_utils.DigitalOceanError(
+                f'No image_id `{image_id}` found') from err
+
+    def instance_type_exists(self, instance_type: str) -> bool:
+        return service_catalog.instance_type_exists(instance_type, 'DO')
+
+    def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
+        return service_catalog.validate_region_zone(region, zone, clouds='DO')

sky/clouds/service_catalog/constants.py

@@ -4,4 +4,4 @@ CATALOG_SCHEMA_VERSION = 'v6'
 CATALOG_DIR = '~/.sky/catalogs'
 ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vsphere', 'cudo', 'fluidstack',
-              'paperspace')
+              'paperspace', 'do')

sky/clouds/service_catalog/do_catalog.py

@@ -0,0 +1,111 @@
+"""Digital ocean service catalog.
+
+This module loads the service catalog file and can be used to
+query instance types and pricing information for digital ocean.
+"""
+
+import typing
+from typing import Dict, List, Optional, Tuple, Union
+
+from sky.clouds.service_catalog import common
+from sky.utils import ux_utils
+
+if typing.TYPE_CHECKING:
+    from sky.clouds import cloud
+
+_df = common.read_catalog('do/vms.csv')
+
+
+def instance_type_exists(instance_type: str) -> bool:
+    return common.instance_type_exists_impl(_df, instance_type)
+
+
+def validate_region_zone(
+        region: Optional[str],
+        zone: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('DO does not support zones.')
+    return common.validate_region_zone_impl('DO', _df, region, zone)
+
+
+def get_hourly_cost(
+    instance_type: str,
+    use_spot: bool = False,
+    region: Optional[str] = None,
+    zone: Optional[str] = None,
+) -> float:
+    """Returns the cost, or the cheapest cost among all zones for spot."""
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('DO does not support zones.')
+    return common.get_hourly_cost_impl(_df, instance_type, use_spot, region,
+                                       zone)
+
+
+def get_vcpus_mem_from_instance_type(
+    instance_type: str,) -> Tuple[Optional[float], Optional[float]]:
+    return common.get_vcpus_mem_from_instance_type_impl(_df, instance_type)
+
+
+def get_default_instance_type(
+    cpus: Optional[str] = None,
+    memory: Optional[str] = None,
+    disk_tier: Optional[str] = None,
+) -> Optional[str]:
+    # NOTE: After expanding catalog to multiple entries, you may
+    # want to specify a default instance type or family.
+    del disk_tier  # unused
+    return common.get_instance_type_for_cpus_mem_impl(_df, cpus, memory)
+
+
+def get_accelerators_from_instance_type(
+        instance_type: str) -> Optional[Dict[str, Union[int, float]]]:
+    return common.get_accelerators_from_instance_type_impl(_df, instance_type)
+
+
+def get_instance_type_for_accelerator(
+    acc_name: str,
+    acc_count: int,
+    cpus: Optional[str] = None,
+    memory: Optional[str] = None,
+    use_spot: bool = False,
+    region: Optional[str] = None,
+    zone: Optional[str] = None,
+) -> Tuple[Optional[List[str]], List[str]]:
+    """Returns a list of instance types that have the given accelerator."""
+    if zone is not None:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError('DO does not support zones.')
+    return common.get_instance_type_for_accelerator_impl(
+        df=_df,
+        acc_name=acc_name,
+        acc_count=acc_count,
+        cpus=cpus,
+        memory=memory,
+        use_spot=use_spot,
+        region=region,
+        zone=zone,
+    )
+
+
+def get_region_zones_for_instance_type(instance_type: str,
+                                       use_spot: bool) -> List['cloud.Region']:
+    df = _df[_df['InstanceType'] == instance_type]
+    return common.get_region_zones(df, use_spot)
+
+
+def list_accelerators(
+    gpus_only: bool,
+    name_filter: Optional[str],
+    region_filter: Optional[str],
+    quantity_filter: Optional[int],
+    case_sensitive: bool = True,
+    all_regions: bool = False,
+    require_price: bool = True,
+) -> Dict[str, List[common.InstanceTypeInfo]]:
+    """Returns all instance types in DO offering GPUs."""
+    del require_price  # unused
+    return common.list_accelerators_impl('DO', _df, gpus_only, name_filter,
+                                         region_filter, quantity_filter,
+                                         case_sensitive, all_regions)

sky/clouds/utils/oci_utils.py

@@ -6,6 +6,10 @@ History:
    configuration.
  - Hysun He (hysun.he@oracle.com) @ Nov.12, 2024: Add the constant
    SERVICE_PORT_RULE_TAG
+ - Hysun He (hysun.he@oracle.com) @ Jan.01, 2025: Set the default image
+   from ubuntu 20.04 to ubuntu 22.04, including:
+   - GPU: skypilot:gpu-ubuntu-2004 -> skypilot:gpu-ubuntu-2204
+   - CPU: skypilot:cpu-ubuntu-2004 -> skypilot:cpu-ubuntu-2204
 """
 import os
 
@@ -117,7 +121,7 @@ class OCIConfig:
         # the sky's user-config file (if not specified, use the hardcode one at
         # last)
         return skypilot_config.get_nested(('oci', 'default', 'image_tag_gpu'),
-                                          'skypilot:gpu-ubuntu-2004')
+                                          'skypilot:gpu-ubuntu-2204')
 
     @classmethod
     def get_default_image_tag(cls) -> str:
@@ -125,7 +129,7 @@ class OCIConfig:
         # set the default image tag in the sky's user-config file. (if not
         # specified, use the hardcode one at last)
         return skypilot_config.get_nested(
-            ('oci', 'default', 'image_tag_general'), 'skypilot:cpu-ubuntu-2004')
+            ('oci', 'default', 'image_tag_general'), 'skypilot:cpu-ubuntu-2204')
 
     @classmethod
     def get_sky_user_config_file(cls) -> str:

sky/provision/do/__init__.py

@@ -0,0 +1,11 @@
+"""DO provisioner for SkyPilot."""
+
+from sky.provision.do.config import bootstrap_instances
+from sky.provision.do.instance import cleanup_ports
+from sky.provision.do.instance import get_cluster_info
+from sky.provision.do.instance import open_ports
+from sky.provision.do.instance import query_instances
+from sky.provision.do.instance import run_instances
+from sky.provision.do.instance import stop_instances
+from sky.provision.do.instance import terminate_instances
+from sky.provision.do.instance import wait_instances

sky/provision/do/config.py

@@ -0,0 +1,14 @@
+"""Paperspace configuration bootstrapping."""
+
+from sky import sky_logging
+from sky.provision import common
+
+logger = sky_logging.init_logger(__name__)
+
+
+def bootstrap_instances(
+        region: str, cluster_name: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    """Bootstraps instances for the given cluster."""
+    del region, cluster_name
+    return config

sky/provision/do/constants.py

@@ -0,0 +1,10 @@
+"""DO cloud constants
+"""
+
+POLL_INTERVAL = 5
+WAIT_DELETE_VOLUMES = 5
+
+GPU_IMAGES = {
+    'gpu-h100x1-80gb': 'gpu-h100x1-base',
+    'gpu-h100x8-640gb': 'gpu-h100x8-base',
+}