skypilot-nightly 1.0.0.dev20241016__py3-none-any.whl → 1.0.0.dev20241018__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '53380e26f01452559012d57b333b17f40dd8a4d1'
+_SKYPILOT_COMMIT_SHA = '71a95f4bf7f1446e80bb5c24d23c1695bc4fc031'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241016'
+__version__ = '1.0.0.dev20241018'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/authentication.py

@@ -43,9 +43,9 @@ from sky.adaptors import gcp
 from sky.adaptors import ibm
 from sky.adaptors import kubernetes
 from sky.adaptors import runpod
-from sky.clouds.utils import lambda_utils
 from sky.provision.fluidstack import fluidstack_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
+from sky.provision.lambda_cloud import lambda_utils
 from sky.utils import common_utils
 from sky.utils import kubernetes_enums
 from sky.utils import subprocess_utils

sky/backends/backend_utils.py

@@ -2772,6 +2772,10 @@ def get_endpoints(cluster: str,
     cluster_records = get_clusters(include_controller=True,
                                    refresh=False,
                                    cluster_names=[cluster])
+    if not cluster_records:
+        with ux_utils.print_exception_no_traceback():
+            raise exceptions.ClusterNotUpError(
+                f'Cluster {cluster!r} not found.', cluster_status=None)
     assert len(cluster_records) == 1, cluster_records
     cluster_record = cluster_records[0]
     if (not skip_status_check and

sky/clouds/aws.py

@@ -32,6 +32,14 @@ if typing.TYPE_CHECKING:
 
 logger = sky_logging.init_logger(__name__)
 
+# Image ID tags
+_DEFAULT_CPU_IMAGE_ID = 'skypilot:custom-cpu-ubuntu'
+# For GPU-related package version,
+# see sky/clouds/service_catalog/images/provisioners/cuda.sh
+_DEFAULT_GPU_IMAGE_ID = 'skypilot:custom-gpu-ubuntu'
+_DEFAULT_GPU_K80_IMAGE_ID = 'skypilot:k80-ubuntu-2004'
+_DEFAULT_NEURON_IMAGE_ID = 'skypilot:neuron-ubuntu-2204'
+
 # This local file (under ~/.aws/) will be uploaded to remote nodes (any
 # cloud), if all of the following conditions hold:
 #   - the current user identity is not using AWS SSO
@@ -217,17 +225,20 @@ class AWS(clouds.Cloud):
     @classmethod
     def _get_default_ami(cls, region_name: str, instance_type: str) -> str:
         acc = cls.get_accelerators_from_instance_type(instance_type)
-        image_id = service_catalog.get_image_id_from_tag(
-            'skypilot:gpu-ubuntu-2004', region_name, clouds='aws')
+        image_id = service_catalog.get_image_id_from_tag(_DEFAULT_CPU_IMAGE_ID,
+                                                         region_name,
+                                                         clouds='aws')
         if acc is not None:
+            image_id = service_catalog.get_image_id_from_tag(
+                _DEFAULT_GPU_IMAGE_ID, region_name, clouds='aws')
             assert len(acc) == 1, acc
             acc_name = list(acc.keys())[0]
             if acc_name == 'K80':
                 image_id = service_catalog.get_image_id_from_tag(
-                    'skypilot:k80-ubuntu-2004', region_name, clouds='aws')
+                    _DEFAULT_GPU_K80_IMAGE_ID, region_name, clouds='aws')
             if acc_name in ['Trainium', 'Inferentia']:
                 image_id = service_catalog.get_image_id_from_tag(
-                    'skypilot:neuron-ubuntu-2204', region_name, clouds='aws')
+                    _DEFAULT_NEURON_IMAGE_ID, region_name, clouds='aws')
         if image_id is not None:
             return image_id
         # Raise ResourcesUnavailableError to make sure the failover in

sky/clouds/gcp.py

@@ -483,7 +483,7 @@ class GCP(clouds.Cloud):
                 if acc in ('A100-80GB', 'L4'):
                     # A100-80GB and L4 have a different name pattern.
                     resources_vars['gpu'] = f'nvidia-{acc.lower()}'
-                elif acc == 'H100':
+                elif acc in ('H100', 'H100-MEGA'):
                     resources_vars['gpu'] = f'nvidia-{acc.lower()}-80gb'
                 else:
                     resources_vars['gpu'] = 'nvidia-tesla-{}'.format(
@@ -546,6 +546,11 @@ class GCP(clouds.Cloud):
         resources_vars[
             'force_enable_external_ips'] = skypilot_config.get_nested(
                 ('gcp', 'force_enable_external_ips'), False)
+
+        # Add gVNIC from config
+        resources_vars['enable_gvnic'] = skypilot_config.get_nested(
+            ('gcp', 'enable_gvnic'), False)
+
         return resources_vars
 
     def _get_feasible_launchable_resources(

sky/clouds/lambda_cloud.py

@@ -8,7 +8,7 @@ import requests
 from sky import clouds
 from sky import status_lib
 from sky.clouds import service_catalog
-from sky.clouds.utils import lambda_utils
+from sky.provision.lambda_cloud import lambda_utils
 from sky.utils import resources_utils
 
 if typing.TYPE_CHECKING:
@@ -48,6 +48,9 @@ class Lambda(clouds.Cloud):
         clouds.CloudImplementationFeatures.HOST_CONTROLLERS: f'Host controllers are not supported in {_REPR}.',
     }
 
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
     @classmethod
     def _unsupported_features_for_resources(
         cls, resources: 'resources_lib.Resources'

sky/clouds/service_catalog/data_fetchers/fetch_aws.py

@@ -538,11 +538,13 @@ if __name__ == '__main__':
     instance_df.to_csv('aws/vms.csv', index=False)
     print('AWS Service Catalog saved to aws/vms.csv')
 
-    image_df = get_all_regions_images_df(user_regions)
-    _check_regions_integrity(image_df, 'images')
+    # Disable refreshing images.csv as we are using skypilot custom AMIs
+    # See sky/clouds/service_catalog/images/README.md for more details.
+    # image_df = get_all_regions_images_df(user_regions)
+    # _check_regions_integrity(image_df, 'images')
 
-    image_df.to_csv('aws/images.csv', index=False)
-    print('AWS Images saved to aws/images.csv')
+    # image_df.to_csv('aws/images.csv', index=False)
+    # print('AWS Images saved to aws/images.csv')
 
     if args.az_mappings:
         az_mappings_df = fetch_availability_zone_mappings()

sky/clouds/service_catalog/data_fetchers/fetch_gcp.py

@@ -419,6 +419,11 @@ def _get_gpus_for_zone(zone: str) -> 'pd.DataFrame':
                 if count != 8:
                     # H100 only has 8 cards.
                     continue
+            if 'H100-MEGA-80GB' in gpu_name:
+                gpu_name = 'H100-MEGA'
+                if count != 8:
+                    # H100-MEGA only has 8 cards.
+                    continue
             if 'VWS' in gpu_name:
                 continue
             if gpu_name.startswith('TPU-'):
@@ -447,6 +452,7 @@ def _gpu_info_from_name(name: str) -> Optional[Dict[str, List[Dict[str, Any]]]]:
         'A100-80GB': 80 * 1024,
         'A100': 40 * 1024,
         'H100': 80 * 1024,
+        'H100-MEGA': 80 * 1024,
         'P4': 8 * 1024,
         'T4': 16 * 1024,
         'V100': 16 * 1024,
@@ -491,12 +497,17 @@ def get_gpu_df(skus: List[Dict[str, Any]],
             if sku['category']['usageType'] != ondemand_or_spot:
                 continue
 
-            gpu_name = row['AcceleratorName']
-            if gpu_name == 'A100-80GB':
-                gpu_name = 'A100 80GB'
-            if gpu_name == 'H100':
-                gpu_name = 'H100 80GB'
-            if f'{gpu_name} GPU' not in sku['description']:
+            gpu_names = [row['AcceleratorName']]
+            if gpu_names[0] == 'A100-80GB':
+                gpu_names = ['A100 80GB']
+            if gpu_names[0] == 'H100':
+                gpu_names = ['H100 80GB']
+            if gpu_names[0] == 'H100-MEGA':
+                # Seems that H100-MEGA has two different descriptions in SKUs in
+                # different regions: 'H100 80GB Mega' and 'H100 80GB Plus'.
+                gpu_names = ['H100 80GB Mega', 'H100 80GB Plus']
+            if not any(f'{gpu_name} GPU' in sku['description']
+                       for gpu_name in gpu_names):
                 continue
 
             unit_price = _get_unit_price(sku)

sky/clouds/service_catalog/gcp_catalog.py

@@ -98,6 +98,9 @@ _ACC_INSTANCE_TYPE_DICTS = {
     },
     'H100': {
         8: ['a3-highgpu-8g'],
+    },
+    'H100-MEGA': {
+        8: ['a3-megagpu-8g'],
     }
 }
 

sky/provision/__init__.py

@@ -19,6 +19,7 @@ from sky.provision import cudo
 from sky.provision import fluidstack
 from sky.provision import gcp
 from sky.provision import kubernetes
+from sky.provision import lambda_cloud
 from sky.provision import runpod
 from sky.provision import vsphere
 from sky.utils import command_runner
@@ -39,6 +40,8 @@ def _route_to_cloud_impl(func):
             provider_name = kwargs.pop('provider_name')
 
         module_name = provider_name.lower()
+        if module_name == 'lambda':
+            module_name = 'lambda_cloud'
         module = globals().get(module_name)
         assert module is not None, f'Unknown provider: {module_name}'
 

sky/provision/gcp/config.py

@@ -670,8 +670,12 @@ def _configure_subnet(region: str, cluster_name: str,
         'accessConfigs': [{
             'name': 'External NAT',
             'type': 'ONE_TO_ONE_NAT',
-        }],
+        }]
     }]
+    # Add gVNIC if specified in config
+    enable_gvnic = config.provider_config.get('enable_gvnic', False)
+    if enable_gvnic:
+        default_interfaces[0]['nicType'] = 'gVNIC'
     enable_external_ips = _enable_external_ips(config)
     if not enable_external_ips:
         # Removing this key means the VM will not be assigned an external IP.

sky/provision/lambda_cloud/__init__.py

@@ -0,0 +1,11 @@
+"""Lambda provisioner for SkyPilot."""
+
+from sky.provision.lambda_cloud.config import bootstrap_instances
+from sky.provision.lambda_cloud.instance import cleanup_ports
+from sky.provision.lambda_cloud.instance import get_cluster_info
+from sky.provision.lambda_cloud.instance import open_ports
+from sky.provision.lambda_cloud.instance import query_instances
+from sky.provision.lambda_cloud.instance import run_instances
+from sky.provision.lambda_cloud.instance import stop_instances
+from sky.provision.lambda_cloud.instance import terminate_instances
+from sky.provision.lambda_cloud.instance import wait_instances

sky/provision/lambda_cloud/config.py

@@ -0,0 +1,10 @@
+"""Lambda Cloud configuration bootstrapping"""
+
+from sky.provision import common
+
+
+def bootstrap_instances(
+        region: str, cluster_name: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    del region, cluster_name  # unused
+    return config

sky/provision/lambda_cloud/instance.py

@@ -0,0 +1,261 @@
+"""Lambda instance provisioning."""
+
+import time
+from typing import Any, Dict, List, Optional
+
+from sky import authentication as auth
+from sky import sky_logging
+from sky import status_lib
+from sky.provision import common
+import sky.provision.lambda_cloud.lambda_utils as lambda_utils
+from sky.utils import common_utils
+from sky.utils import ux_utils
+
+POLL_INTERVAL = 1
+
+logger = sky_logging.init_logger(__name__)
+_lambda_client = None
+
+
+def _get_lambda_client():
+    global _lambda_client
+    if _lambda_client is None:
+        _lambda_client = lambda_utils.LambdaCloudClient()
+    return _lambda_client
+
+
+def _filter_instances(
+        cluster_name_on_cloud: str,
+        status_filters: Optional[List[str]]) -> Dict[str, Dict[str, Any]]:
+    lambda_client = _get_lambda_client()
+    instances = lambda_client.list_instances()
+    possible_names = [
+        f'{cluster_name_on_cloud}-head',
+        f'{cluster_name_on_cloud}-worker',
+    ]
+
+    filtered_instances = {}
+    for instance in instances:
+        if (status_filters is not None and
+                instance['status'] not in status_filters):
+            continue
+        if instance.get('name') in possible_names:
+            filtered_instances[instance['id']] = instance
+    return filtered_instances
+
+
+def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
+    head_instance_id = None
+    for instance_id, instance in instances.items():
+        if instance['name'].endswith('-head'):
+            head_instance_id = instance_id
+            break
+    return head_instance_id
+
+
+def _get_ssh_key_name(prefix: str = '') -> str:
+    lambda_client = _get_lambda_client()
+    _, public_key_path = auth.get_or_generate_keys()
+    with open(public_key_path, 'r', encoding='utf-8') as f:
+        public_key = f.read()
+    name, exists = lambda_client.get_unique_ssh_key_name(prefix, public_key)
+    if not exists:
+        raise lambda_utils.LambdaCloudError('SSH key not found')
+    return name
+
+
+def run_instances(region: str, cluster_name_on_cloud: str,
+                  config: common.ProvisionConfig) -> common.ProvisionRecord:
+    """Runs instances for the given cluster"""
+    lambda_client = _get_lambda_client()
+    pending_status = ['booting']
+    while True:
+        instances = _filter_instances(cluster_name_on_cloud, pending_status)
+        if not instances:
+            break
+        logger.info(f'Waiting for {len(instances)} instances to be ready.')
+        time.sleep(POLL_INTERVAL)
+    exist_instances = _filter_instances(cluster_name_on_cloud, ['active'])
+    head_instance_id = _get_head_instance_id(exist_instances)
+
+    to_start_count = config.count - len(exist_instances)
+    if to_start_count < 0:
+        raise RuntimeError(
+            f'Cluster {cluster_name_on_cloud} already has '
+            f'{len(exist_instances)} nodes, but {config.count} are required.')
+    if to_start_count == 0:
+        if head_instance_id is None:
+            raise RuntimeError(
+                f'Cluster {cluster_name_on_cloud} has no head node.')
+        logger.info(f'Cluster {cluster_name_on_cloud} already has '
+                    f'{len(exist_instances)} nodes, no need to start more.')
+        return common.ProvisionRecord(
+            provider_name='lambda',
+            cluster_name=cluster_name_on_cloud,
+            region=region,
+            zone=None,
+            head_instance_id=head_instance_id,
+            resumed_instance_ids=[],
+            created_instance_ids=[],
+        )
+
+    created_instance_ids = []
+    ssh_key_name = _get_ssh_key_name()
+
+    def launch_nodes(node_type: str, quantity: int) -> List[str]:
+        try:
+            instance_ids = lambda_client.create_instances(
+                instance_type=config.node_config['InstanceType'],
+                region=region,
+                name=f'{cluster_name_on_cloud}-{node_type}',
+                quantity=quantity,
+                ssh_key_name=ssh_key_name,
+            )
+            logger.info(f'Launched {len(instance_ids)} {node_type} node(s), '
+                        f'instance_ids: {instance_ids}')
+            return instance_ids
+        except Exception as e:
+            logger.warning(f'run_instances error: {e}')
+            raise
+
+    if head_instance_id is None:
+        instance_ids = launch_nodes('head', 1)
+        assert len(instance_ids) == 1
+        created_instance_ids.append(instance_ids[0])
+        head_instance_id = instance_ids[0]
+
+    assert head_instance_id is not None, 'head_instance_id should not be None'
+
+    worker_node_count = to_start_count - 1
+    if worker_node_count > 0:
+        instance_ids = launch_nodes('worker', worker_node_count)
+        created_instance_ids.extend(instance_ids)
+
+    while True:
+        instances = _filter_instances(cluster_name_on_cloud, ['active'])
+        if len(instances) == config.count:
+            break
+
+        time.sleep(POLL_INTERVAL)
+
+    return common.ProvisionRecord(
+        provider_name='lambda',
+        cluster_name=cluster_name_on_cloud,
+        region=region,
+        zone=None,
+        head_instance_id=head_instance_id,
+        resumed_instance_ids=[],
+        created_instance_ids=created_instance_ids,
+    )
+
+
+def wait_instances(region: str, cluster_name_on_cloud: str,
+                   state: Optional[status_lib.ClusterStatus]) -> None:
+    del region, cluster_name_on_cloud, state  # Unused.
+
+
+def stop_instances(
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    worker_only: bool = False,
+) -> None:
+    raise NotImplementedError(
+        'stop_instances is not supported for Lambda Cloud')
+
+
+def terminate_instances(
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    worker_only: bool = False,
+) -> None:
+    """See sky/provision/__init__.py"""
+    del provider_config
+    lambda_client = _get_lambda_client()
+    instances = _filter_instances(cluster_name_on_cloud, None)
+
+    instance_ids_to_terminate = []
+    for instance_id, instance in instances.items():
+        if worker_only and not instance['name'].endswith('-worker'):
+            continue
+        instance_ids_to_terminate.append(instance_id)
+
+    try:
+        logger.debug(
+            f'Terminating instances {", ".join(instance_ids_to_terminate)}')
+        lambda_client.remove_instances(instance_ids_to_terminate)
+    except Exception as e:  # pylint: disable=broad-except
+        with ux_utils.print_exception_no_traceback():
+            raise RuntimeError(
+                f'Failed to terminate instances {instance_ids_to_terminate}: '
+                f'{common_utils.format_exception(e, use_bracket=False)}') from e
+
+
+def get_cluster_info(
+    region: str,
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+) -> common.ClusterInfo:
+    del region  # unused
+    running_instances = _filter_instances(cluster_name_on_cloud, ['active'])
+    instances: Dict[str, List[common.InstanceInfo]] = {}
+    head_instance_id = None
+    for instance_id, instance_info in running_instances.items():
+        instances[instance_id] = [
+            common.InstanceInfo(
+                instance_id=instance_id,
+                internal_ip=instance_info['private_ip'],
+                external_ip=instance_info['ip'],
+                ssh_port=22,
+                tags={},
+            )
+        ]
+        if instance_info['name'].endswith('-head'):
+            head_instance_id = instance_id
+
+    return common.ClusterInfo(
+        instances=instances,
+        head_instance_id=head_instance_id,
+        provider_name='lambda',
+        provider_config=provider_config,
+    )
+
+
+def query_instances(
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    non_terminated_only: bool = True,
+) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    """See sky/provision/__init__.py"""
+    assert provider_config is not None, (cluster_name_on_cloud, provider_config)
+    instances = _filter_instances(cluster_name_on_cloud, None)
+
+    status_map = {
+        'booting': status_lib.ClusterStatus.INIT,
+        'active': status_lib.ClusterStatus.UP,
+        'unhealthy': status_lib.ClusterStatus.INIT,
+        'terminating': status_lib.ClusterStatus.INIT,
+    }
+    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    for instance_id, instance in instances.items():
+        status = status_map.get(instance['status'])
+        if non_terminated_only and status is None:
+            continue
+        statuses[instance_id] = status
+    return statuses
+
+
+def open_ports(
+    cluster_name_on_cloud: str,
+    ports: List[str],
+    provider_config: Optional[Dict[str, Any]] = None,
+) -> None:
+    raise NotImplementedError('open_ports is not supported for Lambda Cloud')
+
+
+def cleanup_ports(
+    cluster_name_on_cloud: str,
+    ports: List[str],
+    provider_config: Optional[Dict[str, Any]] = None,
+) -> None:
+    """See sky/provision/__init__.py"""
+    del cluster_name_on_cloud, ports, provider_config  # Unused.

sky/{clouds/utils → provision/lambda_cloud}/lambda_utils.py

@@ -1,4 +1,5 @@
 """Lambda Cloud helper functions."""
+
 import json
 import os
 import time
@@ -76,7 +77,7 @@ class Metadata:
 
 
 def raise_lambda_error(response: requests.Response) -> None:
-    """Raise LambdaCloudError if appropriate. """
+    """Raise LambdaCloudError if appropriate."""
     status_code = response.status_code
     if status_code == 200:
         return
@@ -131,20 +132,22 @@ class LambdaCloudClient:
         self.api_key = self._credentials['api_key']
         self.headers = {'Authorization': f'Bearer {self.api_key}'}
 
-    def create_instances(self,
-                         instance_type: str = 'gpu_1x_a100_sxm4',
-                         region: str = 'us-east-1',
-                         quantity: int = 1,
-                         name: str = '',
-                         ssh_key_name: str = '') -> List[str]:
+    def create_instances(
+        self,
+        instance_type: str = 'gpu_1x_a100_sxm4',
+        region: str = 'us-east-1',
+        quantity: int = 1,
+        name: str = '',
+        ssh_key_name: str = '',
+    ) -> List[str]:
         """Launch new instances."""
         # Optimization:
         # Most API requests are rate limited at ~1 request every second but
         # launch requests are rate limited at ~1 request every 10 seconds.
         # So don't use launch requests to check availability.
         # See https://docs.lambdalabs.com/cloud/rate-limiting/ for more.
-        available_regions = self.list_catalog()[instance_type]\
-                ['regions_with_capacity_available']
+        available_regions = (self.list_catalog()[instance_type]
+                             ['regions_with_capacity_available'])
         available_regions = [reg['name'] for reg in available_regions]
         if region not in available_regions:
             if len(available_regions) > 0:
@@ -163,27 +166,25 @@ class LambdaCloudClient:
             'instance_type_name': instance_type,
             'ssh_key_names': [ssh_key_name],
             'quantity': quantity,
-            'name': name
+            'name': name,
         })
         response = _try_request_with_backoff(
             'post',
             f'{API_ENDPOINT}/instance-operations/launch',
             data=data,
-            headers=self.headers)
+            headers=self.headers,
+        )
         return response.json().get('data', []).get('instance_ids', [])
 
-    def remove_instances(self, *instance_ids: str) -> Dict[str, Any]:
+    def remove_instances(self, instance_ids: List[str]) -> Dict[str, Any]:
         """Terminate instances."""
-        data = json.dumps({
-            'instance_ids': [
-                instance_ids[0]  # TODO(ewzeng) don't hardcode
-            ]
-        })
+        data = json.dumps({'instance_ids': instance_ids})
         response = _try_request_with_backoff(
             'post',
             f'{API_ENDPOINT}/instance-operations/terminate',
             data=data,
-            headers=self.headers)
+            headers=self.headers,
+        )
         return response.json().get('data', []).get('terminated_instances', [])
 
     def list_instances(self) -> List[Dict[str, Any]]:

sky/setup_files/MANIFEST.in

@@ -6,7 +6,6 @@ include sky/setup_files/*
 include sky/skylet/*.sh
 include sky/skylet/LICENSE
 include sky/skylet/providers/ibm/*
-include sky/skylet/providers/lambda_cloud/*
 include sky/skylet/providers/oci/*
 include sky/skylet/providers/scp/*
 include sky/skylet/providers/*.py

sky/templates/gcp-ray.yml.j2

@@ -64,6 +64,9 @@ provider:
   # leakage.
   disable_launch_config_check: true
   use_managed_instance_group: {{ gcp_use_managed_instance_group }}
+{%- if enable_gvnic %}
+  enable_gvnic: {{ enable_gvnic }}
+{%- endif %}
 
 auth:
   ssh_user: gcpuser

sky/templates/lambda-ray.yml.j2

@@ -7,7 +7,7 @@ idle_timeout_minutes: 60
 
 provider:
   type: external
-  module: sky.skylet.providers.lambda_cloud.LambdaNodeProvider
+  module: sky.provision.lambda
   region: {{region}}
   # Disable launch config check for worker nodes as it can cause resource
   # leakage.
@@ -25,14 +25,6 @@ available_node_types:
     resources: {}
     node_config:
       InstanceType: {{instance_type}}
-{% if num_nodes > 1 %}
-  ray_worker_default:
-    min_workers: {{num_nodes - 1}}
-    max_workers: {{num_nodes - 1}}
-    resources: {}
-    node_config:
-      InstanceType: {{instance_type}}
-{%- endif %}
 
 head_node_type: ray_head_default
 
@@ -64,7 +56,10 @@ setup_commands:
   # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
   # Line 'mkdir -p ..': disable host key check
   # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
-  - sudo systemctl stop unattended-upgrades || true;
+  - {%- for initial_setup_command in initial_setup_commands %}
+    {{ initial_setup_command }}
+    {%- endfor %}
+    sudo systemctl stop unattended-upgrades || true;
     sudo systemctl disable unattended-upgrades || true;
     sudo sed -i 's/Unattended-Upgrade "1"/Unattended-Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades || true;
     sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
@@ -81,31 +76,5 @@ setup_commands:
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
     [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf');
 
-# Command to start ray on the head node. You don't need to change this.
-# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
-# connection, which is expensive. Try your best to co-locate commands into fewer
-# items! The same comment applies for worker_start_ray_commands.
-#
-# Increment the following for catching performance bugs easier:
-#   current num items (num SSH connections): 2
-head_start_ray_commands:
-  - {{ sky_activate_python_env }}; {{ sky_ray_cmd }} stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 {{ sky_ray_cmd }} start --disable-usage-stats --head --port={{ray_port}} --min-worker-port 11002 --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
-    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
-    {{dump_port_command}}; {{ray_head_wait_initialized_command}}
-
-{%- if num_nodes > 1 %}
-worker_start_ray_commands:
-  - {{ sky_activate_python_env }}; {{ sky_ray_cmd }} stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 {{ sky_ray_cmd }} start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --min-worker-port 11002 --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
-    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
-{%- else %}
-worker_start_ray_commands: []
-{%- endif %}
-
-head_node: {}
-worker_nodes: {}
-
-# These fields are required for external cloud providers.
-head_setup_commands: []
-worker_setup_commands: []
-cluster_synced_files: []
-file_mounts_sync_continuously: False
+# Command to start ray clusters are now placed in `sky.provision.instance_setup`.
+# We do not need to list it here anymore.

sky/utils/schemas.py

@@ -755,6 +755,9 @@ def get_config_schema():
                 'force_enable_external_ips': {
                     'type': 'boolean'
                 },
+                'enable_gvnic': {
+                    'type': 'boolean'
+                },
                 **_LABELS_SCHEMA,
                 **_NETWORK_CONFIG_SCHEMA,
             },

{skypilot_nightly-1.0.0.dev20241016.dist-info → skypilot_nightly-1.0.0.dev20241018.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241016
+Version: 1.0.0.dev20241018
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241016.dist-info → skypilot_nightly-1.0.0.dev20241018.dist-info}/RECORD

@@ -1,6 +1,6 @@
-sky/__init__.py,sha256=19EG_Nr4EJcbkyLvfF_ZmWhAbfEysS498RvephjOslM,5854
+sky/__init__.py,sha256=ooJaoPt0Vq10nF2ftXGThCKQFJ2HbgQNKQ7Dp6Qg6s4,5854
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
-sky/authentication.py,sha256=TfKkVnmRIetATSEVQFp-rOOIRGqVig2i8faSQQt_ixA,20974
+sky/authentication.py,sha256=pAdCT60OxxiXI9KXDyP2lQ9u9vMc6aMtq5Xi2h_hbdw,20984
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
 sky/cli.py,sha256=PJR6W92twf89j17OWLQJ9RawdazJcGslfW2L_fLB2PM,208545
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
@@ -30,7 +30,7 @@ sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=wwfbrxPhjMPs6PSyy3tAHI8WJhl-xhgzWBsAZjmJJ6g,6249
-sky/backends/backend_utils.py,sha256=u9P7Fd3DB9LaOq51fK7kwKpxtgFGGWmgULY6GoLSUPM,126791
+sky/backends/backend_utils.py,sha256=PA21DAXspXuTZDQ5qA3G5RGJ0oUTpJ7XatRRvhtmtt0,126993
 sky/backends/cloud_vm_ray_backend.py,sha256=9mCLLRUD-x3ksiiPbhrMDsZWIPNU9cVSQwwpmxSia7k,236881
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=0JL5m0YUgOmOL4aWEUe4tmt89dsxjk4_WXkPwgEKEis,16801
@@ -40,16 +40,16 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=eb-i6zYoo-Zkod-T9qtCu1FcYLw--Yyos1SyibUPZNE,26194
 sky/clouds/__init__.py,sha256=WuNIJEnZmBO72tU5awgaaL3rdvFRSkgaYNNeuY68dXo,1356
-sky/clouds/aws.py,sha256=uwz4aesgA2o6WNDhZlrXafQFbILv4EiUPNP67Cc8WGE,48978
+sky/clouds/aws.py,sha256=XJVbOSkVVUHp9HbHDp0rFdHX113JHbY-3sgokGdNJVE,49527
 sky/clouds/azure.py,sha256=Yp_a1Lzvq4s47eRMeyVheDv9pC0hSPogCiTMYf-a5ZE,28687
 sky/clouds/cloud.py,sha256=PPk-Cbf1YbJT8bswcQLtPBtko02OWrRGJKkLzDpytTI,34858
 sky/clouds/cloud_registry.py,sha256=4yQMv-iBSgyN5aNL4Qxbn0JVE-dkVoEUIgj7S1z9S_Q,955
 sky/clouds/cudo.py,sha256=H4VyMo5wWGAv2MXZ3xsbWjlZA_cZYnt4ecNlTOOao8Y,13147
 sky/clouds/fluidstack.py,sha256=iOmoOx52yTrHKMzwBDaxFJCfNo79M61d5tj-Np24Lyc,12436
-sky/clouds/gcp.py,sha256=FKHqtF4YMY06pseloMEbnt4XwIQ5ErDLlrvyXzIzZa4,54308
+sky/clouds/gcp.py,sha256=lUImS2WJIcUOtrgrVz8zaR4yPGqALqZ0lSmLbjN9xLU,54470
 sky/clouds/ibm.py,sha256=M8QdjeSFlwssfoY2aOodxG4q5R3eT9K-4lTPDHYvEYI,21476
 sky/clouds/kubernetes.py,sha256=aWoXWR-S4puZHzuUHroLKxLdTpkqU7j75dQlXECnsmE,28679
-sky/clouds/lambda_cloud.py,sha256=2Al3qCSl-I4iTi7pPPNXcbaLyVfCUgTl__vYBunLB6k,12439
+sky/clouds/lambda_cloud.py,sha256=VtJ2mmwMT1X4zrzgt3FXM61zmrrgoELZHFgsdYVesPY,12562
 sky/clouds/oci.py,sha256=WXtxKwDBgi3He4ayi4qzJ4Y659Bi6xU8hWmYLHwiQYs,27371
 sky/clouds/paperspace.py,sha256=lmUZPYAblaqiBmGQwCunccMiTF_dVA1o3vqY9Q_Nc28,10921
 sky/clouds/runpod.py,sha256=lstUC6f4JDhtcH9NfwkbpCJMmfmvMigoanhPXPbTYds,11540
@@ -63,7 +63,7 @@ sky/clouds/service_catalog/config.py,sha256=ylzqewdEBjDg4awvFek6ldYmFrnvD2bVGLZu
 sky/clouds/service_catalog/constants.py,sha256=ai2yOlsVqBnEpbxaEHXt61COsHBLwOfw6GZXntEPj7k,411
 sky/clouds/service_catalog/cudo_catalog.py,sha256=QXAOpx5fJ_cGCr5LbB7wpHMfKIla7G-q_mMJnv_ArTA,4652
 sky/clouds/service_catalog/fluidstack_catalog.py,sha256=c8MMTldG-q97MJ0zJymudQiOVQC_rxS7vqrZgLrgbQA,5038
-sky/clouds/service_catalog/gcp_catalog.py,sha256=MHWq_-jqm68oNpK1i8AlJIGBkSKT-P6xX7DkpvqvpHU,24323
+sky/clouds/service_catalog/gcp_catalog.py,sha256=v_5fsB3dB9oD8U7lBKnCe5ii6AUWEOiQjNarMnU_qLA,24379
 sky/clouds/service_catalog/ibm_catalog.py,sha256=0dzjmXABFECzaAuIa0E6pVINhVK6-G6U52Mj-L45gK8,4472
 sky/clouds/service_catalog/kubernetes_catalog.py,sha256=6OocEUkgyJtBgHwzu4RPsvru6pj6RwGU-4uSFNQmsSM,8254
 sky/clouds/service_catalog/lambda_catalog.py,sha256=BAhUGqHj8aVe1zUhEQNO7bQUhcd9jAespGvPyQubTJY,5281
@@ -74,17 +74,16 @@ sky/clouds/service_catalog/scp_catalog.py,sha256=4XnaZE5Q4XrrNnDnVhsHkH6jxmWXBeQ
 sky/clouds/service_catalog/vsphere_catalog.py,sha256=yJLWu9SQep-PRn1YdeQ7ZoNqQHTAxJtxf7y6FBrfSW0,4391
 sky/clouds/service_catalog/data_fetchers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/clouds/service_catalog/data_fetchers/analyze.py,sha256=VdksJQs3asFE8H5T3ZV1FJas2xD9WEX6c-V5p7y-wp4,2084
-sky/clouds/service_catalog/data_fetchers/fetch_aws.py,sha256=6gpRtQaQtvT1cMpiVBacNXXQAjBC5MWpAUI-1ELYg0U,22850
+sky/clouds/service_catalog/data_fetchers/fetch_aws.py,sha256=ro2zazdkDF6z9bE7QFyjoeb4VFxmbNZ1WK5IQrdoQWk,23003
 sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=jsSVqbSbBIw_IYmO-y2u4co20AJ-JF713KFjUKdO_VA,12272
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=35nO_VaDOgp5W13kt_lIANSk_CNf7gBiZGJ5fGyZu6o,6808
-sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=2VebGjKkCckQZgOW3MBlfjFvA56U7eaIAh2q-6NYcL0,29070
+sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=VHwYIPX1kGOvGQ67mtvhKe1enmKFF3knveIktDwdYio,29633
 sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=B7H14so38zayuJGgUrD1PJYJKiVZHGnwH6JJop3F7o0,4918
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=SF_gTU74qg6L-DSWneCAbqP0lwZXaaDi5otiMIJbrw0,21462
 sky/clouds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/clouds/utils/aws_utils.py,sha256=W5BRC-2F_VY4BymRA1kS6-MufsI3V8cfY_hv--4gJBU,1986
 sky/clouds/utils/gcp_utils.py,sha256=Xc_COjJfDt__oqVwrCw7ejY2B7ptHjMjDVb8obcpJ6s,6968
-sky/clouds/utils/lambda_utils.py,sha256=n_GwXhswehDOJrby1fGyPXSQuPmPTPQqXZzV4xQm9KI,9991
 sky/clouds/utils/oci_utils.py,sha256=LT_RtPQ2B1wlSF0e9PSD3NWxFFIzovcZeDjO-dyOghU,4482
 sky/clouds/utils/scp_utils.py,sha256=RUp7NwyhKygOoVOwvdAOGdoQNSJjryOG6WSExCf-yas,15812
 sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
@@ -103,7 +102,7 @@ sky/jobs/utils.py,sha256=lYfWkEAPVnYcj2nT6VYdM6PCaWKUH6_AD4TAV_sVCkY,36376
 sky/jobs/dashboard/dashboard.py,sha256=HFShuaxKir97QTeK2x37h6bsY6ncaFaNEg1USZqJPdc,3050
 sky/jobs/dashboard/static/favicon.ico,sha256=uYlvgxSM7gjBmXpZ8wydvZUPAbJiiix-rc2Xe5mma9s,15086
 sky/jobs/dashboard/templates/index.html,sha256=DBKMYEkkJ6sgLYod9ro7drgL8Y_neDsCx_WbwhWDsWM,9837
-sky/provision/__init__.py,sha256=ZBahWZgLw63EvYGBpxTJNnys_bb7TpdtQpESWRyoWC8,6146
+sky/provision/__init__.py,sha256=UhYsGRribEyK1--PPT0Dom9051jlpdn8UCNhO8qpPOc,6262
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=DvHj3wpqdpaSBHMOGIfVWLLWGJoz0eOQAx73DwYMNEk,531
 sky/provision/docker_utils.py,sha256=Z7vDUs9Yjqks_CsWrACcTgABIZuFi3EJVFwkU0WsdD0,18832
@@ -131,7 +130,7 @@ sky/provision/fluidstack/config.py,sha256=hDqesKEVjIhXLTWej3fDdpbHtKBXoybxFGgC6T
 sky/provision/fluidstack/fluidstack_utils.py,sha256=Y21y2IAiHPLk_b-Lp-ld26SZSfWARhxdDEiu7MtfBmc,5693
 sky/provision/fluidstack/instance.py,sha256=jZb_zJFkyCZw3QV3Dt-GbUxl_2HR1kxvOxS7G3lkWcA,13708
 sky/provision/gcp/__init__.py,sha256=zlgjR2JoaGD7sStGStMRu9bJ62f-8NKEIyb-bFHBlzM,528
-sky/provision/gcp/config.py,sha256=THxFEtl9FjRvn9WJPYHbPjUaYNI5dXJRrcaznhRfM9k,33161
+sky/provision/gcp/config.py,sha256=i0PhR1ybGErQiPT8cD6E5OFB7LD6sub4Rc-mhgTREVI,33340
 sky/provision/gcp/constants.py,sha256=ojerfnNEeayJn-0aJq2Uq1iTchxOkpruKrPBbHmdiEw,7448
 sky/provision/gcp/instance.py,sha256=l2-1nHj4pUoHqOu8HMN1hT1bwd4Q96X8MXgOPsNJUN8,25184
 sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHaSvY7c8,70943
@@ -144,6 +143,10 @@ sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMsh
 sky/provision/kubernetes/utils.py,sha256=2N5c4yA7CEn4DjvCiUO73W4XDEjgixcJRVdgs913QQE,89523
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
+sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
+sky/provision/lambda_cloud/config.py,sha256=jq1iLzp4Up61r4JGxvtpVbJlgXnea3LHYQhCQyyl7ik,272
+sky/provision/lambda_cloud/instance.py,sha256=5-XuX-KwlRq8y62NXNzY_p6aJs4iCPGBf5U4pIR4liI,8975
+sky/provision/lambda_cloud/lambda_utils.py,sha256=H8uaaMEpLn5cqGCdhUH_oJiccv_cuMguUNAl0NqB0Ik,9873
 sky/provision/paperspace/__init__.py,sha256=1nbUPWio7UA5gCQkO_rfEDfgXT17u5OtuByxQx4Ez6g,598
 sky/provision/paperspace/config.py,sha256=oNmffSt-V466pE0DmML8hOCX1CiA24jAqE5JEKuqpyI,1541
 sky/provision/paperspace/constants.py,sha256=NcLJGivJxshJwhR28yVHysWQ2gtMAkTVmHC91d3kyKM,957
@@ -180,7 +183,7 @@ sky/serve/serve_state.py,sha256=5BZSKKKxQRk-0mku17Ch4Veu4qOhaFvaOJY3zrZCkLw,1931
 sky/serve/serve_utils.py,sha256=im_1cJoJmufFxkBVnhK4nI6XlHvEXersQyIivNruJJc,38009
 sky/serve/service.py,sha256=fkfJvNJ2BO6rfV0TblZG-QkOXaCyZlpkwbGgrsTzf2w,11872
 sky/serve/service_spec.py,sha256=iRhW95SERvb4NWtV10uCuhgvW31HuSAmZZ55OX0WK8s,15309
-sky/setup_files/MANIFEST.in,sha256=BAR1TfVIHwBFfV3najggE8HDXTJyO3fNN0Yhu5aTitI,634
+sky/setup_files/MANIFEST.in,sha256=CXz8lIJMgWlH9TvYgzIL3vPFtSDoQq-UMfD9K62rtH4,590
 sky/setup_files/setup.py,sha256=o4IgiwFoTB6Sdn3MmOirUIS0OSkoh6qo_0vrgcmrYA4,12093
 sky/skylet/LICENSE,sha256=BnFrJSvUFpMUoH5mOpWnEvaC5R6Uux8W6WXgrte8iYg,12381
 sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -200,8 +203,6 @@ sky/skylet/providers/ibm/__init__.py,sha256=GXo5F9ztvs0qMDI_G9wM5KvzySfYugslJMHH
 sky/skylet/providers/ibm/node_provider.py,sha256=olNtCoCxjXTT-C_youwdQ9UF1DPgO8OVwDueotGFaJI,38280
 sky/skylet/providers/ibm/utils.py,sha256=63vhKqLLOhAZdibSp8VWWONeyCER9F6U2VLrSpzlizk,1292
 sky/skylet/providers/ibm/vpc_provider.py,sha256=GiOGlWYqqeBETfAeKqVj2-9shsMSP7z1WnO8UP5JTNo,34630
-sky/skylet/providers/lambda_cloud/__init__.py,sha256=DNxB-NL97FU7ptGcXTnIZZRt6ZLtNVtp3nayJNaxjhY,112
-sky/skylet/providers/lambda_cloud/node_provider.py,sha256=77AoTaXuSW5cr4Z2cEAbAsHx1e4l8c0fc1ABbdLduKE,14042
 sky/skylet/providers/oci/__init__.py,sha256=LRMTj6OhQoxiFJw4uNxG8cn6PllP8A-lGJL3Cs5DJok,91
 sky/skylet/providers/oci/node_provider.py,sha256=YPqiRag_cysvYMIMDGbMn6lOumvHad6FLJB5DGPr00Q,20492
 sky/skylet/providers/oci/query_helper.py,sha256=dUsvPGzWPNF5O2NjQvuC8tkilT4H11gMj6R7Qel2fDc,17202
@@ -221,7 +222,7 @@ sky/templates/aws-ray.yml.j2,sha256=K0rAuyf1XC_GPFp1BR9df42-Be12A6T2UF0BllVSpYg,
 sky/templates/azure-ray.yml.j2,sha256=RtYAcAmFQd6TB3j-pbxi7ekjWhznqFhJtzdkqH_nXqM,6135
 sky/templates/cudo-ray.yml.j2,sha256=SEHVY57iBauCOE2HYJtYVFEKlriAkdwQu_p86a1n_bA,3548
 sky/templates/fluidstack-ray.yml.j2,sha256=t8TCULgiErCZdtFmBZVsA8ZdcqR7ccwsmQhuDFTBEAU,3541
-sky/templates/gcp-ray.yml.j2,sha256=q2xSWxxYI8MVAq_mA__8FF6PwEqXCAW1SOEOGTt0qPw,9591
+sky/templates/gcp-ray.yml.j2,sha256=y95B-Nk6hFxm6vEIaxI1wFzAIcy_GcKC3XMYo9m-ThI,9662
 sky/templates/ibm-ray.yml.j2,sha256=RMBUqPId8i4CnVwcyfK3DbRapF1jFMuGQlY0E0PFbMU,6669
 sky/templates/jobs-controller.yaml.j2,sha256=Gu3ogFxFYr09VEXP-6zEbrCUOFo1aYxWEjAq7whCrxo,1607
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
@@ -229,7 +230,7 @@ sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8w
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=HlG7CPBBedCVBlL9qv0erW_eKm6Irj0LFyaAWuJW_lc,3148
 sky/templates/kubernetes-ray.yml.j2,sha256=Wq9luXc6-t141uyHbtOy1IDmLMM0PBbePTZfZEtAKw0,18160
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
-sky/templates/lambda-ray.yml.j2,sha256=UrYOUh4EliPlWcfQWPZzQSiIIYSoRloujV2xsZejYPM,5786
+sky/templates/lambda-ray.yml.j2,sha256=oMbrfv3zHoD1v1XXMLCLK1vB7wLBU1Z_jNpC4-5lGVo,3985
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
 sky/templates/oci-ray.yml.j2,sha256=5XfIobW9XuspIpEhI4vFIEcJEFCdtFJqEGfX03zL6DE,7032
 sky/templates/paperspace-ray.yml.j2,sha256=HQjZNamrB_a4fOMCxQXSVdV5JIHtbGtAE0JzEO8uuVQ,4021
@@ -255,7 +256,7 @@ sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM
 sky/utils/log_utils.py,sha256=ptv2sbsiJSgk4NvdccrMsUR-MvOKnbu4BQiRSishgk0,12472
 sky/utils/resources_utils.py,sha256=sJuPextjJKHhvDGAaOPEzeEkteryF2fGNgNgBLqnLic,7419
 sky/utils/rich_utils.py,sha256=hmnI1X5dKvRIQzB7EyNb34FT97qFNve-0QHqM5r0mVk,3066
-sky/utils/schemas.py,sha256=QT0Fxri2o0SiWkky1DlZhA1dzQRQoB5OdVaej0wJvhc,28787
+sky/utils/schemas.py,sha256=qo9j1TJZXqgJlBgbQfqz1oIZAxc3CN8uWooKYPQXXIY,28878
 sky/utils/subprocess_utils.py,sha256=3R54Elc2n8DQeO6Y8MCDJ6N6v27HDGpbNMIfCquqXYQ,6552
 sky/utils/timeline.py,sha256=ao_nm0y52ZQILfL7Y92c3pSEFRyPm_ElORC3DrI5BwQ,3936
 sky/utils/ux_utils.py,sha256=CqyIFGDuSE8fQasPkna_loZMwtboC9KedR09WEQ7qz0,6502
@@ -273,9 +274,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=aRMa_0JRHtXFOPtEg4rFAwR1t57wvvAoGZhn3H3BtGk,1059
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241016.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241016.dist-info/METADATA,sha256=3cBwGMlr5S-mHHm8ZXtnMUNjCcSYeAzwOEf-N4LxLEU,18945
-skypilot_nightly-1.0.0.dev20241016.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-skypilot_nightly-1.0.0.dev20241016.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241016.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241016.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241018.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241018.dist-info/METADATA,sha256=hKti-qYovHe9BXjvZnYoV-88kOo2Qz0-xTDwY08RzrM,18945
+skypilot_nightly-1.0.0.dev20241018.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+skypilot_nightly-1.0.0.dev20241018.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241018.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241018.dist-info/RECORD,,

sky/skylet/providers/lambda_cloud/__init__.py

@@ -1,2 +0,0 @@
-"""Lambda Cloud node provider"""
-from sky.skylet.providers.lambda_cloud.node_provider import LambdaNodeProvider

sky/skylet/providers/lambda_cloud/node_provider.py

@@ -1,320 +0,0 @@
-import logging
-import os
-from threading import RLock
-import time
-from typing import Any, Dict, List, Optional
-
-from ray.autoscaler.node_provider import NodeProvider
-from ray.autoscaler.tags import NODE_KIND_HEAD
-from ray.autoscaler.tags import NODE_KIND_WORKER
-from ray.autoscaler.tags import STATUS_UP_TO_DATE
-from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME
-from ray.autoscaler.tags import TAG_RAY_NODE_KIND
-from ray.autoscaler.tags import TAG_RAY_NODE_NAME
-from ray.autoscaler.tags import TAG_RAY_NODE_STATUS
-from ray.autoscaler.tags import TAG_RAY_USER_NODE_TYPE
-
-from sky import authentication as auth
-from sky.clouds.utils import lambda_utils
-from sky.utils import command_runner
-from sky.utils import common_utils
-from sky.utils import subprocess_utils
-from sky.utils import ux_utils
-
-_TAG_PATH_PREFIX = '~/.sky/generated/lambda_cloud/metadata'
-_REMOTE_SSH_KEY_NAME = '~/.lambda_cloud/ssh_key_name'
-_REMOTE_RAY_SSH_KEY = '~/ray_bootstrap_key.pem'
-_REMOTE_RAY_YAML = '~/ray_bootstrap_config.yaml'
-_GET_INTERNAL_IP_CMD = 's=$(ip -4 -br addr show | grep UP); echo "$s"; echo "$s" | grep -Eo "(10\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|172\.(1[6-9]|2[0-9]|3[0-1])|104\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"'
-
-logger = logging.getLogger(__name__)
-
-
-def synchronized(f):
-
-    def wrapper(self, *args, **kwargs):
-        self.lock.acquire()
-        try:
-            return f(self, *args, **kwargs)
-        finally:
-            self.lock.release()
-
-    return wrapper
-
-
-class LambdaNodeProvider(NodeProvider):
-    """Node Provider for Lambda Cloud.
-
-    This provider assumes Lambda Cloud credentials are set.
-    """
-
-    def __init__(self, provider_config: Dict[str, Any],
-                 cluster_name: str) -> None:
-        NodeProvider.__init__(self, provider_config, cluster_name)
-        self.lock = RLock()
-        self.lambda_client = lambda_utils.LambdaCloudClient()
-        self.cached_nodes: Dict[str, Dict[str, Any]] = {}
-        self.metadata = lambda_utils.Metadata(_TAG_PATH_PREFIX, cluster_name)
-        self.ssh_key_path = os.path.expanduser(auth.PRIVATE_SSH_KEY_PATH)
-
-        def _get_ssh_key_name(prefix: str) -> str:
-            public_key_path = os.path.expanduser(auth.PUBLIC_SSH_KEY_PATH)
-            with open(public_key_path, 'r') as f:
-                public_key = f.read()
-            name, exists = self.lambda_client.get_unique_ssh_key_name(
-                prefix, public_key)
-            if not exists:
-                raise lambda_utils.LambdaCloudError('SSH key not found')
-            return name
-
-        ray_yaml_path = os.path.expanduser(_REMOTE_RAY_YAML)
-        self.on_head = (os.path.exists(ray_yaml_path) and
-                        common_utils.read_yaml(ray_yaml_path)['cluster_name']
-                        == cluster_name)
-
-        if self.on_head:
-            self.ssh_key_path = os.path.expanduser(_REMOTE_RAY_SSH_KEY)
-            ssh_key_name_path = os.path.expanduser(_REMOTE_SSH_KEY_NAME)
-            if os.path.exists(ssh_key_name_path):
-                with open(ssh_key_name_path, 'r') as f:
-                    self.ssh_key_name = f.read()
-            else:
-                # At this point, `~/.ssh/sky-key.pub` contains the public
-                # key used to launch this cluster. Use it to determine
-                # ssh key name and store the name in _REMOTE_SSH_KEY_NAME.
-                # Note: this case only runs during cluster launch, so it is
-                # not possible for ~/.ssh/sky-key.pub to already be regenerated
-                # by the user.
-                self.ssh_key_name = _get_ssh_key_name('')
-                with open(ssh_key_name_path, 'w', encoding='utf-8') as f:
-                    f.write(self.ssh_key_name)
-        else:
-            # On local
-            self.ssh_key_name = _get_ssh_key_name(
-                f'sky-key-{common_utils.get_user_hash()}')
-
-    def _guess_and_add_missing_tags(self, vms: List[Dict[str, Any]]) -> None:
-        """Adds missing vms to local tag file and guesses their tags."""
-        for node in vms:
-            if self.metadata.get(node['id']) is not None:
-                pass
-            elif node['name'] == f'{self.cluster_name}-head':
-                self.metadata.set(
-                    node['id'], {
-                        'tags': {
-                            TAG_RAY_CLUSTER_NAME: self.cluster_name,
-                            TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE,
-                            TAG_RAY_NODE_KIND: NODE_KIND_HEAD,
-                            TAG_RAY_USER_NODE_TYPE: 'ray_head_default',
-                            TAG_RAY_NODE_NAME: f'ray-{self.cluster_name}-head',
-                        }
-                    })
-            elif node['name'] == f'{self.cluster_name}-worker':
-                self.metadata.set(
-                    node['id'], {
-                        'tags': {
-                            TAG_RAY_CLUSTER_NAME: self.cluster_name,
-                            TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE,
-                            TAG_RAY_NODE_KIND: NODE_KIND_WORKER,
-                            TAG_RAY_USER_NODE_TYPE: 'ray_worker_default',
-                            TAG_RAY_NODE_NAME: f'ray-{self.cluster_name}-worker',
-                        }
-                    })
-
-    def _list_instances_in_cluster(self) -> List[Dict[str, Any]]:
-        """List running instances in cluster."""
-        vms = self.lambda_client.list_instances()
-        possible_names = [
-            f'{self.cluster_name}-head', f'{self.cluster_name}-worker'
-        ]
-        return [node for node in vms if node.get('name') in possible_names]
-
-    @synchronized
-    def _get_filtered_nodes(self, tag_filters: Dict[str,
-                                                    str]) -> Dict[str, Any]:
-
-        def _extract_metadata(vm: Dict[str, Any]) -> Dict[str, Any]:
-            metadata = {'id': vm['id'], 'status': vm['status'], 'tags': {}}
-            instance_info = self.metadata.get(vm['id'])
-            if instance_info is not None:
-                metadata['tags'] = instance_info['tags']
-            metadata['external_ip'] = vm.get('ip')
-            return metadata
-
-        def _match_tags(vm: Dict[str, Any]):
-            vm_info = self.metadata.get(vm['id'])
-            tags = {} if vm_info is None else vm_info['tags']
-            for k, v in tag_filters.items():
-                if tags.get(k) != v:
-                    return False
-            return True
-
-        def _get_internal_ip(node: Dict[str, Any]):
-            # TODO(ewzeng): cache internal ips in metadata file to reduce
-            # ssh overhead.
-            if node['external_ip'] is None or node['status'] != 'active':
-                node['internal_ip'] = None
-                return
-            runner = command_runner.SSHCommandRunner(
-                node=(node['external_ip'], 22),
-                ssh_user='ubuntu',
-                ssh_private_key=self.ssh_key_path)
-            rc, stdout, stderr = runner.run(_GET_INTERNAL_IP_CMD,
-                                            require_outputs=True,
-                                            stream_logs=False)
-            subprocess_utils.handle_returncode(
-                rc,
-                _GET_INTERNAL_IP_CMD,
-                'Failed get obtain private IP from node',
-                stderr=stdout + stderr)
-            node['internal_ip'] = stdout.strip()
-
-        vms = self._list_instances_in_cluster()
-        self.metadata.refresh([node['id'] for node in vms])
-        self._guess_and_add_missing_tags(vms)
-        nodes = [_extract_metadata(vm) for vm in filter(_match_tags, vms)]
-        nodes = [
-            node for node in nodes
-            if node['status'] not in ['terminating', 'terminated']
-        ]
-        subprocess_utils.run_in_parallel(_get_internal_ip, nodes)
-        self.cached_nodes = {node['id']: node for node in nodes}
-        return self.cached_nodes
-
-    def non_terminated_nodes(self, tag_filters: Dict[str, str]) -> List[str]:
-        """Return a list of node ids filtered by the specified tags dict.
-
-        This list must not include terminated nodes. For performance reasons,
-        providers are allowed to cache the result of a call to
-        non_terminated_nodes() to serve single-node queries
-        (e.g. is_running(node_id)). This means that non_terminated_nodes() must
-        be called again to refresh results.
-
-        Examples:
-            >>> provider.non_terminated_nodes({TAG_RAY_NODE_KIND: "worker"})
-            ["node-1", "node-2"]
-        """
-        nodes = self._get_filtered_nodes(tag_filters=tag_filters)
-        return [k for k, _ in nodes.items()]
-
-    def is_running(self, node_id: str) -> bool:
-        """Return whether the specified node is running."""
-        return self._get_cached_node(node_id=node_id) is not None
-
-    def is_terminated(self, node_id: str) -> bool:
-        """Return whether the specified node is terminated."""
-        return self._get_cached_node(node_id=node_id) is None
-
-    def node_tags(self, node_id: str) -> Dict[str, str]:
-        """Returns the tags of the given node (string dict)."""
-        node = self._get_cached_node(node_id=node_id)
-        if node is None:
-            return {}
-        return node['tags']
-
-    def external_ip(self, node_id: str) -> Optional[str]:
-        """Returns the external ip of the given node."""
-        node = self._get_cached_node(node_id=node_id)
-        if node is None:
-            return None
-        ip = node.get('external_ip')
-        with ux_utils.print_exception_no_traceback():
-            if ip is None:
-                raise lambda_utils.LambdaCloudError(
-                    'A node ip address was not found. Either '
-                    '(1) Lambda Cloud has internally errored, or '
-                    '(2) the cluster is still booting. '
-                    'You can manually terminate the cluster on the '
-                    'Lambda Cloud console or (in case 2) wait for '
-                    'booting to finish (~2 minutes).')
-        return ip
-
-    def internal_ip(self, node_id: str) -> Optional[str]:
-        """Returns the internal ip (Ray ip) of the given node."""
-        node = self._get_cached_node(node_id=node_id)
-        if node is None:
-            return None
-        ip = node.get('internal_ip')
-        with ux_utils.print_exception_no_traceback():
-            if ip is None:
-                raise lambda_utils.LambdaCloudError(
-                    'A node ip address was not found. Either '
-                    '(1) Lambda Cloud has internally errored, or '
-                    '(2) the cluster is still booting. '
-                    'You can manually terminate the cluster on the '
-                    'Lambda Cloud console or (in case 2) wait for '
-                    'booting to finish (~2 minutes).')
-        return ip
-
-    def create_node(self, node_config: Dict[str, Any], tags: Dict[str, str],
-                    count: int) -> None:
-        """Creates a number of nodes within the namespace."""
-        # Get tags
-        config_tags = node_config.get('tags', {}).copy()
-        config_tags.update(tags)
-        config_tags[TAG_RAY_CLUSTER_NAME] = self.cluster_name
-
-        # Create nodes
-        instance_type = node_config['InstanceType']
-        region = self.provider_config['region']
-
-        if config_tags[TAG_RAY_NODE_KIND] == NODE_KIND_HEAD:
-            name = f'{self.cluster_name}-head'
-            # Occasionally, the head node will continue running for a short
-            # period after termination. This can lead to the following bug:
-            #   1. Head node autodowns but continues running.
-            #   2. The next autodown event is triggered, which executes ray up.
-            #   3. Head node stops running.
-            # In this case, a new head node is created after the cluster has
-            # terminated. We avoid this with the following check:
-            if self.on_head:
-                raise lambda_utils.LambdaCloudError('Head already exists.')
-        else:
-            name = f'{self.cluster_name}-worker'
-
-        # Lambda launch api only supports launching one node at a time,
-        # so we do a loop. Remove loop when launch api allows quantity > 1
-        booting_list = []
-        for _ in range(count):
-            vm_id = self.lambda_client.create_instances(
-                instance_type=instance_type,
-                region=region,
-                quantity=1,
-                name=name,
-                ssh_key_name=self.ssh_key_name)[0]
-            self.metadata.set(vm_id, {'tags': config_tags})
-            booting_list.append(vm_id)
-            time.sleep(10)  # Avoid api rate limits
-
-        # Wait for nodes to finish booting
-        while True:
-            vms = self._list_instances_in_cluster()
-            for vm_id in booting_list.copy():
-                for vm in vms:
-                    if vm['id'] == vm_id and vm['status'] == 'active':
-                        booting_list.remove(vm_id)
-            if len(booting_list) == 0:
-                return
-            time.sleep(10)
-
-    @synchronized
-    def set_node_tags(self, node_id: str, tags: Dict[str, str]) -> None:
-        """Sets the tag values (string dict) for the specified node."""
-        node = self._get_node(node_id)
-        assert node is not None, node_id
-        node['tags'].update(tags)
-        self.metadata.set(node_id, {'tags': node['tags']})
-
-    def terminate_node(self, node_id: str) -> None:
-        """Terminates the specified node."""
-        self.lambda_client.remove_instances(node_id)
-        self.metadata.set(node_id, None)
-
-    def _get_node(self, node_id: str) -> Optional[Dict[str, Any]]:
-        self._get_filtered_nodes({})  # Side effect: updates cache
-        return self.cached_nodes.get(node_id, None)
-
-    def _get_cached_node(self, node_id: str) -> Optional[Dict[str, Any]]:
-        if node_id in self.cached_nodes:
-            return self.cached_nodes[node_id]
-        return self._get_node(node_id=node_id)