skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/aws/config.py

@@ -19,6 +19,7 @@ import colorama
 from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import aws
+from sky.clouds import aws as aws_cloud
 from sky.provision import common
 from sky.provision.aws import utils
 from sky.utils import annotations
@@ -86,6 +87,9 @@ def bootstrap_instances(
         use_internal_ips=config.provider_config.get('use_internal_ips', False),
         vpc_name=config.provider_config.get('vpc_name'))
 
+    max_efa_interfaces = config.provider_config.get('max_efa_interfaces', 0)
+    enable_efa = max_efa_interfaces > 0
+
     # Cluster workers should be in a security group that permits traffic within
     # the group, and also SSH access from outside.
     if security_group_ids is None:
@@ -102,7 +106,32 @@ def bootstrap_instances(
             extended_ip_rules = []
         security_group_ids = _configure_security_group(ec2, vpc_id,
                                                        expected_sg_name,
-                                                       extended_ip_rules)
+                                                       extended_ip_rules,
+                                                       enable_efa)
+        if expected_sg_name != aws_cloud.DEFAULT_SECURITY_GROUP_NAME:
+            logger.debug('Attempting to create the default security group.')
+            # Attempt to create the default security group. This is needed
+            # to enable us to use the default security group to quickly
+            # delete the cluster. If the default security group is not created,
+            # we will need to block on instance termination to delete the
+            # security group.
+            try:
+                _configure_security_group(ec2, vpc_id,
+                                          aws_cloud.DEFAULT_SECURITY_GROUP_NAME,
+                                          [], enable_efa)
+                logger.debug('Default security group created.')
+            except exceptions.NoClusterLaunchedError as e:
+                if 'not authorized to perform: ec2:CreateSecurityGroup' in str(
+                        e):
+                    # User does not have permission to create the default
+                    # security group.
+                    logger.debug('User does not have permission to create '
+                                 'the default security group. '
+                                 f'{e}')
+                    pass
+                else:
+                    raise e
+
         end_time = time.time()
         elapsed = end_time - start_time
         logger.info(
@@ -123,6 +152,37 @@ def bootstrap_instances(
     return config
 
 
+def _configure_placement_group(ec2: 'mypy_boto3_ec2.ServiceResource',
+                               placement_group_name: str):
+    """Configure placement group for the cluster."""
+    # Create the placement group
+    logger.info(f'Creating placement group {placement_group_name}.')
+    try:
+        ec2.meta.client.create_placement_group(GroupName=placement_group_name,
+                                               Strategy='cluster')
+    except aws.botocore_exceptions().ClientError as exc:
+        if exc.response.get(
+                'Error', {}).get('Code') == 'InvalidPlacementGroup.Duplicate':
+            logger.debug(
+                f'Placement group {placement_group_name} already exists.')
+        else:
+            raise exc
+
+
+def delete_placement_group(ec2: 'mypy_boto3_ec2.ServiceResource',
+                           placement_group_name: str):
+    """Delete the placement group."""
+    try:
+        ec2.meta.client.delete_placement_group(GroupName=placement_group_name)
+    except aws.botocore_exceptions().ClientError as exc:
+        if exc.response.get('Error',
+                            {}).get('Code') == 'InvalidPlacementGroup.Unknown':
+            logger.debug(
+                f'Placement group {placement_group_name} does not exist.')
+        else:
+            raise exc
+
+
 def _configure_iam_role(iam) -> Dict[str, Any]:
 
     def _get_instance_profile(profile_name: str):
@@ -245,7 +305,10 @@ def _get_route_tables(ec2: 'mypy_boto3_ec2.ServiceResource',
     Returns:
         A list of route tables associated with the options VPC and region
     """
-    filters = [{'Name': 'association.main', 'Values': [str(main).lower()]}]
+    filters: List['ec2_type_defs.FilterTypeDef'] = [{
+        'Name': 'association.main',
+        'Values': [str(main).lower()],
+    }]
     if vpc_id is not None:
         filters.append({'Name': 'vpc-id', 'Values': [vpc_id]})
     logger.debug(
@@ -346,10 +409,26 @@ def _usable_subnets(
                 s for s in candidate_subnets if s.vpc_id == vpc_id_of_sg
             ]
 
+            if not candidate_subnets:
+                _skypilot_log_error_and_exit_for_failover(
+                    'No candidate subnets found in specified VPC '
+                    f'{vpc_id_of_sg}.')
+
         available_subnets = [
             s for s in candidate_subnets if s.state == 'available'
         ]
 
+        if not available_subnets:
+            _skypilot_log_error_and_exit_for_failover(
+                'All candidate subnets are pending in specified VPC '
+                f'{vpc_id_of_sg}.')
+
+        if len(candidate_subnets) > len(available_subnets):
+            num_pruned = len(candidate_subnets) - len(available_subnets)
+            logger.debug(
+                f'{num_pruned} candidate subnets pruned since they are not '
+                'available.')
+
         if use_internal_ips:
             # Get private subnets.
             #
@@ -361,6 +440,10 @@ def _usable_subnets(
                 if not _is_subnet_public(ec2, s.subnet_id, vpc_id_of_sg) and
                 not s.map_public_ip_on_launch
             ]
+            if not subnets:
+                _skypilot_log_error_and_exit_for_failover(
+                    'The use_internal_ips option is set to True, but all '
+                    'candidate subnets are public.')
         else:
             # Get public subnets.
             #
@@ -376,6 +459,10 @@ def _usable_subnets(
                 s for s in available_subnets
                 if _is_subnet_public(ec2, s.subnet_id, vpc_id_of_sg)
             ]
+            if not subnets:
+                _skypilot_log_error_and_exit_for_failover(
+                    'All candidate subnets are private, did you mean to '
+                    'set use_internal_ips to True?')
 
         subnets = sorted(
             subnets,
@@ -389,18 +476,7 @@ def _usable_subnets(
                                 'Failed to fetch available subnets from AWS.')
         raise exc
 
-    if not subnets:
-        vpc_msg = (f'Does a default VPC exist in region '
-                   f'{ec2.meta.client.meta.region_name}? ') if (
-                       vpc_id_of_sg is None) else ''
-        _skypilot_log_error_and_exit_for_failover(
-            f'No usable subnets found. {vpc_msg}'
-            'Try manually creating an instance in your specified region to '
-            'populate the list of subnets and try again. '
-            'Note that the subnet must map public IPs '
-            'on instance launch unless you set `use_internal_ips: true` in '
-            'the `provider` config.')
-    elif _are_user_subnets_pruned(subnets):
+    if _are_user_subnets_pruned(subnets):
         _skypilot_log_error_and_exit_for_failover(
             f'The specified subnets are not '
             f'usable: {_get_pruned_subnets(subnets)}')
@@ -473,8 +549,8 @@ def _vpc_id_from_security_group_ids(ec2: 'mypy_boto3_ec2.ServiceResource',
     return vpc_ids[0]
 
 
-def _get_vpc_id_by_name(ec2: 'mypy_boto3_ec2.ServiceResource', vpc_name: str,
-                        region: str) -> str:
+def get_vpc_id_by_name(ec2: 'mypy_boto3_ec2.ServiceResource', vpc_name: str,
+                       region: str) -> str:
     """Returns the VPC ID of the unique VPC with a given name.
 
     Exits with code 1 if:
@@ -507,7 +583,7 @@ def _get_subnet_and_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
                            use_internal_ips: bool,
                            vpc_name: Optional[str]) -> Tuple[Any, str]:
     if vpc_name is not None:
-        vpc_id_of_sg = _get_vpc_id_by_name(ec2, vpc_name, region)
+        vpc_id_of_sg = get_vpc_id_by_name(ec2, vpc_name, region)
     elif security_group_ids:
         vpc_id_of_sg = _vpc_id_from_security_group_ids(ec2, security_group_ids)
     else:
@@ -519,6 +595,11 @@ def _get_subnet_and_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
     # not want SkyPilot to use.
     if vpc_id_of_sg is None:
         all_subnets = [s for s in all_subnets if s.vpc.is_default]
+        if not all_subnets:
+            _skypilot_log_error_and_exit_for_failover(
+                f'The default VPC in {region} either does not exist or '
+                'has no subnets.')
+
     subnets, vpc_id = _usable_subnets(
         ec2,
         user_specified_subnets=None,
@@ -532,7 +613,8 @@ def _get_subnet_and_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
 
 def _configure_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
                               vpc_id: str, expected_sg_name: str,
-                              extended_ip_rules: List) -> List[str]:
+                              extended_ip_rules: List,
+                              enable_efa: bool) -> List[str]:
     security_group = _get_or_create_vpc_security_group(ec2, vpc_id,
                                                        expected_sg_name)
     sg_ids = [security_group.id]
@@ -558,16 +640,55 @@ def _configure_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
         },
         *extended_ip_rules,
     ]
+    outbound_rules = []
+    if enable_efa:
+        # EFA requires that outbound rules permit the same security group to
+        # communicate with each other
+        # Refer to https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start-nccl.html#nccl-start-base-setup # pylint: disable=line-too-long
+        outbound_rules.append({
+            'FromPort': -1,
+            'ToPort': -1,
+            'IpProtocol': '-1',
+            'UserIdGroupPairs': [{
+                'GroupId': i
+            } for i in sg_ids],
+        })
     # upsert the default security group
     if not security_group.ip_permissions:
         # If users specify security groups, we should not change the rules
         # of these security groups. Here we change it because it is the default
         # security group for SkyPilot.
         security_group.authorize_ingress(IpPermissions=inbound_rules)
+    if _need_to_update_outbound_rules(security_group, outbound_rules):
+        security_group.authorize_egress(IpPermissions=outbound_rules)
 
     return sg_ids
 
 
+def _need_to_update_outbound_rules(
+    security_group: Any,
+    outbound_rules: List[Dict[str, Any]],
+) -> bool:
+    """Check if we need to update the outbound rules of the security group."""
+    if not security_group.ip_permissions_egress:
+        return True  # No outbound rules, we need to add them
+    existing_group_ids = []
+    for rule in security_group.ip_permissions_egress:
+        if 'UserIdGroupPairs' in rule:
+            group_pairs = rule['UserIdGroupPairs']
+            for pair in group_pairs:
+                existing_group_ids.append(pair['GroupId'])
+    logger.debug(f'Existing group ids: {existing_group_ids}')
+    for rule in outbound_rules:
+        if 'UserIdGroupPairs' in rule:
+            group_pairs = rule['UserIdGroupPairs']
+            for pair in group_pairs:
+                if pair['GroupId'] not in existing_group_ids:
+                    logger.debug(f'New group id: {pair["GroupId"]}')
+                    return True  # New group id, we need to add it
+    return False  # No need to update
+
+
 def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
                                       vpc_id: str,
                                       expected_sg_name: str) -> Any:
@@ -589,8 +710,8 @@ def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
             due to AWS service issues.
     """
     # Figure out which security groups with this name exist for each VPC...
-    security_group = _get_security_group_from_vpc_id(ec2, vpc_id,
-                                                     expected_sg_name)
+    security_group = get_security_group_from_vpc_id(ec2, vpc_id,
+                                                    expected_sg_name)
     if security_group is not None:
         return security_group
 
@@ -606,7 +727,7 @@ def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
             # The security group already exists, but we didn't see it
             # because of eventual consistency.
             logger.warning(f'{expected_sg_name} already exists when creating.')
-            security_group = _get_security_group_from_vpc_id(
+            security_group = get_security_group_from_vpc_id(
                 ec2, vpc_id, expected_sg_name)
             assert (security_group is not None and
                     security_group.group_name == expected_sg_name), (
@@ -621,8 +742,8 @@ def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
         logger.warning(message)
         raise exceptions.NoClusterLaunchedError(message) from e
 
-    security_group = _get_security_group_from_vpc_id(ec2, vpc_id,
-                                                     expected_sg_name)
+    security_group = get_security_group_from_vpc_id(ec2, vpc_id,
+                                                    expected_sg_name)
     assert security_group is not None, 'Failed to create security group'
     logger.info(f'Created new security group {colorama.Style.BRIGHT}'
                 f'{security_group.group_name}{colorama.Style.RESET_ALL} '
@@ -630,9 +751,9 @@ def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
     return security_group
 
 
-def _get_security_group_from_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
-                                    vpc_id: str,
-                                    group_name: str) -> Optional[Any]:
+def get_security_group_from_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
+                                   vpc_id: str,
+                                   group_name: str) -> Optional[Any]:
     """Get security group by VPC ID and group name."""
     existing_groups = list(
         ec2.security_groups.filter(Filters=[{

sky/provision/aws/instance.py

@@ -10,7 +10,7 @@ from multiprocessing import pool
 import re
 import time
 import typing
-from typing import Any, Callable, Dict, List, Optional, Set, TypeVar
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar
 
 from sky import sky_logging
 from sky.adaptors import aws
@@ -18,6 +18,7 @@ from sky.clouds import aws as aws_cloud
 from sky.clouds.utils import aws_utils
 from sky.provision import common
 from sky.provision import constants
+from sky.provision.aws import config as aws_config
 from sky.provision.aws import utils
 from sky.utils import common_utils
 from sky.utils import resources_utils
@@ -183,9 +184,15 @@ def _merge_tag_specs(tag_specs: List[Dict[str, Any]],
             tag_specs += [user_tag_spec]
 
 
-def _create_instances(ec2_fail_fast, cluster_name: str,
-                      node_config: Dict[str, Any], tags: Dict[str, str],
-                      count: int, associate_public_ip_address: bool) -> List:
+def _create_instances(
+    ec2_fail_fast,
+    cluster_name: str,
+    node_config: Dict[str, Any],
+    tags: Dict[str, str],
+    count: int,
+    associate_public_ip_address: bool,
+    max_efa_interfaces: int,
+) -> List:
     tags = {
         'Name': cluster_name,
         constants.TAG_RAY_CLUSTER_NAME: cluster_name,
@@ -238,7 +245,36 @@ def _create_instances(ec2_fail_fast, cluster_name: str,
                 # Whether the VM(s) should have a public IP.
                 'AssociatePublicIpAddress': associate_public_ip_address,
                 'Groups': security_group_ids,
+                'InterfaceType': 'efa'
+                                 if max_efa_interfaces > 0 else 'interface',
             }]
+            # Due to AWS limitation, if an instance type supports multiple
+            # network cards, we cannot assign public IP addresses to the
+            # instance during creation, which will raise the following error:
+            #   (InvalidParameterCombination) when calling the RunInstances
+            #   operation: The associatePublicIPAddress parameter cannot be
+            #   specified when launching with multiple network interfaces.
+            # So we only attach multiple network interfaces if public IP is
+            # not required.
+            # TODO(hailong): support attaching/detaching elastic IP to expose
+            # public IP in this case.
+            if max_efa_interfaces > 1 and not associate_public_ip_address:
+                instance_type = conf['InstanceType']
+                for i in range(1, max_efa_interfaces):
+                    interface_type = 'efa-only'
+                    # Special handling for P5 instances
+                    # Refer to https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-acc-inst-types.html#efa-for-p5 for more details. # pylint: disable=line-too-long
+                    if (instance_type == 'p5.48xlarge' or
+                            instance_type == 'p5e.48xlarge'):
+                        interface_type = 'efa' if i % 4 == 0 else 'efa-only'
+                    network_interfaces.append({
+                        'SubnetId': subnet_id,
+                        'DeviceIndex': 1,
+                        'NetworkCardIndex': i,
+                        'AssociatePublicIpAddress': False,
+                        'Groups': security_group_ids,
+                        'InterfaceType': interface_type,
+                    })
             conf['NetworkInterfaces'] = network_interfaces
 
             instances = _ec2_call_with_retry_on_server_error(
@@ -275,9 +311,10 @@ def _get_head_instance_id(instances: List) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     ec2 = _default_ec2_resource(region)
     # NOTE: We set max_attempts=0 for fast failing when the resource is not
     # available (although the doc says it will only retry for network
@@ -288,6 +325,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     zone = None
     resumed_instance_ids: List[str] = []
     created_instance_ids: List[str] = []
+    max_efa_interfaces = config.provider_config.get('max_efa_interfaces', 0)
 
     # sort tags by key to support deterministic unit test stubbing
     tags = dict(sorted(copy.deepcopy(config.tags).items()))
@@ -503,7 +541,8 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                     tags,
                     reservation_count,
                     associate_public_ip_address=(
-                        not config.provider_config['use_internal_ips']))
+                        not config.provider_config['use_internal_ips']),
+                    max_efa_interfaces=max_efa_interfaces)
                 created_instances.extend(created_reserved_instances)
                 to_start_count -= reservation_count
                 if to_start_count <= 0:
@@ -526,7 +565,9 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                 tags,
                 to_start_count,
                 associate_public_ip_address=(
-                    not config.provider_config['use_internal_ips']))
+                    not config.provider_config['use_internal_ips']),
+                max_efa_interfaces=max_efa_interfaces)
+
             created_instances.extend(created_remaining_instances)
         created_instances.sort(key=lambda x: x.id)
 
@@ -585,11 +626,14 @@ def _filter_instances(ec2: 'mypy_boto3_ec2.ServiceResource',
 # stop() and terminate() for example already implicitly assume non-terminated.
 @common_utils.retry
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     region = provider_config['region']
     ec2 = _default_ec2_resource(region)
@@ -608,12 +652,13 @@ def query_instances(
         'shutting-down': None,
         'terminated': None,
     }
-    statuses = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst in instances:
         status = status_map[inst.state['Name']]
         if non_terminated_only and status is None:
             continue
-        statuses[inst.id] = status
+        statuses[inst.id] = (status, None)
     return statuses
 
 
@@ -681,19 +726,43 @@ def terminate_instances(
                                   filters,
                                   included_instances=None,
                                   excluded_instances=None)
-    instances_list = list(instances)
-    instances.terminate()
-    if (sg_name == aws_cloud.DEFAULT_SECURITY_GROUP_NAME or
-            not managed_by_skypilot):
-        # Using default AWS SG or user specified security group. We don't need
-        # to wait for the termination of the instances, as we do not need to
-        # delete the SG.
-        return
-    # If ports are specified, we need to delete the newly created Security
-    # Group. Here we wait for all instances to be terminated, since the
-    # Security Group dependent on them.
-    for instance in instances_list:
-        instance.wait_until_terminated()
+    instance_list = list(instances)
+    default_sg = aws_config.get_security_group_from_vpc_id(
+        ec2, _get_vpc_id(provider_config),
+        aws_cloud.DEFAULT_SECURITY_GROUP_NAME)
+    if sg_name == aws_cloud.DEFAULT_SECURITY_GROUP_NAME:
+        # Case 1: The default SG is used, we don't need to ensure instance are
+        # terminated.
+        instances.terminate()
+    elif not managed_by_skypilot:
+        # Case 2: We are not managing the non-default sg. We don't need to
+        # ensure instances are terminated.
+        instances.terminate()
+    elif (managed_by_skypilot and default_sg is not None):
+        # Case 3: We are managing the non-default sg. The default SG exists
+        # so we can move the instances to the default SG and terminate them
+        # without blocking.
+
+        # Make this multithreaded: modify all instances' SGs in parallel.
+        def modify_instance_sg(instance):
+            instance.modify_attribute(Groups=[default_sg.id])
+            logger.debug(f'Instance {instance.id} modified to use default SG:'
+                         f'{default_sg.id} for quick deletion.')
+
+        with pool.ThreadPool() as thread_pool:
+            thread_pool.map(modify_instance_sg, instances)
+            thread_pool.close()
+            thread_pool.join()
+
+        instances.terminate()
+    else:
+        # Case 4: We are managing the non-default sg. The default SG does not
+        # exist. We must block on instance termination so that we can
+        # delete the security group.
+        instances.terminate()
+        for instance in instance_list:
+            instance.wait_until_terminated()
+
     # TODO(suquark): Currently, the implementation of GCP and Azure will
     #  wait util the cluster is fully terminated, while other clouds just
     #  trigger the termination process (via http call) and then return.
@@ -702,30 +771,6 @@ def terminate_instances(
     #  of most cloud implementations (including AWS).
 
 
-def _get_sg_from_name(
-    ec2: Any,
-    sg_name: str,
-) -> Any:
-    # GroupNames will only filter SGs in the default VPC, so we need to use
-    # Filters here. Ref:
-    # https://boto3.amazonaws.com/v1/documentation/api/1.26.112/reference/services/ec2/service-resource/security_groups.html  # pylint: disable=line-too-long
-    sgs = ec2.security_groups.filter(Filters=[{
-        'Name': 'group-name',
-        'Values': [sg_name]
-    }])
-    num_sg = len(list(sgs))
-    if num_sg == 0:
-        logger.warning(f'Expected security group {sg_name} not found. ')
-        return None
-    if num_sg > 1:
-        # TODO(tian): Better handle this case. Maybe we can check when creating
-        # the SG and throw an error if there is already an existing SG with the
-        # same name.
-        logger.warning(f'Found {num_sg} security groups with name {sg_name}. ')
-        return None
-    return list(sgs)[0]
-
-
 def _maybe_move_to_new_sg(
     instance: Any,
     expected_sg: Any,
@@ -778,7 +823,9 @@ def open_ports(
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Instance with cluster name '
                              f'{cluster_name_on_cloud} not found.')
-    sg = _get_sg_from_name(ec2, sg_name)
+    sg = aws_config.get_security_group_from_vpc_id(ec2,
+                                                   _get_vpc_id(provider_config),
+                                                   sg_name)
     if sg is None:
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Cannot find new security group '
@@ -836,7 +883,23 @@ def open_ports(
 
     # For the case when every new ports is already opened.
     if ip_permissions:
-        sg.authorize_ingress(IpPermissions=ip_permissions)
+        # Filter out any permissions that already exist in the security group
+        existing_permissions = set()
+        for rule in sg.ip_permissions:
+            if rule['IpProtocol'] == 'tcp':
+                for ip_range in rule.get('IpRanges', []):
+                    if ip_range.get('CidrIp') == '0.0.0.0/0':
+                        existing_permissions.add(
+                            (rule['FromPort'], rule['ToPort']))
+
+        # Remove any permissions that already exist
+        filtered_permissions = []
+        for perm in ip_permissions:
+            if (perm['FromPort'], perm['ToPort']) not in existing_permissions:
+                filtered_permissions.append(perm)
+
+        if filtered_permissions:
+            sg.authorize_ingress(IpPermissions=filtered_permissions)
 
 
 def cleanup_ports(
@@ -858,7 +921,9 @@ def cleanup_ports(
         # We only want to delete the SG that is dedicated to this cluster (i.e.,
         # this cluster have opened some ports).
         return
-    sg = _get_sg_from_name(ec2, sg_name)
+    sg = aws_config.get_security_group_from_vpc_id(ec2,
+                                                   _get_vpc_id(provider_config),
+                                                   sg_name)
     if sg is None:
         logger.warning(
             'Find security group failed. Skip cleanup security group.')
@@ -969,3 +1034,23 @@ def get_cluster_info(
         provider_name='aws',
         provider_config=provider_config,
     )
+
+
+def _get_vpc_id(provider_config: Dict[str, Any]) -> str:
+    region = provider_config['region']
+    ec2 = _default_ec2_resource(provider_config['region'])
+    if 'vpc_name' in provider_config:
+        return aws_config.get_vpc_id_by_name(ec2, provider_config['vpc_name'],
+                                             region)
+    else:
+        # Retrieve the default VPC name from the region.
+        response = ec2.meta.client.describe_vpcs(Filters=[{
+            'Name': 'isDefault',
+            'Values': ['true']
+        }])
+        if len(response['Vpcs']) == 0:
+            raise ValueError(f'No default VPC found in region {region}')
+        elif len(response['Vpcs']) > 1:
+            raise ValueError(f'Multiple default VPCs found in region {region}')
+        else:
+            return response['Vpcs'][0]['VpcId']

sky/provision/azure/instance.py

@@ -362,9 +362,10 @@ def _create_instances(compute_client: 'azure_compute.ComputeManagementClient',
     return instances
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     # TODO(zhwu): This function is too long. We should refactor it.
     provider_config = config.provider_config
     resource_group = provider_config['resource_group']
@@ -952,11 +953,14 @@ def delete_vm_and_attached_resources(subscription_id: str, resource_group: str,
 
 @common_utils.retry
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, cluster_name_on_cloud
 
     subscription_id = provider_config['subscription_id']
@@ -964,7 +968,8 @@ def query_instances(
     filters = {constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud}
     compute_client = azure.get_client('compute', subscription_id)
     nodes = _filter_instances(compute_client, resource_group, filters)
-    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
 
     def _fetch_and_map_status(node, resource_group: str) -> None:
         compute_client = azure.get_client('compute', subscription_id)
@@ -972,8 +977,8 @@ def query_instances(
 
         if status is None and non_terminated_only:
             return
-        statuses[node.name] = (None if status is None else
-                               status.to_cluster_status())
+        statuses[node.name] = ((None if status is None else
+                                status.to_cluster_status()), None)
 
     with pool.ThreadPool() as p:
         p.starmap(_fetch_and_map_status,