skypilot-nightly 1.0.0.dev20241028__py3-none-any.whl → 1.0.0.dev20241029__py3-none-any.whl

sky/provision/azure/instance.py

@@ -41,6 +41,15 @@ UNIQUE_ID_LEN = 4
 _TAG_SKYPILOT_VM_ID = 'skypilot-vm-id'
 _WAIT_CREATION_TIMEOUT_SECONDS = 600
 
+_RESOURCE_MANAGED_IDENTITY_TYPE = (
+    'Microsoft.ManagedIdentity/userAssignedIdentities')
+_RESOURCE_NETWORK_SECURITY_GROUP_TYPE = (
+    'Microsoft.Network/networkSecurityGroups')
+_RESOURCE_VIRTUAL_NETWORK_TYPE = 'Microsoft.Network/virtualNetworks'
+_RESOURCE_PUBLIC_IP_ADDRESS_TYPE = 'Microsoft.Network/publicIPAddresses'
+_RESOURCE_VIRTUAL_MACHINE_TYPE = 'Microsoft.Compute/virtualMachines'
+_RESOURCE_NETWORK_INTERFACE_TYPE = 'Microsoft.Network/networkInterfaces'
+
 _RESOURCE_GROUP_NOT_FOUND_ERROR_MESSAGE = 'ResourceGroupNotFound'
 _POLL_INTERVAL = 1
 # TODO(Doyoung): _LEGACY_NSG_NAME can be remove this after 0.8.0 to ignore
@@ -282,6 +291,7 @@ def _create_vm(
         image_reference=image_reference,
         os_disk=compute.OSDisk(
             create_option=compute.DiskCreateOptionTypes.FROM_IMAGE,
+            delete_option=compute.DiskDeleteOptionTypes.DELETE,
             managed_disk=compute.ManagedDiskParameters(
                 storage_account_type=node_config['azure_arm_parameters']
                 ['osDiskTier']),
@@ -697,18 +707,30 @@ def terminate_instances(
 
     assert provider_config is not None, cluster_name_on_cloud
 
-    resource_group_client = azure.get_client('resource', subscription_id)
-    delete_resource_group = _get_azure_sdk_function(
-        client=resource_group_client.resource_groups, function_name='delete')
-
-    try:
-        delete_resource_group(resource_group, force_deletion_types=None)
-    except azure.exceptions().ResourceNotFoundError as e:
-        if 'ResourceGroupNotFound' in str(e):
-            logger.warning(f'Resource group {resource_group} not found. Skip '
-                           'terminating it.')
-            return
-        raise
+    use_external_resource_group = provider_config.get(
+        'use_external_resource_group', False)
+    # When user specified resource group through config.yaml to create a VM, we
+    # cannot remove the entire resource group as it may contain other resources
+    # unrelated to this VM being removed.
+    if use_external_resource_group:
+        delete_vm_and_attached_resources(subscription_id, resource_group,
+                                         cluster_name_on_cloud)
+    else:
+        # For SkyPilot default resource groups, delete entire resource group.
+        # This automatically terminates all resources within, including VMs
+        resource_group_client = azure.get_client('resource', subscription_id)
+        delete_resource_group = _get_azure_sdk_function(
+            client=resource_group_client.resource_groups,
+            function_name='delete')
+        try:
+            delete_resource_group(resource_group, force_deletion_types=None)
+        except azure.exceptions().ResourceNotFoundError as e:
+            if 'ResourceGroupNotFound' in str(e):
+                logger.warning(
+                    f'Resource group {resource_group} not found. Skip '
+                    'terminating it.')
+                return
+            raise
 
 
 def _get_instance_status(
@@ -770,6 +792,188 @@ def _filter_instances(
     return nodes
 
 
+def _delete_nic_with_retries(network_client,
+                             resource_group,
+                             nic_name,
+                             max_retries=15,
+                             retry_interval=20):
+    """Delete a NIC with retries.
+
+    When a VM is created, its NIC is reserved for 180 seconds, preventing its
+    immediate deletion. If the NIC is in this reserved state, we must retry
+    deletion with intervals until the reservation expires. This situation
+    commonly arises if a VM termination is followed by a failover to another
+    region due to provisioning failures.
+    """
+    delete_network_interfaces = _get_azure_sdk_function(
+        client=network_client.network_interfaces, function_name='begin_delete')
+    for _ in range(max_retries):
+        try:
+            delete_network_interfaces(resource_group_name=resource_group,
+                                      network_interface_name=nic_name).result()
+            return
+        except azure.exceptions().HttpResponseError as e:
+            if 'NicReservedForAnotherVm' in str(e):
+                # Retry when deletion fails with reserved NIC.
+                logger.warning(f'NIC {nic_name} is reserved. '
+                               f'Retrying in {retry_interval} seconds...')
+                time.sleep(retry_interval)
+            else:
+                raise e
+    logger.error(
+        f'Failed to delete NIC {nic_name} after {max_retries} attempts.')
+
+
+def delete_vm_and_attached_resources(subscription_id: str, resource_group: str,
+                                     cluster_name_on_cloud: str) -> None:
+    """Removes VM with attached resources and Deployments.
+
+    This function deletes a virtual machine and its associated resources
+    (public IP addresses, virtual networks, managed identities, network
+    interface and network security groups) that match cluster_name_on_cloud.
+    There is one attached resources that is not removed within this
+    method: OS disk. It is configured to be deleted when VM is terminated while
+    setting up storage profile from _create_vm.
+
+    Args:
+        subscription_id: The Azure subscription ID.
+        resource_group: The name of the resource group.
+        cluster_name_on_cloud: The name of the cluster to filter resources.
+    """
+    resource_client = azure.get_client('resource', subscription_id)
+    try:
+        list_resources = _get_azure_sdk_function(
+            client=resource_client.resources,
+            function_name='list_by_resource_group')
+        resources = list(list_resources(resource_group))
+    except azure.exceptions().ResourceNotFoundError as e:
+        if _RESOURCE_GROUP_NOT_FOUND_ERROR_MESSAGE in str(e):
+            return
+        raise
+
+    filtered_resources: Dict[str, List[str]] = {
+        _RESOURCE_VIRTUAL_MACHINE_TYPE: [],
+        _RESOURCE_MANAGED_IDENTITY_TYPE: [],
+        _RESOURCE_NETWORK_SECURITY_GROUP_TYPE: [],
+        _RESOURCE_VIRTUAL_NETWORK_TYPE: [],
+        _RESOURCE_PUBLIC_IP_ADDRESS_TYPE: [],
+        _RESOURCE_NETWORK_INTERFACE_TYPE: []
+    }
+
+    for resource in resources:
+        if (resource.type in filtered_resources and
+                cluster_name_on_cloud in resource.name):
+            filtered_resources[resource.type].append(resource.name)
+
+    network_client = azure.get_client('network', subscription_id)
+    msi_client = azure.get_client('msi', subscription_id)
+    compute_client = azure.get_client('compute', subscription_id)
+    auth_client = azure.get_client('authorization', subscription_id)
+
+    delete_virtual_machine = _get_azure_sdk_function(
+        client=compute_client.virtual_machines, function_name='delete')
+    delete_public_ip_addresses = _get_azure_sdk_function(
+        client=network_client.public_ip_addresses, function_name='begin_delete')
+    delete_virtual_networks = _get_azure_sdk_function(
+        client=network_client.virtual_networks, function_name='begin_delete')
+    delete_managed_identity = _get_azure_sdk_function(
+        client=msi_client.user_assigned_identities, function_name='delete')
+    delete_network_security_group = _get_azure_sdk_function(
+        client=network_client.network_security_groups,
+        function_name='begin_delete')
+    delete_role_assignment = _get_azure_sdk_function(
+        client=auth_client.role_assignments, function_name='delete')
+
+    for vm_name in filtered_resources[_RESOURCE_VIRTUAL_MACHINE_TYPE]:
+        try:
+            # Before removing Network Interface, we need to wait for the VM to
+            # be completely removed with .result() so the dependency of VM on
+            # Network Interface is disassociated. This takes abour ~30s.
+            delete_virtual_machine(resource_group_name=resource_group,
+                                   vm_name=vm_name).result()
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete VM: {}'.format(e))
+
+    for nic_name in filtered_resources[_RESOURCE_NETWORK_INTERFACE_TYPE]:
+        try:
+            # Before removing Public IP Address, we need to wait for the
+            # Network Interface to be completely removed with .result() so the
+            # dependency of Network Interface on Public IP Address is
+            # disassociated. This takes about ~1s.
+            _delete_nic_with_retries(network_client, resource_group, nic_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete nic: {}'.format(e))
+
+    for public_ip_name in filtered_resources[_RESOURCE_PUBLIC_IP_ADDRESS_TYPE]:
+        try:
+            delete_public_ip_addresses(resource_group_name=resource_group,
+                                       public_ip_address_name=public_ip_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete public ip: {}'.format(e))
+
+    for vnet_name in filtered_resources[_RESOURCE_VIRTUAL_NETWORK_TYPE]:
+        try:
+            delete_virtual_networks(resource_group_name=resource_group,
+                                    virtual_network_name=vnet_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete vnet: {}'.format(e))
+
+    for msi_name in filtered_resources[_RESOURCE_MANAGED_IDENTITY_TYPE]:
+        user_assigned_identities = (
+            msi_client.user_assigned_identities.list_by_resource_group(
+                resource_group_name=resource_group))
+        for identity in user_assigned_identities:
+            if msi_name == identity.name:
+                # We use the principal_id to find the correct guid converted
+                # role assignment name because each managed identity has a
+                # unique principal_id, and role assignments are associated
+                # with security principals (like managed identities) via this
+                # principal_id.
+                target_principal_id = identity.principal_id
+                scope = (f'/subscriptions/{subscription_id}'
+                         f'/resourceGroups/{resource_group}')
+                role_assignments = auth_client.role_assignments.list_for_scope(
+                    scope)
+                for assignment in role_assignments:
+                    if target_principal_id == assignment.principal_id:
+                        guid_role_assignment_name = assignment.name
+                        try:
+                            delete_role_assignment(
+                                scope=scope,
+                                role_assignment_name=guid_role_assignment_name)
+                        except Exception as e:  # pylint: disable=broad-except
+                            logger.warning('Failed to delete role '
+                                           'assignment: {}'.format(e))
+                        break
+        try:
+            delete_managed_identity(resource_group_name=resource_group,
+                                    resource_name=msi_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete msi: {}'.format(e))
+
+    for nsg_name in filtered_resources[_RESOURCE_NETWORK_SECURITY_GROUP_TYPE]:
+        try:
+            delete_network_security_group(resource_group_name=resource_group,
+                                          network_security_group_name=nsg_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete nsg: {}'.format(e))
+
+    delete_deployment = _get_azure_sdk_function(
+        client=resource_client.deployments, function_name='begin_delete')
+    deployment_names = [
+        constants.EXTERNAL_RG_BOOTSTRAP_DEPLOYMENT_NAME.format(
+            cluster_name_on_cloud=cluster_name_on_cloud),
+        constants.EXTERNAL_RG_VM_DEPLOYMENT_NAME.format(
+            cluster_name_on_cloud=cluster_name_on_cloud)
+    ]
+    for deployment_name in deployment_names:
+        try:
+            delete_deployment(resource_group_name=resource_group,
+                              deployment_name=deployment_name)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning('Failed to delete deployment: {}'.format(e))
+
+
 @common_utils.retry
 def query_instances(
     cluster_name_on_cloud: str,
@@ -842,66 +1046,67 @@ def open_ports(
     update_network_security_groups = _get_azure_sdk_function(
         client=network_client.network_security_groups,
         function_name='create_or_update')
+    list_network_security_groups = _get_azure_sdk_function(
+        client=network_client.network_security_groups, function_name='list')
 
-    try:
-        # Wait for the NSG creation to be finished before opening a port. The
-        # cluster provisioning triggers the NSG creation, but it may not be
-        # finished yet.
-        backoff = common_utils.Backoff(max_backoff_factor=1)
-        start_time = time.time()
-        while True:
-            nsg = _get_cluster_nsg(network_client, resource_group,
-                                   cluster_name_on_cloud)
-            if nsg.provisioning_state not in ['Creating', 'Updating']:
-                break
-            if time.time() - start_time > _WAIT_CREATION_TIMEOUT_SECONDS:
-                with ux_utils.print_exception_no_traceback():
-                    raise TimeoutError(
-                        f'Timed out while waiting for the Network '
-                        f'Security Group {nsg.name!r} to be ready for '
-                        f'cluster {cluster_name_on_cloud!r} in '
-                        f'resource group {resource_group!r}. The NSG '
-                        f'did not reach a stable state '
-                        '(Creating/Updating) within the allocated '
-                        f'{_WAIT_CREATION_TIMEOUT_SECONDS} seconds. '
-                        'Consequently, the operation to open ports '
-                        f'{ports} failed.')
-
-            backoff_time = backoff.current_backoff()
-            logger.info(f'NSG {nsg.name} is not created yet. Waiting for '
+    for nsg in list_network_security_groups(resource_group):
+        # Given resource group can contain network security groups that are
+        # irrelevant to this provisioning especially with user specified
+        # resource group at ~/.sky/config. So we make sure to check for the
+        # completion of nsg relevant to the VM being provisioned.
+        if cluster_name_on_cloud in nsg.name:
+            try:
+                # Wait the NSG creation to be finished before opening a port.
+                # The cluster provisioning triggers the NSG creation, but it
+                # may not be finished yet.
+                backoff = common_utils.Backoff(max_backoff_factor=1)
+                start_time = time.time()
+                while True:
+                    if nsg.provisioning_state not in ['Creating', 'Updating']:
+                        break
+                    if time.time(
+                    ) - start_time > _WAIT_CREATION_TIMEOUT_SECONDS:
+                        logger.warning(
+                            f'Fails to wait for the creation of NSG {nsg.name}'
+                            f' in {resource_group} within '
+                            f'{_WAIT_CREATION_TIMEOUT_SECONDS} seconds. '
+                            'Skip this NSG.')
+                    backoff_time = backoff.current_backoff()
+                    logger.info(
+                        f'NSG {nsg.name} is not created yet. Waiting for '
                         f'{backoff_time} seconds before checking again.')
-            time.sleep(backoff_time)
-
-        # Azure NSG rules have a priority field that determines the order
-        # in which they are applied. The priority must be unique across
-        # all inbound rules in one NSG.
-        priority = max(rule.priority
-                       for rule in nsg.security_rules
-                       if rule.direction == 'Inbound') + 1
-        nsg.security_rules.append(
-            azure.create_security_rule(
-                name=f'sky-ports-{cluster_name_on_cloud}-{priority}',
-                priority=priority,
-                protocol='Tcp',
-                access='Allow',
-                direction='Inbound',
-                source_address_prefix='*',
-                source_port_range='*',
-                destination_address_prefix='*',
-                destination_port_ranges=ports,
-            ))
-        poller = update_network_security_groups(resource_group, nsg.name, nsg)
-        poller.wait()
-        if poller.status() != 'Succeeded':
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(f'Failed to open ports {ports} in NSG '
-                                 f'{nsg.name}: {poller.status()}')
-
-    except azure.exceptions().HttpResponseError as e:
-        with ux_utils.print_exception_no_traceback():
-            raise ValueError(f'Failed to open ports {ports} in NSG for cluster '
-                             f'{cluster_name_on_cloud!r} within resource group '
-                             f'{resource_group!r}.') from e
+                    time.sleep(backoff_time)
+
+                # Azure NSG rules have a priority field that determines the
+                # order in which they are applied. The priority must be unique
+                # across all inbound rules in one NSG.
+                priority = max(rule.priority
+                               for rule in nsg.security_rules
+                               if rule.direction == 'Inbound') + 1
+                nsg.security_rules.append(
+                    azure.create_security_rule(
+                        name=f'sky-ports-{cluster_name_on_cloud}-{priority}',
+                        priority=priority,
+                        protocol='Tcp',
+                        access='Allow',
+                        direction='Inbound',
+                        source_address_prefix='*',
+                        source_port_range='*',
+                        destination_address_prefix='*',
+                        destination_port_ranges=ports,
+                    ))
+                poller = update_network_security_groups(resource_group,
+                                                        nsg.name, nsg)
+                poller.wait()
+                if poller.status() != 'Succeeded':
+                    with ux_utils.print_exception_no_traceback():
+                        raise ValueError(f'Failed to open ports {ports} in NSG '
+                                         f'{nsg.name}: {poller.status()}')
+            except azure.exceptions().HttpResponseError as e:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError(
+                        f'Failed to open ports {ports} in NSG {nsg.name}.'
+                    ) from e
 
 
 def cleanup_ports(

sky/provision/constants.py

@@ -16,3 +16,10 @@ WORKER_NODE_TAGS = {
     TAG_RAY_NODE_KIND: 'worker',
     TAG_SKYPILOT_HEAD_NODE: '0',
 }
+
+# Names for Azure Deployments.
+DEPLOYMENT_NAME = 'skypilot-config'
+LEGACY_DEPLOYMENT_NAME = 'ray-config'
+EXTERNAL_RG_BOOTSTRAP_DEPLOYMENT_NAME = (
+    'skypilot-bootstrap-{cluster_name_on_cloud}')
+EXTERNAL_RG_VM_DEPLOYMENT_NAME = 'skypilot-vm-{cluster_name_on_cloud}'

sky/provision/gcp/instance.py

@@ -632,13 +632,6 @@ def cleanup_ports(
     del ports  # Unused.
     assert provider_config is not None, cluster_name_on_cloud
     project_id = provider_config['project_id']
-    if 'ports' in provider_config:
-        # Backward compatibility for old provider config.
-        # TODO(tian): remove this after 2 minor releases, 0.6.0.
-        for port in provider_config['ports']:
-            firewall_rule_name = f'user-ports-{cluster_name_on_cloud}-{port}'
-            instance_utils.GCPComputeInstance.delete_firewall_rule(
-                project_id, firewall_rule_name)
     if 'firewall_rule' in provider_config:
         firewall_rule_name = provider_config['firewall_rule']
         instance_utils.GCPComputeInstance.delete_firewall_rule(

sky/serve/core.py

@@ -572,8 +572,6 @@ def status(
             'controller_port': (Optional[int]) controller port,
             'load_balancer_port': (Optional[int]) load balancer port,
             'policy': (Optional[str]) load balancer policy description,
-            'requested_resources': (sky.Resources) requested resources
-              for replica (deprecated),
             'requested_resources_str': (str) str representation of
               requested resources,
             'replica_info': (List[Dict[str, Any]]) replica information,

sky/serve/serve_state.py

@@ -34,7 +34,7 @@ _DB_PATH: str = _get_db_path()
 def create_table(cursor: 'sqlite3.Cursor', conn: 'sqlite3.Connection') -> None:
     """Creates the service and replica tables if they do not exist."""
 
-    # auto_restart column is deprecated.
+    # auto_restart and requested_resources column is deprecated.
     cursor.execute("""\
         CREATE TABLE IF NOT EXISTS services (
         name TEXT PRIMARY KEY,
@@ -323,8 +323,8 @@ def set_service_load_balancer_port(service_name: str,
 
 def _get_service_from_row(row) -> Dict[str, Any]:
     (current_version, name, controller_job_id, controller_port,
-     load_balancer_port, status, uptime, policy, _, requested_resources,
-     requested_resources_str, _, active_versions) = row[:13]
+     load_balancer_port, status, uptime, policy, _, _, requested_resources_str,
+     _, active_versions) = row[:13]
     return {
         'name': name,
         'controller_job_id': controller_job_id,
@@ -340,10 +340,6 @@ def _get_service_from_row(row) -> Dict[str, Any]:
         # The versions that is active for the load balancer. This is a list of
         # integers in json format. This is mainly for display purpose.
         'active_versions': json.loads(active_versions),
-        # TODO(tian): Backward compatibility.
-        # Remove after 2 minor release, 0.6.0.
-        'requested_resources': pickle.loads(requested_resources)
-                               if requested_resources is not None else None,
         'requested_resources_str': requested_resources_str,
     }
 

sky/serve/serve_utils.py

@@ -825,12 +825,7 @@ def format_service_table(service_records: List[Dict[str, Any]],
         replicas = _get_replicas(record)
         endpoint = get_endpoint(record)
         policy = record['policy']
-        # TODO(tian): Backward compatibility.
-        # Remove `requested_resources` field after 2 minor release, 0.6.0.
-        if record.get('requested_resources_str') is None:
-            requested_resources_str = str(record['requested_resources'])
-        else:
-            requested_resources_str = record['requested_resources_str']
+        requested_resources_str = record['requested_resources_str']
 
         service_values = [
             service_name,
@@ -1004,15 +999,8 @@ class ServeCodeGen:
     @classmethod
     def update_service(cls, service_name: str, version: int, mode: str) -> str:
         code = [
-            # Backward compatibility for old serve version on the remote
-            # machine. The `mode` argument was added in #3249, and if the remote
-            # machine has an old SkyPilot version before that, we need to avoid
-            # passing the `mode` argument to the job_lib functions.
-            # TODO(zhwu): Remove this in 0.7.0 release.
-            f'mode_kwargs = {{"mode": {mode!r}}} '
-            'if getattr(constants, "SERVE_VERSION", 0) >= 1 else {}',
             f'msg = serve_utils.update_service_encoded({service_name!r}, '
-            f'{version}, **mode_kwargs)',
+            f'{version}, mode={mode!r})',
             'print(msg, end="", flush=True)',
         ]
         return cls._build(code)

sky/serve/service_spec.py

@@ -29,13 +29,6 @@ class SkyServiceSpec:
         base_ondemand_fallback_replicas: Optional[int] = None,
         upscale_delay_seconds: Optional[int] = None,
         downscale_delay_seconds: Optional[int] = None,
-        # The following arguments are deprecated.
-        # TODO(ziming): remove this after 2 minor release, i.e. 0.6.0.
-        # Deprecated: Always be True
-        auto_restart: Optional[bool] = None,
-        # Deprecated: replaced by the target_qps_per_replica.
-        qps_upper_threshold: Optional[float] = None,
-        qps_lower_threshold: Optional[float] = None,
     ) -> None:
         if max_replicas is not None and max_replicas < min_replicas:
             with ux_utils.print_exception_no_traceback():
@@ -62,21 +55,6 @@ class SkyServiceSpec:
                 raise ValueError('readiness_path must start with a slash (/). '
                                  f'Got: {readiness_path}')
 
-        # TODO(tian): Following field are deprecated. Remove after 2 minor
-        # release, i.e. 0.6.0.
-        if qps_upper_threshold is not None or qps_lower_threshold is not None:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(
-                    'Field `qps_upper_threshold` and `qps_lower_threshold`'
-                    'under `replica_policy` are deprecated. '
-                    'Please use target_qps_per_replica instead.')
-        if auto_restart is not None:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(
-                    'Field `auto_restart` under `replica_policy` is deprecated.'
-                    'Currently, SkyServe will cleanup failed replicas'
-                    'and auto restart it to keep the service running.')
-
         self._readiness_path: str = readiness_path
         self._initial_delay_seconds: int = initial_delay_seconds
         self._readiness_timeout_seconds: int = readiness_timeout_seconds
@@ -160,14 +138,8 @@ class SkyServiceSpec:
             service_config['min_replicas'] = policy_section['min_replicas']
             service_config['max_replicas'] = policy_section.get(
                 'max_replicas', None)
-            service_config['qps_upper_threshold'] = policy_section.get(
-                'qps_upper_threshold', None)
-            service_config['qps_lower_threshold'] = policy_section.get(
-                'qps_lower_threshold', None)
             service_config['target_qps_per_replica'] = policy_section.get(
                 'target_qps_per_replica', None)
-            service_config['auto_restart'] = policy_section.get(
-                'auto_restart', None)
             service_config['upscale_delay_seconds'] = policy_section.get(
                 'upscale_delay_seconds', None)
             service_config['downscale_delay_seconds'] = policy_section.get(

sky/skylet/job_lib.py

@@ -827,14 +827,6 @@ class JobLibCodeGen:
         'import os',
         'import getpass',
         'from sky.skylet import job_lib, log_lib, constants',
-        # Backward compatibility for old skylet lib version on the remote
-        # machine. The `job_owner` argument was removed in #3037, and if the
-        # remote machine has an old SkyPilot version before that, we need to
-        # pass the `job_owner` argument to the job_lib functions.
-        # TODO(zhwu): Remove this in 0.7.0 release.
-        'job_owner_kwargs = {} '
-        'if getattr(constants, "SKYLET_LIB_VERSION", 0) >= 1 '
-        'else {"job_owner": getpass.getuser()}',
     ]
 
     @classmethod
@@ -861,7 +853,7 @@ class JobLibCodeGen:
 
     @classmethod
     def update_status(cls) -> str:
-        code = ['job_lib.update_status(**job_owner_kwargs)']
+        code = ['job_lib.update_status()']
         return cls._build(code)
 
     @classmethod
@@ -879,7 +871,7 @@ class JobLibCodeGen:
         """See job_lib.cancel_jobs()."""
         code = [
             (f'cancelled = job_lib.cancel_jobs_encoded_results('
-             f' {job_ids!r}, {cancel_all}, **job_owner_kwargs)'),
+             f' {job_ids!r}, {cancel_all})'),
             # Print cancelled IDs. Caller should parse by decoding.
             'print(cancelled, flush=True)',
         ]
@@ -902,7 +894,7 @@ class JobLibCodeGen:
             'run_timestamp = job_lib.get_run_timestamp(job_id)',
             f'log_dir = None if run_timestamp is None else os.path.join({constants.SKY_LOGS_DIRECTORY!r}, run_timestamp)',
             f'log_lib.tail_logs(job_id=job_id, log_dir=log_dir, '
-            f'managed_job_id={managed_job_id!r}, follow={follow}, **job_owner_kwargs)',
+            f'managed_job_id={managed_job_id!r}, follow={follow})',
         ]
         return cls._build(code)
 

sky/skylet/log_lib.py

@@ -186,20 +186,11 @@ def run_with_log(
             daemon_script = os.path.join(
                 os.path.dirname(os.path.abspath(job_lib.__file__)),
                 'subprocess_daemon.py')
-            if not hasattr(constants, 'SKY_GET_PYTHON_PATH_CMD'):
-                # Backward compatibility: for cluster started before #3326, this
-                # constant does not exist. Since we generate the job script
-                # in backends.cloud_vm_ray_backend with inspect, so the
-                # the lates `run_with_log` will be used, but the `constants` is
-                # not updated. We fallback to `python3` in this case.
-                # TODO(zhwu): remove this after 0.7.0.
-                python_path = 'python3'
-            else:
-                python_path = subprocess.check_output(
-                    constants.SKY_GET_PYTHON_PATH_CMD,
-                    shell=True,
-                    stderr=subprocess.DEVNULL,
-                    encoding='utf-8').strip()
+            python_path = subprocess.check_output(
+                constants.SKY_GET_PYTHON_PATH_CMD,
+                shell=True,
+                stderr=subprocess.DEVNULL,
+                encoding='utf-8').strip()
             daemon_cmd = [
                 python_path,
                 daemon_script,

sky/templates/azure-ray.yml.j2

@@ -34,6 +34,7 @@ provider:
   # instead of the cluster_name. This ensures that ray creates new instances
   # for different cluster_name.
   resource_group: {{resource_group}}
+  use_external_resource_group: {{use_external_resource_group}}
   # Keep (otherwise cannot reuse when re-provisioning).
   # teardown(terminate=True) will override this.
   cache_stopped_nodes: True

sky/utils/schemas.py

@@ -357,19 +357,6 @@ def get_service_schema():
                     'downscale_delay_seconds': {
                         'type': 'number',
                     },
-                    # TODO(MaoZiming): Fields `qps_upper_threshold`,
-                    # `qps_lower_threshold` and `auto_restart` are deprecated.
-                    # Temporarily keep these fields for backward compatibility.
-                    # Remove after 2 minor release, i.e., 0.6.0.
-                    'auto_restart': {
-                        'type': 'boolean',
-                    },
-                    'qps_upper_threshold': {
-                        'type': 'number',
-                    },
-                    'qps_lower_threshold': {
-                        'type': 'number',
-                    },
                 }
             },
             'replicas': {
@@ -595,7 +582,7 @@ _NETWORK_CONFIG_SCHEMA = {
 
 _LABELS_SCHEMA = {
     # Deprecated: 'instance_tags' is replaced by 'labels'. Keeping for backward
-    # compatibility. Will be removed after 0.7.0.
+    # compatibility. Will be removed after 0.8.0.
     'instance_tags': {
         'type': 'object',
         'required': [],
@@ -771,6 +758,9 @@ def get_config_schema():
                 'storage_account': {
                     'type': 'string',
                 },
+                'resource_group_vm': {
+                    'type': 'string',
+                },
             }
         },
         'kubernetes': {