skypilot-nightly 1.0.0.dev20241022__py3-none-any.whl → 1.0.0.dev20241024__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '900819da6707d5ba5e3b9e97dca3cf9e3fda8c48'
+_SKYPILOT_COMMIT_SHA = 'cbf5c0022ad920edb4f41cfad65a2cf4909d5930'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241022'
+__version__ = '1.0.0.dev20241024'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -128,6 +128,7 @@ GCP = clouds.GCP
 Lambda = clouds.Lambda
 SCP = clouds.SCP
 Kubernetes = clouds.Kubernetes
+K8s = Kubernetes
 OCI = clouds.OCI
 Paperspace = clouds.Paperspace
 RunPod = clouds.RunPod
@@ -143,6 +144,7 @@ __all__ = [
     'GCP',
     'IBM',
     'Kubernetes',
+    'K8s',
     'Lambda',
     'OCI',
     'Paperspace',

sky/adaptors/azure.py

@@ -69,6 +69,17 @@ def exceptions():
     return azure_exceptions
 
 
+@functools.lru_cache()
+@common.load_lazy_modules(modules=_LAZY_MODULES)
+def azure_mgmt_models(name: str):
+    if name == 'compute':
+        from azure.mgmt.compute import models
+        return models
+    elif name == 'network':
+        from azure.mgmt.network import models
+        return models
+
+
 # We should keep the order of the decorators having 'lru_cache' followed
 # by 'load_lazy_modules' as we need to make sure a caller can call
 # 'get_client.cache_clear', which is a function provided by 'lru_cache'

sky/cli.py

@@ -339,7 +339,6 @@ def _get_shell_complete_args(complete_fn):
 
 
 _RELOAD_ZSH_CMD = 'source ~/.zshrc'
-_RELOAD_FISH_CMD = 'source ~/.config/fish/config.fish'
 _RELOAD_BASH_CMD = 'source ~/.bashrc'
 
 
@@ -378,7 +377,9 @@ def _install_shell_completion(ctx: click.Context, param: click.Parameter,
         cmd = '_SKY_COMPLETE=fish_source sky > \
                 ~/.config/fish/completions/sky.fish'
 
-        reload_cmd = _RELOAD_FISH_CMD
+        # Fish does not need to be reloaded and will automatically pick up
+        # completions.
+        reload_cmd = None
 
     elif value == 'zsh':
         install_cmd = f'_SKY_COMPLETE=zsh_source sky > \
@@ -398,9 +399,10 @@ def _install_shell_completion(ctx: click.Context, param: click.Parameter,
                        check=True,
                        executable=shutil.which('bash'))
         click.secho(f'Shell completion installed for {value}', fg='green')
-        click.echo(
-            'Completion will take effect once you restart the terminal: ' +
-            click.style(f'{reload_cmd}', bold=True))
+        if reload_cmd is not None:
+            click.echo(
+                'Completion will take effect once you restart the terminal: ' +
+                click.style(f'{reload_cmd}', bold=True))
     except subprocess.CalledProcessError as e:
         click.secho(f'> Installation failed with code {e.returncode}', fg='red')
     ctx.exit()
@@ -431,7 +433,9 @@ def _uninstall_shell_completion(ctx: click.Context, param: click.Parameter,
 
     elif value == 'fish':
         cmd = 'rm -f ~/.config/fish/completions/sky.fish'
-        reload_cmd = _RELOAD_FISH_CMD
+        # Fish does not need to be reloaded and will automatically pick up
+        # completions.
+        reload_cmd = None
 
     elif value == 'zsh':
         cmd = 'sed -i"" -e "/# For SkyPilot shell completion/d" ~/.zshrc && \
@@ -447,8 +451,10 @@ def _uninstall_shell_completion(ctx: click.Context, param: click.Parameter,
     try:
         subprocess.run(cmd, shell=True, check=True)
         click.secho(f'Shell completion uninstalled for {value}', fg='green')
-        click.echo('Changes will take effect once you restart the terminal: ' +
-                   click.style(f'{reload_cmd}', bold=True))
+        if reload_cmd is not None:
+            click.echo(
+                'Changes will take effect once you restart the terminal: ' +
+                click.style(f'{reload_cmd}', bold=True))
     except subprocess.CalledProcessError as e:
         click.secho(f'> Uninstallation failed with code {e.returncode}',
                     fg='red')
@@ -3056,7 +3062,8 @@ def show_gpus(
 
     # This will validate 'cloud' and raise if not found.
     cloud_obj = sky_clouds.CLOUD_REGISTRY.from_str(cloud)
-    service_catalog.validate_region_zone(region, None, clouds=cloud)
+    cloud_name = cloud_obj.canonical_name() if cloud_obj is not None else None
+    service_catalog.validate_region_zone(region, None, clouds=cloud_name)
     show_all = all
     if show_all and accelerator_str is not None:
         raise click.UsageError('--all is only allowed without a GPU name.')
@@ -3142,8 +3149,8 @@ def show_gpus(
         # Optimization - do not poll for Kubernetes API for fetching
         # common GPUs because that will be fetched later for the table after
         # common GPUs.
-        clouds_to_list = cloud
-        if cloud is None:
+        clouds_to_list = cloud_name
+        if cloud_name is None:
             clouds_to_list = [
                 c for c in service_catalog.ALL_CLOUDS if c != 'kubernetes'
             ]
@@ -3153,7 +3160,8 @@ def show_gpus(
             # Collect k8s related messages in k8s_messages and print them at end
             print_section_titles = False
             # If cloud is kubernetes, we want to show real-time capacity
-            if kubernetes_is_enabled and (cloud is None or cloud_is_kubernetes):
+            if kubernetes_is_enabled and (cloud_name is None or
+                                          cloud_is_kubernetes):
                 if region:
                     context = region
                 else:
@@ -3263,8 +3271,8 @@ def show_gpus(
                 name, quantity = accelerator_str, None
 
         print_section_titles = False
-        if (kubernetes_is_enabled and (cloud is None or cloud_is_kubernetes) and
-                not show_all):
+        if (kubernetes_is_enabled and
+            (cloud_name is None or cloud_is_kubernetes) and not show_all):
             # Print section title if not showing all and instead a specific
             # accelerator is requested
             print_section_titles = True
@@ -3336,7 +3344,7 @@ def show_gpus(
         if len(result) == 0:
             quantity_str = (f' with requested quantity {quantity}'
                             if quantity else '')
-            cloud_str = f' on {cloud_obj}.' if cloud else ' in cloud catalogs.'
+            cloud_str = f' on {cloud_obj}.' if cloud_name else ' in cloud catalogs.'
             yield f'Resources \'{name}\'{quantity_str} not found{cloud_str} '
             yield 'To show available accelerators, run: sky show-gpus --all'
             return

sky/clouds/cloud.py

@@ -819,6 +819,10 @@ class Cloud:
 
     # === End of image related methods ===
 
+    @classmethod
+    def canonical_name(cls) -> str:
+        return cls.__name__.lower()
+
     def __repr__(self):
         return self._REPR
 

sky/clouds/cloud_registry.py

@@ -1,7 +1,7 @@
 """Clouds need to be registered in CLOUD_REGISTRY to be discovered"""
 
 import typing
-from typing import Optional, Type
+from typing import Callable, Dict, List, Optional, overload, Type, Union
 
 from sky.utils import ux_utils
 
@@ -12,20 +12,65 @@ if typing.TYPE_CHECKING:
 class _CloudRegistry(dict):
     """Registry of clouds."""
 
+    def __init__(self) -> None:
+        super().__init__()
+        self.aliases: Dict[str, str] = {}
+
     def from_str(self, name: Optional[str]) -> Optional['cloud.Cloud']:
+        """Returns the cloud instance from the canonical name or alias."""
         if name is None:
             return None
-        if name.lower() not in self:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(f'Cloud {name!r} is not a valid cloud among '
-                                 f'{list(self.keys())}')
-        return self.get(name.lower())
 
+        search_name = name.lower()
+
+        if search_name in self:
+            return self[search_name]
+
+        if search_name in self.aliases:
+            return self[self.aliases[search_name]]
+
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(f'Cloud {name!r} is not a valid cloud among '
+                             f'{[*self.keys(), *self.aliases.keys()]}')
+
+    @overload
     def register(self, cloud_cls: Type['cloud.Cloud']) -> Type['cloud.Cloud']:
-        name = cloud_cls.__name__.lower()
-        assert name not in self, f'{name} already registered'
-        self[name] = cloud_cls()
-        return cloud_cls
+        ...
+
+    @overload
+    def register(
+        self,
+        cloud_cls: None = None,
+        aliases: Optional[List[str]] = None,
+    ) -> Callable[[Type['cloud.Cloud']], Type['cloud.Cloud']]:
+        ...
+
+    def register(
+        self,
+        cloud_cls: Optional[Type['cloud.Cloud']] = None,
+        aliases: Optional[List[str]] = None,
+    ) -> Union[Type['cloud.Cloud'], Callable[[Type['cloud.Cloud']],
+                                             Type['cloud.Cloud']]]:
+
+        def _register(cloud_cls: Type['cloud.Cloud']) -> Type['cloud.Cloud']:
+            name = cloud_cls.canonical_name()
+            assert name not in self, f'{name} already registered'
+            self[name] = cloud_cls()
+
+            for alias in aliases or []:
+                alias = alias.lower()
+                assert alias not in self.aliases, (
+                    f'alias {alias} already registered')
+                self.aliases[alias] = name
+
+            return cloud_cls
+
+        if cloud_cls is not None:
+            # invocation without parens (e.g. just `@register`)
+            return _register(cloud_cls)
+
+        # Invocation with parens (e.g. `@register(aliases=['alias'])`)
+        return _register
 
 
 CLOUD_REGISTRY: _CloudRegistry = _CloudRegistry()

sky/clouds/kubernetes.py

@@ -33,7 +33,7 @@ CREDENTIAL_PATH = os.environ.get('KUBECONFIG', DEFAULT_KUBECONFIG_PATH)
 _SKYPILOT_SYSTEM_NAMESPACE = 'skypilot-system'
 
 
-@clouds.CLOUD_REGISTRY.register
+@clouds.CLOUD_REGISTRY.register(aliases=['k8s'])
 class Kubernetes(clouds.Cloud):
     """Kubernetes."""
 

sky/provision/azure/config.py

@@ -46,6 +46,7 @@ def bootstrap_instances(
         region: str, cluster_name_on_cloud: str,
         config: common.ProvisionConfig) -> common.ProvisionConfig:
     """See sky/provision/__init__.py"""
+    # TODO: use new azure sdk instead of ARM deployment.
     del region  # unused
     provider_config = config.provider_config
     subscription_id = provider_config.get('subscription_id')

sky/provision/azure/instance.py

@@ -2,10 +2,8 @@
 import base64
 import copy
 import enum
-import json
 import logging
 from multiprocessing import pool
-import pathlib
 import time
 import typing
 from typing import Any, Callable, Dict, List, Optional, Tuple
@@ -23,7 +21,9 @@ from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     from azure.mgmt import compute as azure_compute
-    from azure.mgmt import resource as azure_resource
+    from azure.mgmt import network as azure_network
+    from azure.mgmt.compute import models as azure_compute_models
+    from azure.mgmt.network import models as azure_network_models
 
 logger = sky_logging.init_logger(__name__)
 
@@ -184,14 +184,150 @@ def _get_head_instance_id(instances: List) -> Optional[str]:
     return head_instance_id
 
 
-def _create_instances(
-        compute_client: 'azure_compute.ComputeManagementClient',
-        resource_client: 'azure_resource.ResourceManagementClient',
-        cluster_name_on_cloud: str, resource_group: str,
-        provider_config: Dict[str, Any], node_config: Dict[str, Any],
-        tags: Dict[str, str], count: int) -> List:
+def _create_network_interface(
+        network_client: 'azure_network.NetworkManagementClient', vm_name: str,
+        provider_config: Dict[str,
+                              Any]) -> 'azure_network_models.NetworkInterface':
+    network = azure.azure_mgmt_models('network')
+    compute = azure.azure_mgmt_models('compute')
+    logger.info(f'Start creating network interface for {vm_name}...')
+    if provider_config.get('use_internal_ips', False):
+        name = f'{vm_name}-nic-private'
+        ip_config = network.IPConfiguration(
+            name=f'ip-config-private-{vm_name}',
+            subnet=compute.SubResource(id=provider_config['subnet']),
+            private_ip_allocation_method=network.IPAllocationMethod.DYNAMIC)
+    else:
+        name = f'{vm_name}-nic-public'
+        public_ip_address = network.PublicIPAddress(
+            location=provider_config['location'],
+            public_ip_allocation_method='Static',
+            public_ip_address_version='IPv4',
+            sku=network.PublicIPAddressSku(name='Basic', tier='Regional'))
+        ip_poller = network_client.public_ip_addresses.begin_create_or_update(
+            resource_group_name=provider_config['resource_group'],
+            public_ip_address_name=f'{vm_name}-ip',
+            parameters=public_ip_address)
+        logger.info(f'Created public IP address {ip_poller.result().name} '
+                    f'with address {ip_poller.result().ip_address}.')
+        ip_config = network.IPConfiguration(
+            name=f'ip-config-public-{vm_name}',
+            subnet=compute.SubResource(id=provider_config['subnet']),
+            private_ip_allocation_method=network.IPAllocationMethod.DYNAMIC,
+            public_ip_address=network.PublicIPAddress(id=ip_poller.result().id))
+
+    ni_poller = network_client.network_interfaces.begin_create_or_update(
+        resource_group_name=provider_config['resource_group'],
+        network_interface_name=name,
+        parameters=network.NetworkInterface(
+            location=provider_config['location'],
+            ip_configurations=[ip_config],
+            network_security_group=network.NetworkSecurityGroup(
+                id=provider_config['nsg'])))
+    logger.info(f'Created network interface {ni_poller.result().name}.')
+    return ni_poller.result()
+
+
+def _create_vm(
+        compute_client: 'azure_compute.ComputeManagementClient', vm_name: str,
+        node_tags: Dict[str, str], provider_config: Dict[str, Any],
+        node_config: Dict[str, Any],
+        network_interface_id: str) -> 'azure_compute_models.VirtualMachine':
+    compute = azure.azure_mgmt_models('compute')
+    logger.info(f'Start creating VM {vm_name}...')
+    hardware_profile = compute.HardwareProfile(
+        vm_size=node_config['azure_arm_parameters']['vmSize'])
+    network_profile = compute.NetworkProfile(network_interfaces=[
+        compute.NetworkInterfaceReference(id=network_interface_id, primary=True)
+    ])
+    public_key = node_config['azure_arm_parameters']['publicKey']
+    username = node_config['azure_arm_parameters']['adminUsername']
+    os_linux_custom_data = base64.b64encode(
+        node_config['azure_arm_parameters']['cloudInitSetupCommands'].encode(
+            'utf-8')).decode('utf-8')
+    os_profile = compute.OSProfile(
+        admin_username=username,
+        computer_name=vm_name,
+        admin_password=public_key,
+        linux_configuration=compute.LinuxConfiguration(
+            disable_password_authentication=True,
+            ssh=compute.SshConfiguration(public_keys=[
+                compute.SshPublicKey(
+                    path=f'/home/{username}/.ssh/authorized_keys',
+                    key_data=public_key)
+            ])),
+        custom_data=os_linux_custom_data)
+    community_image_id = node_config['azure_arm_parameters'].get(
+        'communityGalleryImageId', None)
+    if community_image_id is not None:
+        # Prioritize using community gallery image if specified.
+        image_reference = compute.ImageReference(
+            community_gallery_image_id=community_image_id)
+        logger.info(
+            f'Used community_image_id: {community_image_id} for VM {vm_name}.')
+    else:
+        image_reference = compute.ImageReference(
+            publisher=node_config['azure_arm_parameters']['imagePublisher'],
+            offer=node_config['azure_arm_parameters']['imageOffer'],
+            sku=node_config['azure_arm_parameters']['imageSku'],
+            version=node_config['azure_arm_parameters']['imageVersion'])
+    storage_profile = compute.StorageProfile(
+        image_reference=image_reference,
+        os_disk=compute.OSDisk(
+            create_option=compute.DiskCreateOptionTypes.FROM_IMAGE,
+            managed_disk=compute.ManagedDiskParameters(
+                storage_account_type=node_config['azure_arm_parameters']
+                ['osDiskTier']),
+            disk_size_gb=node_config['azure_arm_parameters']['osDiskSizeGB']))
+    vm_instance = compute.VirtualMachine(
+        location=provider_config['location'],
+        tags=node_tags,
+        hardware_profile=hardware_profile,
+        os_profile=os_profile,
+        storage_profile=storage_profile,
+        network_profile=network_profile,
+        identity=compute.VirtualMachineIdentity(
+            type='UserAssigned',
+            user_assigned_identities={provider_config['msi']: {}}))
+    vm_poller = compute_client.virtual_machines.begin_create_or_update(
+        resource_group_name=provider_config['resource_group'],
+        vm_name=vm_name,
+        parameters=vm_instance,
+    )
+    # poller.result() will block on async operation until it's done.
+    logger.info(f'Created VM {vm_poller.result().name}.')
+    # Configure driver extension for A10 GPUs. A10 GPUs requires a
+    # special type of drivers which is available at Microsoft HPC
+    # extension. Reference:
+    # https://forums.developer.nvidia.com/t/ubuntu-22-04-installation-driver-error-nvidia-a10/285195/2
+    # This can take more than 20mins for setting up the A10 GPUs
+    if node_config.get('need_nvidia_driver_extension', False):
+        ext_poller = compute_client.virtual_machine_extensions.\
+            begin_create_or_update(
+            resource_group_name=provider_config['resource_group'],
+            vm_name=vm_name,
+            vm_extension_name='NvidiaGpuDriverLinux',
+            extension_parameters=compute.VirtualMachineExtension(
+                location=provider_config['location'],
+                publisher='Microsoft.HpcCompute',
+                type_properties_type='NvidiaGpuDriverLinux',
+                type_handler_version='1.9',
+                auto_upgrade_minor_version=True,
+                settings='{}'))
+        logger.info(
+            f'Created VM extension {ext_poller.result().name} for VM {vm_name}.'
+        )
+    return vm_poller.result()
+
+
+def _create_instances(compute_client: 'azure_compute.ComputeManagementClient',
+                      network_client: 'azure_network.NetworkManagementClient',
+                      cluster_name_on_cloud: str, resource_group: str,
+                      provider_config: Dict[str, Any], node_config: Dict[str,
+                                                                         Any],
+                      tags: Dict[str, str], count: int) -> List:
     vm_id = uuid4().hex[:UNIQUE_ID_LEN]
-    tags = {
+    all_tags = {
         constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud,
         constants.TAG_SKYPILOT_CLUSTER_NAME: cluster_name_on_cloud,
         **constants.WORKER_NODE_TAGS,
@@ -199,83 +335,19 @@ def _create_instances(
         **tags,
     }
     node_tags = node_config['tags'].copy()
-    node_tags.update(tags)
-
-    # load the template file
-    current_path = pathlib.Path(__file__).parent
-    template_path = current_path.joinpath('azure-vm-template.json')
-    with open(template_path, 'r', encoding='utf-8') as template_fp:
-        template = json.load(template_fp)
-
-    vm_name = f'{cluster_name_on_cloud}-{vm_id}'
-    use_internal_ips = provider_config.get('use_internal_ips', False)
+    node_tags.update(all_tags)
 
-    template_params = node_config['azure_arm_parameters'].copy()
-    # We don't include 'head' or 'worker' in the VM name as on Azure the VM
-    # name is immutable and we may change the node type for existing VM in the
-    # multi-node cluster, due to manual termination of the head node.
-    template_params['vmName'] = vm_name
-    template_params['provisionPublicIp'] = not use_internal_ips
-    template_params['vmTags'] = node_tags
-    template_params['vmCount'] = count
-    template_params['msi'] = provider_config['msi']
-    template_params['nsg'] = provider_config['nsg']
-    template_params['subnet'] = provider_config['subnet']
-    # In Azure, cloud-init script must be encoded in base64. For more
-    # information, see:
-    # https://learn.microsoft.com/en-us/azure/virtual-machines/custom-data
-    template_params['cloudInitSetupCommands'] = (base64.b64encode(
-        template_params['cloudInitSetupCommands'].encode('utf-8')).decode(
-            'utf-8'))
+    # Create VM instances in parallel.
+    def create_single_instance(vm_i):
+        vm_name = f'{cluster_name_on_cloud}-{vm_id}-{vm_i}'
+        network_interface = _create_network_interface(network_client, vm_name,
+                                                      provider_config)
+        _create_vm(compute_client, vm_name, node_tags, provider_config,
+                   node_config, network_interface.id)
 
-    if node_config.get('need_nvidia_driver_extension', False):
-        # pylint: disable=line-too-long
-        # Configure driver extension for A10 GPUs. A10 GPUs requires a
-        # special type of drivers which is available at Microsoft HPC
-        # extension. Reference: https://forums.developer.nvidia.com/t/ubuntu-22-04-installation-driver-error-nvidia-a10/285195/2
-        for r in template['resources']:
-            if r['type'] == 'Microsoft.Compute/virtualMachines':
-                # Add a nested extension resource for A10 GPUs
-                r['resources'] = [
-                    {
-                        'type': 'extensions',
-                        'apiVersion': '2015-06-15',
-                        'location': '[variables(\'location\')]',
-                        'dependsOn': [
-                            '[concat(\'Microsoft.Compute/virtualMachines/\', parameters(\'vmName\'), copyIndex())]'
-                        ],
-                        'name': 'NvidiaGpuDriverLinux',
-                        'properties': {
-                            'publisher': 'Microsoft.HpcCompute',
-                            'type': 'NvidiaGpuDriverLinux',
-                            'typeHandlerVersion': '1.9',
-                            'autoUpgradeMinorVersion': True,
-                            'settings': {},
-                        },
-                    },
-                ]
-                break
-
-    parameters = {
-        'properties': {
-            'mode': azure.deployment_mode().incremental,
-            'template': template,
-            'parameters': {
-                key: {
-                    'value': value
-                } for key, value in template_params.items()
-            },
-        }
-    }
-
-    create_or_update = _get_azure_sdk_function(
-        client=resource_client.deployments, function_name='create_or_update')
-    create_or_update(
-        resource_group_name=resource_group,
-        deployment_name=vm_name,
-        parameters=parameters,
-    ).wait()
+    subprocess_utils.run_in_parallel(create_single_instance, range(count))
 
+    # Update disk performance tier
     performance_tier = node_config.get('disk_performance_tier', None)
     if performance_tier is not None:
         disks = compute_client.disks.list_by_resource_group(resource_group)
@@ -286,12 +358,14 @@ def _create_instances(
                 f'az disk update -n {name} -g {resource_group} '
                 f'--set tier={performance_tier}')
 
+    # Validation
     filters = {
         constants.TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud,
         _TAG_SKYPILOT_VM_ID: vm_id
     }
     instances = _filter_instances(compute_client, resource_group, filters)
     assert len(instances) == count, (len(instances), count)
+
     return instances
 
 
@@ -303,7 +377,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     resource_group = provider_config['resource_group']
     subscription_id = provider_config['subscription_id']
     compute_client = azure.get_client('compute', subscription_id)
-
+    network_client = azure.get_client('network', subscription_id)
     instances_to_resume = []
     resumed_instance_ids: List[str] = []
     created_instance_ids: List[str] = []
@@ -439,12 +513,11 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     to_start_count -= len(resumed_instance_ids)
 
     if to_start_count > 0:
-        resource_client = azure.get_client('resource', subscription_id)
         logger.debug(f'run_instances: Creating {to_start_count} instances.')
         try:
             created_instances = _create_instances(
                 compute_client=compute_client,
-                resource_client=resource_client,
+                network_client=network_client,
                 cluster_name_on_cloud=cluster_name_on_cloud,
                 resource_group=resource_group,
                 provider_config=provider_config,

sky/templates/azure-ray.yml.j2

@@ -67,6 +67,8 @@ available_node_types:
             imageOffer: {{image_offer}}
             imageSku: "{{image_sku}}"
             imageVersion: {{image_version}}
+            # Community Gallery Image ID 
+            communityGalleryImageId: {{community_gallery_image_id}}
             osDiskSizeGB: {{disk_size}}
             osDiskTier: {{disk_tier}}
             {%- if use_spot %}

sky/utils/command_runner.py

@@ -838,8 +838,9 @@ class KubernetesCommandRunner(CommandRunner):
         # default delimiter for options and arguments.
         # rsync_helper.sh will parse the namespace_context by reverting the
         # encoding and pass it to kubectl exec.
-        encoded_namespace_context = namespace_context.replace(
-            ':', '%3A').replace('/', '%2F').replace('+', '%2B')
+        encoded_namespace_context = (namespace_context.replace(
+            '@', '%40').replace(':', '%3A').replace('/',
+                                                    '%2F').replace('+', '%2B'))
         self._rsync(
             source,
             target,

sky/utils/kubernetes/k8s_gpu_labeler_job.yaml

@@ -14,9 +14,10 @@ spec:
       containers:
       - name: gpu-labeler
         image: us-central1-docker.pkg.dev/skypilot-375900/skypilotk8s/skypilot-gpu:latest # Using this image also serves as a way to "pre-pull" the image onto nodes
-        command:
-          - "python"
-          - "/label_gpus.py"
+        command: ["/bin/bash", "-i", "-c"]
+        args:
+          - |
+            python /label_gpus.py
         env:
         - name: MY_NODE_NAME
           valueFrom:

sky/utils/kubernetes/rsync_helper.sh

@@ -7,7 +7,7 @@ shift
 echo "pod: $pod" >&2
 encoded_namespace_context=$1
 # Revert the encoded namespace+context to the original string.
-namespace_context=$(echo "$encoded_namespace_context" | sed 's|%3A|:|g' | sed 's|%2B|+|g' | sed 's|%2F|/|g')
+namespace_context=$(echo "$encoded_namespace_context" | sed 's|%40|@|g' | sed 's|%3A|:|g' | sed 's|%2B|+|g' | sed 's|%2F|/|g')
 echo "namespace_context: $namespace_context" >&2
 namespace=$(echo $namespace_context | cut -d+ -f1)
 echo "namespace: $namespace" >&2

{skypilot_nightly-1.0.0.dev20241022.dist-info → skypilot_nightly-1.0.0.dev20241024.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241022
+Version: 1.0.0.dev20241024
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -234,7 +234,7 @@ Paste the following into a file `my_task.yaml`:
 
 ```yaml
 resources:
-  accelerators: V100:1  # 1x NVIDIA V100 GPU
+  accelerators: A100:8  # 8x NVIDIA A100 GPU
 
 num_nodes: 1  # Number of VMs to launch
 

{skypilot_nightly-1.0.0.dev20241022.dist-info → skypilot_nightly-1.0.0.dev20241024.dist-info}/RECORD

@@ -1,8 +1,8 @@
-sky/__init__.py,sha256=tGoB9kUj49S8LrP-Rj2tnmsQl6a9rNxpOGrRZDiQqdM,5854
+sky/__init__.py,sha256=6pKM5lRNTckGEImvaBniSM_EuAzDT5xbSHqP8AhenhU,5882
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=pAdCT60OxxiXI9KXDyP2lQ9u9vMc6aMtq5Xi2h_hbdw,20984
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
-sky/cli.py,sha256=XcQeVtH5J7xcARGagYicmTUfd2145jN2nvnZaZXtZlI,209981
+sky/cli.py,sha256=PdqIKhGs6P2lHEShX2-iSwAtbpS5TTFKWNKCRl2i6kw,210345
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
 sky/core.py,sha256=DW9OGE2kS2CmsvQ1grrpRnNFS3woMGWSHu5GE99e-I4,38190
 sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
@@ -17,7 +17,7 @@ sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
 sky/task.py,sha256=KDsTBIxYpkCOPHv3_ei5H3LDMiGHvDeS9_2HeL6yyLA,49766
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=jz3E8YyeGscgLZvFE6A1SkxpH6o_inZ-0NiXdRnxSGA,6863
-sky/adaptors/azure.py,sha256=edWy5YgW6dAUl8pPWvSJ8x5mRgCew1KmDpntTw65vHo,21464
+sky/adaptors/azure.py,sha256=n4PXHGJ8ambcbzYzWifj9BsoShUhg62pF6udmUw45S8,21762
 sky/adaptors/cloudflare.py,sha256=0JA41HeZeYoqcJoEyol01caYgt8UrlpcZJJ6ToU-Fg8,7594
 sky/adaptors/common.py,sha256=VhgAFPx-GmhXoob8B1YvqGC2BXs2HwtqoA8oFMaqkbc,2551
 sky/adaptors/cudo.py,sha256=WGvIQrlzJkGDe02Ve7pygA56tHwUc4kwS3XHW8kMFAA,239
@@ -42,13 +42,13 @@ sky/benchmark/benchmark_utils.py,sha256=eb-i6zYoo-Zkod-T9qtCu1FcYLw--Yyos1SyibUP
 sky/clouds/__init__.py,sha256=WuNIJEnZmBO72tU5awgaaL3rdvFRSkgaYNNeuY68dXo,1356
 sky/clouds/aws.py,sha256=XJVbOSkVVUHp9HbHDp0rFdHX113JHbY-3sgokGdNJVE,49527
 sky/clouds/azure.py,sha256=jTgynKU5tuOyBe97n2I7_k9P0Sw0QFU-6wLDLFwQhfM,28634
-sky/clouds/cloud.py,sha256=PPk-Cbf1YbJT8bswcQLtPBtko02OWrRGJKkLzDpytTI,34858
-sky/clouds/cloud_registry.py,sha256=4yQMv-iBSgyN5aNL4Qxbn0JVE-dkVoEUIgj7S1z9S_Q,955
+sky/clouds/cloud.py,sha256=BBu1G-gkmylffldL50cvJ2DkDJ8vjVPziOPUAsvgJ2o,34948
+sky/clouds/cloud_registry.py,sha256=oLoYFjm_SDTgdHokY7b6A5Utq80HXRQNxV0fLjDdVsQ,2361
 sky/clouds/cudo.py,sha256=H4VyMo5wWGAv2MXZ3xsbWjlZA_cZYnt4ecNlTOOao8Y,13147
 sky/clouds/fluidstack.py,sha256=iOmoOx52yTrHKMzwBDaxFJCfNo79M61d5tj-Np24Lyc,12436
 sky/clouds/gcp.py,sha256=m_dH04HqgU-DdW4R9wrSr66IpPt9JMKHEvHEGGFpeRo,54655
 sky/clouds/ibm.py,sha256=M8QdjeSFlwssfoY2aOodxG4q5R3eT9K-4lTPDHYvEYI,21476
-sky/clouds/kubernetes.py,sha256=aWoXWR-S4puZHzuUHroLKxLdTpkqU7j75dQlXECnsmE,28679
+sky/clouds/kubernetes.py,sha256=j3imm_sbtyyZXvJ6qbqZmXok2C9OQIcGpyulljbTSJ4,28696
 sky/clouds/lambda_cloud.py,sha256=11dKUSunHUgaPZ1t8O85X29_NJ-o26sCt5DjwAPFgl4,12697
 sky/clouds/oci.py,sha256=ecVgcbCVJwDLtaYXs-yGDzwPYRr23KvjnzFOXwaY2O0,26914
 sky/clouds/paperspace.py,sha256=lmUZPYAblaqiBmGQwCunccMiTF_dVA1o3vqY9Q_Nc28,10921
@@ -116,9 +116,8 @@ sky/provision/aws/instance.py,sha256=eCslJ2XfJo_pkQMnKFQqhGnUIRvwKiT12oxBY5-klss
 sky/provision/aws/utils.py,sha256=m49pS-SHGW7Au3bhDeTPsL8N5iRzbwOXzyEWRCc1Vho,3238
 sky/provision/azure/__init__.py,sha256=87cgk1_Ws7n9rqaDDPv-HpfrkVeSQMdFQnhnXwyx9g4,548
 sky/provision/azure/azure-config-template.json,sha256=dwTO-DG70UtBifN59NzsJwPZHZ4uCs7_oLooHgXG_N8,4349
-sky/provision/azure/azure-vm-template.json,sha256=k0A4xclQ-xQbAdcjo516sZqMDsU-n8_jeRuPxtGKBFI,10901
-sky/provision/azure/config.py,sha256=qNUNcX1HboosuZASEVgJ_K-g7CNNWExodWXFIsoq5V8,7288
-sky/provision/azure/instance.py,sha256=3xHRkWcElAgaOuOzcBdwfnLni8zD8KwNgYmj3A7gljA,33244
+sky/provision/azure/config.py,sha256=-VUYxmwLZKHzxNkRmE0q8ZkZ_0iq-gi0q8XPvRD_Y0U,7345
+sky/provision/azure/instance.py,sha256=pDdGScePohZHTRLEtr92k6vO9LIyd-RBCKOkZ5oDV58,37463
 sky/provision/cudo/__init__.py,sha256=KAEl26MVPsk7IoP9Gg-MOJJRIV6-X9B0fbyHdyJWdLo,741
 sky/provision/cudo/config.py,sha256=RYOVkV0MoUqVBJRZiKhBZhjFygeyFs7eUdVMdPg1vds,327
 sky/provision/cudo/cudo_machine_type.py,sha256=_VNXWPELmlFXbtdcnPvkuLuyE9CZ923BUCdiac-ClDY,696
@@ -219,7 +218,7 @@ sky/skylet/ray_patches/resource_demand_scheduler.py.patch,sha256=AVV-Hw-Rxw16aFm
 sky/skylet/ray_patches/updater.py.patch,sha256=ZNMGVYICPBB44jLbEx2KvCgIY7BWYdDv3-2b2HJWmAQ,289
 sky/skylet/ray_patches/worker.py.patch,sha256=_OBhibdr3xOy5Qje6Tt8D1eQVm_msi50TJbCJmOTxVU,565
 sky/templates/aws-ray.yml.j2,sha256=K0rAuyf1XC_GPFp1BR9df42-Be12A6T2UF0BllVSpYg,8005
-sky/templates/azure-ray.yml.j2,sha256=RtYAcAmFQd6TB3j-pbxi7ekjWhznqFhJtzdkqH_nXqM,6135
+sky/templates/azure-ray.yml.j2,sha256=zjASvx-Y0ZNEvCg9hsBhp3mt0Je5fvUv6XYUAYdrNqI,6245
 sky/templates/cudo-ray.yml.j2,sha256=SEHVY57iBauCOE2HYJtYVFEKlriAkdwQu_p86a1n_bA,3548
 sky/templates/fluidstack-ray.yml.j2,sha256=t8TCULgiErCZdtFmBZVsA8ZdcqR7ccwsmQhuDFTBEAU,3541
 sky/templates/gcp-ray.yml.j2,sha256=y95B-Nk6hFxm6vEIaxI1wFzAIcy_GcKC3XMYo9m-ThI,9662
@@ -245,7 +244,7 @@ sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
 sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
-sky/utils/command_runner.py,sha256=TEFJlmIGzlZxZppcBdwDK4AscM0-08L2XRFwQIRK9OA,34784
+sky/utils/command_runner.py,sha256=3CDcqRXEmoe3C-t2P58McgcRg6p9m5haUWYj1rOLuqM,34858
 sky/utils/command_runner.pyi,sha256=mJOzCgcYZAfHwnY_6Wf1YwlTEJGb9ihzc2f0rE0Kw98,7751
 sky/utils/common_utils.py,sha256=MwFhIcvCEMBo7kbENUjN3qRNO5SoMV0fzAORc65c5x0,24525
 sky/utils/controller_utils.py,sha256=V05hiLJIjqqXssYzs_Gchk4-tijgpMgLJsRW8ymhS-E,40625
@@ -270,13 +269,13 @@ sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=vGj0mD0tejHDRy8ulwKOvOF2mfL
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=AcYhuuG5jXWGHUmyRuH-oKy5qcn92gXhu6bXOt6eD6g,9274
 sky/utils/kubernetes/gpu_labeler.py,sha256=MEUv0U4ACDcNwtFVltlv017XJMjxx1Bndf6fL0i6eqg,6960
-sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTogMEFxotE35IosIpRg,1186
+sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
-sky/utils/kubernetes/rsync_helper.sh,sha256=aRMa_0JRHtXFOPtEg4rFAwR1t57wvvAoGZhn3H3BtGk,1059
+sky/utils/kubernetes/rsync_helper.sh,sha256=hyYDaYSNxYaNvzUQBzC8AidB7nDeojizjkzc_CTxycY,1077
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241022.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241022.dist-info/METADATA,sha256=nksoco77N9enU2hNApZ3YwtYVesLMbeLZYPDOXc09qg,19540
-skypilot_nightly-1.0.0.dev20241022.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-skypilot_nightly-1.0.0.dev20241022.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241022.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241022.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241024.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241024.dist-info/METADATA,sha256=R0zBIXcaURnFidv6hVzwuO_p5xUp-L8ZB5_l9-NG0yc,19540
+skypilot_nightly-1.0.0.dev20241024.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+skypilot_nightly-1.0.0.dev20241024.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241024.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241024.dist-info/RECORD,,

sky/provision/azure/azure-vm-template.json

@@ -1,301 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
-    "contentVersion": "1.0.0.0",
-    "parameters": {
-        "vmName": {
-            "type": "string",
-            "metadata": {
-                "description": "The name of you Virtual Machine."
-            }
-        },
-        "adminUsername": {
-            "type": "string",
-            "metadata": {
-                "description": "Username for the Virtual Machine."
-            }
-        },
-        "publicKey": {
-            "type": "securestring",
-            "metadata": {
-                "description": "SSH Key for the Virtual Machine"
-            }
-        },
-        "imagePublisher": {
-            "type": "string",
-            "metadata": {
-                "description": "The publisher of the VM image"
-            }
-        },
-        "imageOffer": {
-            "type": "string",
-            "metadata": {
-                "description": "The offer of the VM image"
-            }
-        },
-        "imageSku": {
-            "type": "string",
-            "metadata": {
-                "description": "The sku of the VM image"
-            }
-        },
-        "imageVersion": {
-            "type": "string",
-            "metadata": {
-                "description": "The version of the VM image"
-            }
-        },
-        "vmSize": {
-            "type": "string",
-            "metadata": {
-                "description": "The size of the VM"
-            }
-        },
-        "vmTags": { 
-            "type": "object",
-            "metadata": {
-                "description": "Tags for the VM"
-            }
-        },
-        "vmCount": {
-            "type": "int",
-            "metadata": {
-                "description": "Number of VMs to deploy"
-            }
-        },
-        "provisionPublicIp": {
-            "type": "bool",
-            "defaultValue": true,
-            "metadata": {
-                "description": "If true creates a public ip"
-            }
-        },
-        "priority": {
-            "type": "string",
-            "defaultValue": "Regular",
-            "metadata": {
-                "description": "Specifies the priority for the virtual machine."
-            }
-        },
-        "billingProfile": {
-            "type": "object",
-            "defaultValue": {},
-            "metadata": {
-                "description": "Specifies the maximum price to pay for Azure Spot VM."
-            }
-        },
-        "osDiskSizeGB": {
-            "type": "int",
-            "metadata": {
-                "description": "OS disk size in GBs."
-            }
-        },
-        "msi": {
-            "type": "string",
-            "metadata": {
-                "description": "Managed service identity resource id."
-            }
-        },
-        "nsg": {
-            "type": "string",
-            "metadata": {
-                "description": "Network security group resource id."
-            }
-        },
-        "subnet": {
-            "type": "string",
-            "metadata": {
-                "descriptions": "Subnet resource id."
-            }
-        },
-        "osDiskTier": {
-            "type": "string",
-            "allowedValues": [
-                "Premium_LRS",
-                "StandardSSD_LRS",
-                "Standard_LRS"
-            ],
-            "metadata": {
-                "description": "OS disk tier."
-            }
-        },
-        "cloudInitSetupCommands": {
-            "type": "string",
-            "metadata": {
-                "description": "Base64 encoded cloud-init setup commands."
-            }
-        }
-    },
-    "variables": {
-        "location": "[resourceGroup().location]",
-        "networkInterfaceNamePrivate": "[concat(parameters('vmName'), '-nic')]",
-        "networkInterfaceNamePublic": "[concat(parameters('vmName'), '-nic-public')]",
-        "networkInterfaceName": "[if(parameters('provisionPublicIp'), variables('networkInterfaceNamePublic'), variables('networkInterfaceNamePrivate'))]",
-        "networkIpConfig": "[guid(resourceGroup().id, parameters('vmName'))]",
-        "publicIpAddressName": "[concat(parameters('vmName'), '-ip')]"
-    },
-    "resources": [
-        {
-            "type": "Microsoft.Network/networkInterfaces",
-            "apiVersion": "2020-06-01",
-            "name": "[concat(variables('networkInterfaceNamePublic'), copyIndex())]",
-            "location": "[variables('location')]",
-            "dependsOn": [
-                "[resourceId('Microsoft.Network/publicIpAddresses/', concat(variables('publicIpAddressName'), copyIndex()))]"
-            ],
-            "copy": {
-                "name": "NICPublicCopy",
-                "count": "[parameters('vmCount')]"
-            },
-            "properties": {
-                "ipConfigurations": [
-                    {
-                        "name": "[variables('networkIpConfig')]",
-                        "properties": {
-                            "subnet": {
-                                "id": "[parameters('subnet')]"
-                            },
-                            "privateIPAllocationMethod": "Dynamic",
-                            "publicIpAddress": {
-                                "id": "[resourceId('Microsoft.Network/publicIPAddresses', concat(variables('publicIPAddressName'), copyIndex()))]"
-                            }
-                        }
-                    }
-                ],
-                "networkSecurityGroup": {
-                    "id": "[parameters('nsg')]"
-                }
-            },
-            "condition": "[parameters('provisionPublicIp')]"
-        },
-        {
-            "type": "Microsoft.Network/networkInterfaces",
-            "apiVersion": "2020-06-01",
-            "name": "[concat(variables('networkInterfaceNamePrivate'), copyIndex())]",
-            "location": "[variables('location')]",
-            "copy": {
-                "name": "NICPrivateCopy",
-                "count": "[parameters('vmCount')]"
-            },
-            "properties": {
-                "ipConfigurations": [
-                    {
-                        "name": "[variables('networkIpConfig')]",
-                        "properties": {
-                            "subnet": {
-                                "id": "[parameters('subnet')]"
-                            },
-                            "privateIPAllocationMethod": "Dynamic"
-                        }
-                    }
-                ],
-                "networkSecurityGroup": {
-                    "id": "[parameters('nsg')]"
-                }
-            },
-            "condition": "[not(parameters('provisionPublicIp'))]"
-        },
-        {
-            "type": "Microsoft.Network/publicIpAddresses",
-            "apiVersion": "2019-02-01",
-            "name": "[concat(variables('publicIpAddressName'), copyIndex())]",
-            "location": "[variables('location')]",
-            "properties": {
-                "publicIpAllocationMethod": "Static",
-                "publicIPAddressVersion": "IPv4"
-            },
-            "copy": {
-                "name": "PublicIpCopy",
-                "count": "[parameters('vmCount')]"
-            },
-            "sku": {
-                "name": "Basic",
-                "tier": "Regional"
-            },
-            "condition": "[parameters('provisionPublicIp')]"
-        },
-        {
-            "type": "Microsoft.Compute/virtualMachines",
-            "apiVersion": "2019-03-01",
-            "name": "[concat(parameters('vmName'), copyIndex())]",
-            "location": "[variables('location')]",
-            "dependsOn": [
-                "[resourceId('Microsoft.Network/networkInterfaces/', concat(variables('networkInterfaceName'), copyIndex()))]"
-            ],
-            "copy": {
-                "name": "VmCopy",
-                "count": "[parameters('vmCount')]"
-            },
-            "tags": "[parameters('vmTags')]",
-            "properties": {
-                "hardwareProfile": {
-                    "vmSize": "[parameters('vmSize')]"
-                },
-                "storageProfile": {
-                    "osDisk": {
-                        "createOption": "fromImage",
-                        "managedDisk": {
-                            "storageAccountType": "[parameters('osDiskTier')]"
-                        },
-                        "diskSizeGB": "[parameters('osDiskSizeGB')]"
-                    },
-                    "imageReference": {
-                        "publisher": "[parameters('imagePublisher')]",
-                        "offer": "[parameters('imageOffer')]",
-                        "sku": "[parameters('imageSku')]",
-                        "version": "[parameters('imageVersion')]"
-                    }
-                },
-                "networkProfile": {
-                    "networkInterfaces": [
-                        {
-                            "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(variables('networkInterfaceName'), copyIndex()))]"
-                        }
-                    ]
-                },
-                "osProfile": {
-                    "computerName": "[concat(parameters('vmName'), copyIndex())]",
-                    "adminUsername": "[parameters('adminUsername')]",
-                    "adminPassword": "[parameters('publicKey')]",
-                    "linuxConfiguration": {
-                        "disablePasswordAuthentication": true,
-                        "ssh": {
-                            "publicKeys": [
-                                {
-                                    "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
-                                    "keyData": "[parameters('publicKey')]"
-                                }
-                            ]
-                        }
-                    },
-                    "customData": "[parameters('cloudInitSetupCommands')]"
-                },
-                "priority": "[parameters('priority')]",
-                "billingProfile": "[parameters('billingProfile')]"
-            },
-            "identity": {
-                "type": "UserAssigned",
-                "userAssignedIdentities": {
-                    "[parameters('msi')]": {
-                    }
-                }
-            }
-        }
-    ],
-    "outputs": {
-        "publicIp": {
-            "type": "array",
-            "copy": {
-                "count": "[parameters('vmCount')]",
-                "input": "[reference(concat(variables('publicIpAddressName'), copyIndex())).ipAddress]"
-            },
-            "condition": "[parameters('provisionPublicIp')]"
-        },
-        "privateIp": {
-            "type": "array",
-            "copy": {
-                "count": "[parameters('vmCount')]",
-                "input": "[reference(concat(variables('networkInterfaceName'), copyIndex())).ipConfigurations[0].properties.privateIPAddress]"
-            }
-        }
-    }
-}