skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/common.py

@@ -6,6 +6,7 @@ import os
 from typing import Any, Dict, List, Optional, Tuple
 
 from sky import sky_logging
+from sky.utils import env_options
 from sky.utils import resources_utils
 
 # NOTE: we can use pydantic instead of dataclasses or namedtuples, because
@@ -96,6 +97,8 @@ class InstanceInfo:
     external_ip: Optional[str]
     tags: Dict[str, str]
     ssh_port: int = 22
+    # The internal service address of the instance on Kubernetes.
+    internal_svc: Optional[str] = None
 
     def get_feasible_ip(self) -> str:
         """Get the most feasible IPs of the instance. This function returns
@@ -238,12 +241,21 @@ class Endpoint:
 
 @dataclasses.dataclass
 class SocketEndpoint(Endpoint):
-    """Socket endpoint accesible via a host and a port."""
+    """Socket endpoint accessible via a host and a port."""
     port: Optional[int]
     host: str = ''
 
     def url(self, override_ip: Optional[str] = None) -> str:
         host = override_ip if override_ip else self.host
+        if env_options.Options.RUNNING_IN_BUILDKITE.get(
+        ) and 'localhost' in host:
+            # In Buildkite CI, we run a kind (Kubernetes in Docker) cluster.
+            # The controller pod runs inside this kind cluster, which itself
+            # runs in a container. When the pod tries to access 'localhost',
+            # it can't reach the host machine's localhost. Using
+            # 'host.docker.internal' allows the pod to properly communicate
+            # with services running on the host machine's localhost.
+            host = 'host.docker.internal'
         return f'{host}{":" + str(self.port) if self.port else ""}'
 
 

sky/provision/cudo/cudo_machine_type.py

@@ -1,7 +1,7 @@
 """Cudo Compute VM spec helper for SkyPilot."""
 import csv
 
-from sky.clouds.service_catalog.common import get_catalog_path
+from sky.catalog.common import get_catalog_path
 
 VMS_CSV = 'cudo/vms.csv'
 

sky/provision/cudo/cudo_utils.py

@@ -1,22 +1,28 @@
 """Cudo catalog helper."""
 
 cudo_gpu_model = {
-    'NVIDIA V100': 'V100',
-    'NVIDIA A40': 'A40',
-    'RTX 3080': 'RTX3080',
-    'RTX A4000': 'RTXA4000',
-    'RTX A4500': 'RTXA4500',
+    'H100 NVL': 'H100',
+    'H100 SXM': 'H100-SXM',
+    'L40S (compute mode)': 'L40S',
+    'L40S (graphics mode)': 'L40S',
+    'A40 (compute mode)': 'A40',
+    'A40 (graphics mode)': 'A40',
     'RTX A5000': 'RTXA5000',
     'RTX A6000': 'RTXA6000',
+    'A100 80GB PCIe': 'A100',
+    'A800 PCIe': 'A800',
+    'V100': 'V100',
 }
 
 cudo_gpu_mem = {
-    'RTX3080': 12,
+    'H100': 94,
+    'H100-SXM': 80,
+    'L40S': 48,
     'A40': 48,
-    'RTXA4000': 16,
-    'RTXA4500': 20,
     'RTXA5000': 24,
     'RTXA6000': 48,
+    'A100': 80,
+    'A800': 80,
     'V100': 16,
 }
 

sky/provision/cudo/cudo_wrapper.py

@@ -4,7 +4,7 @@ from typing import Dict
 
 from sky import sky_logging
 from sky.adaptors import cudo
-import sky.provision.cudo.cudo_utils as utils
+from sky.provision.cudo import cudo_utils as utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -28,12 +28,10 @@ def launch(name: str, data_center_id: str, ssh_key: str, machine_type: str,
                                  size_gib=disk_size),
         metadata=tags)
 
-    try:
-        api = cudo.cudo.cudo_api.virtual_machines()
-        vm = api.create_vm(cudo.cudo.cudo_api.project_id_throwable(), request)
-        return vm.to_dict()['id']
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    api = cudo.cudo.cudo_api.virtual_machines()
+    vm = api.create_vm(cudo.cudo.cudo_api.project_id_throwable(), request)
+
+    return vm.to_dict()['id']
 
 
 def remove(instance_id: str):
@@ -54,11 +52,8 @@ def remove(instance_id: str):
     state = 'unknown'
     project_id = cudo.cudo.cudo_api.project_id_throwable()
     while retry_count < max_retries:
-        try:
-            vm = api.get_vm(project_id, instance_id)
-            state = vm.to_dict()['vm']['short_state']
-        except cudo.cudo.rest.ApiException as e:
-            raise e
+        vm = api.get_vm(project_id, instance_id)
+        state = vm.to_dict()['vm']['short_state']
 
         if state in terminate_ok:
             break
@@ -69,76 +64,82 @@ def remove(instance_id: str):
             'Timeout error, could not terminate due to VM state: {}'.format(
                 state))
 
-    try:
-        api.terminate_vm(project_id, instance_id)
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    api.terminate_vm(project_id, instance_id)
 
 
 def set_tags(instance_id: str, tags: Dict):
     """Sets the tags for the given instance."""
-    try:
-        api = cudo.cudo.cudo_api.virtual_machines()
-        api.update_vm_metadata(
-            cudo.cudo.cudo_api.project_id(), instance_id,
-            cudo.cudo.UpdateVMMetadataBody(
-                metadata=tags,
-                merge=True))  # TODO (skypilot team) merge or overwrite?
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    api = cudo.cudo.cudo_api.virtual_machines()
+    api.update_vm_metadata(
+        cudo.cudo.cudo_api.project_id(), instance_id,
+        cudo.cudo.UpdateVMMetadataBody(
+            metadata=tags,
+            merge=True))  # TODO (skypilot team) merge or overwrite?
 
 
 def get_instance(vm_id):
-    try:
-        api = cudo.cudo.cudo_api.virtual_machines()
-        vm = api.get_vm(cudo.cudo.cudo_api.project_id_throwable(), vm_id)
-        vm_dict = vm.to_dict()
-        return vm_dict
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    api = cudo.cudo.cudo_api.virtual_machines()
+    vm = api.get_vm(cudo.cudo.cudo_api.project_id_throwable(), vm_id)
+    vm_dict = vm.to_dict()
+    return vm_dict
 
 
 def list_instances():
-    try:
-        api = cudo.cudo.cudo_api.virtual_machines()
-        vms = api.list_vms(cudo.cudo.cudo_api.project_id_throwable())
-        instances = {}
-        for vm in vms.to_dict()['vms']:
-            ex_ip = vm['external_ip_address']
-            in_ip = vm['internal_ip_address']
-            if not in_ip:
-                in_ip = ex_ip
-            instance = {
-                # active_state, init_state, lcm_state, short_state
-                'status': vm['short_state'],
-                'tags': vm['metadata'],
-                'name': vm['id'],
-                'ip': ex_ip,
-                'external_ip': ex_ip,
-                'internal_ip': in_ip
-            }
-            instances[vm['id']] = instance
-        return instances
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    api = cudo.cudo.cudo_api.virtual_machines()
+    vms = api.list_vms(cudo.cudo.cudo_api.project_id_throwable())
+    instances = {}
+    for vm in vms.to_dict()['vms']:
+        ex_ip = vm['external_ip_address']
+        in_ip = vm['internal_ip_address']
+        if not in_ip:
+            in_ip = ex_ip
+        instance = {
+            # active_state, init_state, lcm_state, short_state
+            'status': vm['short_state'],
+            'tags': vm['metadata'],
+            'name': vm['id'],
+            'ip': ex_ip,
+            'external_ip': ex_ip,
+            'internal_ip': in_ip
+        }
+        instances[vm['id']] = instance
+    return instances
 
 
 def vm_available(to_start_count, gpu_count, gpu_model, data_center_id, mem,
                  cpus):
-    try:
-        gpu_model = utils.skypilot_gpu_to_cudo_gpu(gpu_model)
-        api = cudo.cudo.cudo_api.virtual_machines()
-        types = api.list_vm_machine_types(mem,
-                                          cpus,
-                                          gpu=gpu_count,
-                                          gpu_model=gpu_model,
-                                          data_center_id=data_center_id)
-        types_dict = types.to_dict()
-        hc = types_dict['host_configs']
-        total_count = sum(item['count_vm_available'] for item in hc)
-        if total_count < to_start_count:
-            raise Exception(
-                'Too many VMs requested, try another gpu type or region')
-        return total_count
-    except cudo.cudo.rest.ApiException as e:
-        raise e
+    gpu_model = utils.skypilot_gpu_to_cudo_gpu(gpu_model)
+    api = cudo.cudo.cudo_api.virtual_machines()
+    types = api.list_vm_machine_types2()
+    types_dict = types.to_dict()
+    machine_types = types_dict['machine_types']
+
+    # Filter machine types based on requirements
+    matching_types = []
+    for machine_type in machine_types:
+        # Check if this machine type matches our requirements
+        if (machine_type['data_center_id'] == data_center_id and
+                machine_type['gpu_model'] == gpu_model and
+                machine_type['min_vcpu'] <= cpus <= machine_type.get(
+                    'max_vcpu_free', float('inf')) and
+                machine_type['min_memory_gib'] <= mem <= machine_type.get(
+                    'max_memory_gib_free', float('inf'))):
+
+            # Calculate available VMs based on resource constraints
+            max_vms_by_vcpu = machine_type[
+                'total_vcpu_free'] // cpus if cpus > 0 else float('inf')
+            max_vms_by_memory = machine_type[
+                'total_memory_gib_free'] // mem if mem > 0 else float('inf')
+            max_vms_by_gpu = machine_type[
+                'total_gpu_free'] // gpu_count if gpu_count > 0 else float(
+                    'inf')
+
+            available_vms = min(max_vms_by_vcpu, max_vms_by_memory,
+                                max_vms_by_gpu)
+            matching_types.append(available_vms)
+
+    total_count = sum(matching_types)
+    if total_count < to_start_count:
+        raise Exception(
+            'Too many VMs requested, try another gpu type or region')
+    return total_count

sky/provision/cudo/instance.py

@@ -1,7 +1,7 @@
 """Cudo Compute instance provisioning."""
 
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from sky import sky_logging
 from sky.provision import common
@@ -40,10 +40,10 @@ def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['pend', 'init', 'prol', 'boot']
 
     while True:
@@ -191,11 +191,14 @@ def get_cluster_info(
 
 
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     instances = _filter_instances(cluster_name_on_cloud, None)
 
@@ -210,12 +213,13 @@ def query_instances(
         'done': status_lib.ClusterStatus.STOPPED,
         'poff': status_lib.ClusterStatus.STOPPED,
     }
-    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst_id, inst in instances.items():
         status = status_map[inst['status']]
         if non_terminated_only and status is None:
             continue
-        statuses[inst_id] = status
+        statuses[inst_id] = (status, None)
     return statuses
 
 

sky/provision/do/instance.py

@@ -1,7 +1,7 @@
 """DigitalOcean instance provisioning."""
 
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 import uuid
 
 from sky import sky_logging
@@ -26,10 +26,10 @@ def _get_head_instance(
     return None
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['new']
     newly_started_instances = utils.filter_instances(cluster_name_on_cloud,
                                                      pending_status + ['off'])
@@ -242,11 +242,14 @@ def get_cluster_info(
 
 
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     # terminated instances are not retrieved by the
     # API making `non_terminated_only` argument moot.
     del non_terminated_only
@@ -260,10 +263,11 @@ def query_instances(
         'active': status_lib.ClusterStatus.UP,
         'off': status_lib.ClusterStatus.STOPPED,
     }
-    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for instance_meta in instances.values():
         status = status_map[instance_meta['status']]
-        statuses[instance_meta['name']] = status
+        statuses[instance_meta['name']] = (status, None)
     return statuses
 
 

sky/provision/do/utils.py

@@ -17,6 +17,7 @@ from sky.provision import constants as provision_constants
 from sky.provision.do import constants
 from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -30,7 +31,7 @@ POSSIBLE_CREDENTIALS_PATHS = [
 INITIAL_BACKOFF_SECONDS = 10
 MAX_BACKOFF_FACTOR = 10
 MAX_ATTEMPTS = 6
-SSH_KEY_NAME_ON_DO = f'sky-key-{common_utils.get_user_hash()}'
+SSH_KEY_NAME_ON_DO_PREFIX = 'sky-key-'
 
 _client = None
 _ssh_key_id = None
@@ -61,7 +62,7 @@ def _init_client():
     if get_credentials_path() is None:
         raise DigitalOceanError(
             'No credentials found, please run `doctl auth init`')
-    credentials = common_utils.read_yaml(get_credentials_path())
+    credentials = yaml_utils.read_yaml(get_credentials_path())
     default_token = credentials.get('access-token', None)
     if default_token is not None:
         try:
@@ -125,7 +126,7 @@ def ssh_key_id(public_key: str):
 
         request = {
             'public_key': public_key,
-            'name': SSH_KEY_NAME_ON_DO,
+            'name': SSH_KEY_NAME_ON_DO_PREFIX + common_utils.get_user_hash(),
         }
         _ssh_key_id = client().ssh_keys.create(body=request)['ssh_key']
     return _ssh_key_id

sky/provision/docker_utils.py

@@ -3,7 +3,7 @@
 import dataclasses
 import shlex
 import time
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from sky import sky_logging
 from sky.skylet import constants
@@ -15,10 +15,14 @@ logger = sky_logging.init_logger(__name__)
 # Configure environment variables. A docker image can have environment variables
 # set in the Dockerfile with `ENV``. We need to export these variables to the
 # shell environment, so that our ssh session can access them.
+# Filter out RAY_RUNTIME_ENV_HOOK to prevent Ray version conflicts.
+# Docker images with Ray 2.48.0+ set this for UV package manager support,
+# but it causes FAILED_DRIVER errors with SkyPilot's Ray 2.9.3.
+# See: https://github.com/skypilot-org/skypilot/pull/7181
 SETUP_ENV_VARS_CMD = (
     'prefix_cmd() '
     '{ if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; } && '
-    'export -p > ~/container_env_var.sh && '
+    'export -p | grep -v RAY_RUNTIME_ENV_HOOK > ~/container_env_var.sh && '
     '$(prefix_cmd) '
     'mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh;')
 
@@ -32,6 +36,30 @@ DOCKER_SOCKET_NOT_READY_STR = ('Is the docker daemon running?')
 
 _DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS = 30
 
+# Install AWS CLI v2 (not v1 from pip) as it's required for ECR authentication
+# AWS CLI v2 is installed as a standalone binary, not a Python package. See:
+# https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
+INSTALL_AWS_CLI_CMD = (
+    'which aws || ((command -v unzip >/dev/null 2>&1 || '
+    '(sudo apt-get update && sudo apt-get install -y unzip)) && '
+    'curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" '
+    '-o "/tmp/awscliv2.zip" && '
+    'unzip -q /tmp/awscliv2.zip -d /tmp && sudo /tmp/aws/install '
+    '&& rm -rf /tmp/awscliv2.zip /tmp/aws)')
+
+
+def _extract_region_from_ecr_server(server: str) -> str:
+    """Extract AWS region from ECR server URL.
+
+    ECR server format: <account-id>.dkr.ecr.<region>.amazonaws.com
+    Returns the region part from the URL.
+    """
+    # Split: ['<account-id>', 'dkr', 'ecr', '<region>', 'amazonaws', 'com']
+    parts = server.split('.')
+    if len(parts) >= 6 and parts[1] == 'dkr' and parts[2] == 'ecr':
+        return parts[3]
+    raise ValueError(f'Invalid ECR server format: {server}')
+
 
 @dataclasses.dataclass
 class DockerLoginConfig:
@@ -83,6 +111,21 @@ def check_docker_image(cname, docker_cmd):
     return _check_helper(cname, '.Config.Image', docker_cmd)
 
 
+def maybe_remove_container_cmds(container_name, docker_cmd):
+    """Remove the container if it exists. If not, it will be a no-op.
+    """
+    docker_rm = [
+        docker_cmd,
+        'rm',
+        '-f',
+        container_name,
+        '2>/dev/null',
+        '||',
+        'true',
+    ]
+    return ' '.join(docker_rm)
+
+
 def docker_start_cmds(
     image,
     container_name,
@@ -149,12 +192,16 @@ class DockerInitializer:
         self.docker_cmd = 'podman' if use_podman else 'docker'
         self.log_path = log_path
 
-    def _run(self,
-             cmd,
-             run_env='host',
-             wait_for_docker_daemon: bool = False,
-             separate_stderr: bool = False,
-             log_err_when_fail: bool = True) -> str:
+    def _run(
+        self,
+        cmd,
+        run_env='host',
+        wait_for_docker_daemon: bool = False,
+        separate_stderr: bool = False,
+        log_err_when_fail: bool = True,
+        flock_name: Optional[str] = None,
+        flock_args: Optional[str] = None,
+    ) -> str:
 
         if run_env == 'docker':
             cmd = self._docker_expand_user(cmd, any_char=True)
@@ -163,8 +210,13 @@ class DockerInitializer:
             # an error: `the input device is not a TTY`, and it works without
             # `-it` flag.
             # TODO(zhwu): ray use the `-it` flag, we need to check why.
-            cmd = (f'{self.docker_cmd} exec {self.container_name} /bin/bash -c'
-                   f' {shlex.quote(cmd)} ')
+            cmd = (f'{self.docker_cmd} exec -u 0 {self.container_name}'
+                   f' /bin/bash -c {shlex.quote(cmd)} ')
+
+        if flock_name is not None:
+            flock_args = flock_args or ''
+            cmd = (f'flock {flock_args} /tmp/{flock_name} '
+                   f'-c {shlex.quote(cmd)}')
 
         logger.debug(f'+ {cmd}')
         start = time.time()
@@ -216,14 +268,17 @@ class DockerInitializer:
         if self._check_container_exited():
             self.initialized = True
             self._run(f'{self.docker_cmd} start {self.container_name}')
-            self._run('sudo service ssh start', run_env='docker')
+            self._run('sudo service ssh start',
+                      run_env='docker',
+                      flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                      flock_args='-s -w 1')
             return self._run('whoami', run_env='docker')
 
         # SkyPilot: Docker login if user specified a private docker registry.
         if 'docker_login_config' in self.docker_config:
-            # TODO(tian): Maybe support a command to get the login password?
             docker_login_config = DockerLoginConfig(
                 **self.docker_config['docker_login_config'])
+
             if docker_login_config.password:
                 # Password is allowed to be empty, in that case, we will not run
                 # the login command, and assume that the image pulling is
@@ -234,6 +289,25 @@ class DockerInitializer:
                     f'--password {shlex.quote(docker_login_config.password)} '
                     f'{shlex.quote(docker_login_config.server)}',
                     wait_for_docker_daemon=True)
+            elif (docker_login_config.server.endswith('.amazonaws.com') and
+                  '.dkr.ecr.' in docker_login_config.server):
+                # AWS ECR: Use aws ecr get-login-password for authentication
+                # ECR format: <account-id>.dkr.ecr.<region>.amazonaws.com
+                # This command uses the IAM credentials from the EC2 instance
+                # Ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html # pylint: disable=line-too-long
+                region = _extract_region_from_ecr_server(
+                    docker_login_config.server)
+
+                # AWS CLI is not pre-installed on AWS instances, unlike gcloud
+                # on GCP instances, so we need to install it first
+                self._run(INSTALL_AWS_CLI_CMD, wait_for_docker_daemon=False)
+
+                self._run(
+                    f'aws ecr get-login-password --region {region} | '
+                    f'{self.docker_cmd} login --username AWS '
+                    f'--password-stdin '
+                    f'{shlex.quote(docker_login_config.server)}',
+                    wait_for_docker_daemon=True)
             elif docker_login_config.server.endswith('-docker.pkg.dev'):
                 # Docker image server is on GCR, we need to do additional setup
                 # to pull the image.
@@ -285,6 +359,10 @@ class DockerInitializer:
                 'sudo mv /tmp/daemon.json /etc/docker/daemon.json;'
                 'sudo systemctl restart docker; } || true')
             user_docker_run_options = self.docker_config.get('run_options', [])
+            remove_container_cmd = maybe_remove_container_cmds(
+                self.container_name,
+                self.docker_cmd,
+            )
             start_command = docker_start_cmds(
                 specific_image,
                 self.container_name,
@@ -292,7 +370,9 @@ class DockerInitializer:
                     self._auto_configure_shm(user_docker_run_options)),
                 self.docker_cmd,
             )
-            self._run(start_command)
+            self._run(f'{remove_container_cmd} && {start_command}',
+                      flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                      flock_args='-x -w 10')
 
         # SkyPilot: Setup Commands.
         # TODO(zhwu): the following setups should be aligned with the kubernetes
@@ -310,14 +390,18 @@ class DockerInitializer:
             'echo "export DEBIAN_FRONTEND=noninteractive" >> ~/.bashrc;',
             run_env='docker')
         # Install dependencies.
-        self._run(
-            'sudo apt-get update; '
+        cmd = (
+            'bash -lc \''
+            'exec 200>/var/tmp/sky_apt.lock; '
+            'flock -x -w 120 200 || exit 1; '
+            'export DEBIAN_FRONTEND=noninteractive; '
+            'apt-get -yq update && '
             # Our mount script will install gcsfuse without fuse package.
             # We need to install fuse package first to enable storage mount.
             # The dpkg option is to suppress the prompt for fuse installation.
-            'sudo apt-get -o DPkg::Options::="--force-confnew" install -y '
-            'rsync curl wget patch openssh-server python3-pip fuse;',
-            run_env='docker')
+            'apt-get -o DPkg::Options::=--force-confnew install -y '
+            'rsync curl wget patch openssh-server python3-pip fuse\'')
+        self._run(cmd, run_env='docker')
 
         # Copy local authorized_keys to docker container.
         # Stop and disable jupyter service. This is to avoid port conflict on
@@ -343,13 +427,16 @@ class DockerInitializer:
         # `mesg: ttyname failed: inappropriate ioctl for device`.
         # see https://www.educative.io/answers/error-mesg-ttyname-failed-inappropriate-ioctl-for-device  # pylint: disable=line-too-long
         port = constants.DEFAULT_DOCKER_PORT
+        # In case the port is already configured in the sshd_config file
+        # in some images, we delete it first and then append the new one.
         # pylint: disable=anomalous-backslash-in-string
         self._run(
-            f'sudo sed -i "s/#Port 22/Port {port}/" /etc/ssh/sshd_config;'
+            'sudo sed -i "/^Port .*/d" /etc/ssh/sshd_config;'
+            f'echo "Port {port}" | sudo tee -a /etc/ssh/sshd_config > /dev/null;'
             'mkdir -p ~/.ssh;'
             'cat /tmp/host_ssh_authorized_keys >> ~/.ssh/authorized_keys;'
             'sudo service ssh start;'
-            'sudo sed -i "s/mesg n/tty -s \&\& mesg n/" ~/.profile;'
+            'sudo sed -i "s/mesg n/tty -s \\&\\& mesg n/" ~/.profile;'
             f'{SETUP_ENV_VARS_CMD}',
             run_env='docker')
 
@@ -390,9 +477,13 @@ class DockerInitializer:
         user_pos = string.find('~')
         if user_pos > -1:
             if self.home_dir is None:
-                cmd = (f'{self.docker_cmd} exec {self.container_name} '
-                       'printenv HOME')
-                self.home_dir = self._run(cmd, separate_stderr=True)
+                cmd = (f'{self.docker_cmd} exec {self.container_name}'
+                       ' printenv HOME')
+                self.home_dir = self._run(
+                    cmd,
+                    separate_stderr=True,
+                    flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                    flock_args='-s -w 1')
                 # Check for unexpected newline in home directory, which can be
                 # a common issue when the output is mixed with stderr.
                 assert '\n' not in self.home_dir, (