skypilot-nightly 1.0.0.dev20250828__tar.gz → 1.0.0.dev20250829__tar.gz

{skypilot_nightly-1.0.0.dev20250828/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250829}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250828
+Version: 1.0.0.dev20250829
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -34,7 +34,7 @@ Requires-Dist: python-dotenv
 Requires-Dist: rich
 Requires-Dist: tabulate
 Requires-Dist: typing_extensions
-Requires-Dist: filelock>=3.6.0
+Requires-Dist: filelock>=3.15.0
 Requires-Dist: packaging
 Requires-Dist: psutil
 Requires-Dist: pulp
@@ -59,6 +59,8 @@ Requires-Dist: gitpython
 Requires-Dist: types-paramiko
 Requires-Dist: alembic
 Requires-Dist: aiohttp
+Requires-Dist: aiosqlite
+Requires-Dist: anyio
 Provides-Extra: aws
 Requires-Dist: awscli>=1.27.10; extra == "aws"
 Requires-Dist: botocore>=1.29.10; extra == "aws"
@@ -136,8 +138,10 @@ Requires-Dist: sqlalchemy_adapter; extra == "server"
 Requires-Dist: passlib; extra == "server"
 Requires-Dist: pyjwt; extra == "server"
 Requires-Dist: aiohttp; extra == "server"
+Requires-Dist: anyio; extra == "server"
 Requires-Dist: grpcio>=1.63.0; extra == "server"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "server"
+Requires-Dist: aiosqlite; extra == "server"
 Provides-Extra: all
 Requires-Dist: awscli>=1.27.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"
@@ -194,8 +198,10 @@ Requires-Dist: sqlalchemy_adapter; extra == "all"
 Requires-Dist: passlib; extra == "all"
 Requires-Dist: pyjwt; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: anyio; extra == "all"
 Requires-Dist: grpcio>=1.63.0; extra == "all"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
+Requires-Dist: aiosqlite; extra == "all"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'ff93214498e29e0aa9a73868b73613535f96b8a3'
+_SKYPILOT_COMMIT_SHA = '27f74c78af59ef98180b59a30c43410e46e3ce37'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250828'
+__version__ = '1.0.0.dev20250829'
 __root_dir__ = directory_utils.get_sky_dir()
 
 

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/backends/backend_utils.py

@@ -1409,6 +1409,62 @@ def ssh_credential_from_yaml(
     return credentials
 
 
+def ssh_credentials_from_handles(
+    handles: List['cloud_vm_ray_backend.CloudVmRayResourceHandle'],
+) -> List[Dict[str, Any]]:
+    """Returns ssh_user, ssh_private_key and ssh_control name.
+    """
+    non_empty_cluster_yaml_paths = [
+        handle.cluster_yaml
+        for handle in handles
+        if handle.cluster_yaml is not None
+    ]
+    cluster_yaml_dicts = global_user_state.get_cluster_yaml_dict_multiple(
+        non_empty_cluster_yaml_paths)
+    cluster_yaml_dicts_to_index = {
+        cluster_yaml_path: cluster_yaml_dict
+        for cluster_yaml_path, cluster_yaml_dict in zip(
+            non_empty_cluster_yaml_paths, cluster_yaml_dicts)
+    }
+
+    credentials_to_return: List[Dict[str, Any]] = []
+    for handle in handles:
+        if handle.cluster_yaml is None:
+            credentials_to_return.append(dict())
+            continue
+        ssh_user = handle.ssh_user
+        docker_user = handle.docker_user
+        config = cluster_yaml_dicts_to_index[handle.cluster_yaml]
+        auth_section = config['auth']
+        if ssh_user is None:
+            ssh_user = auth_section['ssh_user'].strip()
+        ssh_private_key_path = auth_section.get('ssh_private_key')
+        ssh_control_name = config.get('cluster_name', '__default__')
+        ssh_proxy_command = auth_section.get('ssh_proxy_command')
+
+        # Update the ssh_user placeholder in proxy command, if required
+        if (ssh_proxy_command is not None and
+                constants.SKY_SSH_USER_PLACEHOLDER in ssh_proxy_command):
+            ssh_proxy_command = ssh_proxy_command.replace(
+                constants.SKY_SSH_USER_PLACEHOLDER, ssh_user)
+
+        credentials = {
+            'ssh_user': ssh_user,
+            'ssh_private_key': ssh_private_key_path,
+            'ssh_control_name': ssh_control_name,
+            'ssh_proxy_command': ssh_proxy_command,
+        }
+        if docker_user is not None:
+            credentials['docker_user'] = docker_user
+        ssh_provider_module = config['provider']['module']
+        # If we are running ssh command on kubernetes node.
+        if 'kubernetes' in ssh_provider_module:
+            credentials['disable_control_master'] = True
+        credentials_to_return.append(credentials)
+
+    return credentials_to_return
+
+
 def parallel_data_transfer_to_nodes(
         runners: List[command_runner.CommandRunner],
         source: Optional[str],
@@ -2056,7 +2112,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 f'{output}\n', stderr)
         return (*_count_healthy_nodes_from_ray(output), output, stderr)
 
+    ray_status_details: Optional[str] = None
+
     def run_ray_status_to_check_ray_cluster_healthy() -> bool:
+        nonlocal ray_status_details
         try:
             # NOTE: fetching the IPs is very slow as it calls into
             # `ray get head-ip/worker-ips`. Using cached IPs is safe because
@@ -2134,19 +2193,25 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 #   showing up
                 time.sleep(1)
 
+            ray_status_details = (
+                f'{ready_head + ready_workers}/{total_nodes} ready')
             raise RuntimeError(
                 f'Refreshing status ({cluster_name!r}): ray status not showing '
                 f'all nodes ({ready_head + ready_workers}/'
                 f'{total_nodes});\noutput:\n{output}\nstderr:\n{stderr}')
 
         except exceptions.FetchClusterInfoError:
+            ray_status_details = 'failed to get IPs'
             logger.debug(
                 f'Refreshing status ({cluster_name!r}) failed to get IPs.')
         except RuntimeError as e:
+            if ray_status_details is None:
+                ray_status_details = str(e)
             logger.debug(common_utils.format_exception(e))
         except Exception as e:  # pylint: disable=broad-except
             # This can be raised by `external_ssh_ports()`, due to the
             # underlying call to kubernetes API.
+            ray_status_details = str(e)
             logger.debug(f'Refreshing status ({cluster_name!r}) failed: ',
                          exc_info=e)
         return False
@@ -2259,6 +2324,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
     #  (2) Otherwise, we will reset the autostop setting, unless the cluster is
     #      autostopping/autodowning.
     some_nodes_terminated = 0 < len(node_statuses) < handle.launched_nodes
+    # If all nodes are up and ray cluster is health, we would have returned
+    # earlier. So if all_nodes_up is True and we are here, it means the ray
+    # cluster must have been unhealthy.
+    ray_cluster_unhealthy = all_nodes_up
     some_nodes_not_stopped = any(status[0] != status_lib.ClusterStatus.STOPPED
                                  for status in node_statuses)
     is_abnormal = (some_nodes_terminated or some_nodes_not_stopped)
@@ -2269,8 +2338,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
 
         if some_nodes_terminated:
             init_reason = 'one or more nodes terminated'
+        elif ray_cluster_unhealthy:
+            init_reason = f'ray cluster is unhealthy ({ray_status_details})'
         elif some_nodes_not_stopped:
-            init_reason = 'some nodes are up and some nodes are stopped'
+            init_reason = 'some but not all nodes are stopped'
         logger.debug('The cluster is abnormal. Setting to INIT status. '
                      f'node_statuses: {node_statuses}')
         if record['autostop'] >= 0:
@@ -2365,7 +2436,8 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
             # Some status reason clears after a certain time (e.g. k8s events
             # are only stored for an hour by default), so it is possible that
             # the previous event has a status reason, but now it does not.
-            init_reason_regex = f'^Cluster is abnormal because {init_reason}.*'
+            init_reason_regex = (f'^Cluster is abnormal because '
+                                 f'{re.escape(init_reason)}.*')
         log_message = f'Cluster is abnormal because {init_reason}'
         if status_reason:
             log_message += f' ({status_reason})'
@@ -2385,10 +2457,17 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
         return global_user_state.get_cluster_from_name(cluster_name)
     # Now is_abnormal is False: either node_statuses is empty or all nodes are
     # STOPPED.
+    verb = 'terminated' if to_terminate else 'stopped'
     backend = backends.CloudVmRayBackend()
     global_user_state.add_cluster_event(
-        cluster_name, None, 'All nodes terminated, cleaning up the cluster.',
-        global_user_state.ClusterEventType.STATUS_CHANGE)
+        cluster_name,
+        None,
+        f'All nodes {verb}, cleaning up the cluster.',
+        global_user_state.ClusterEventType.STATUS_CHANGE,
+        # This won't do anything for a terminated cluster, but it's needed for a
+        # stopped cluster.
+        nop_if_duplicate=True,
+    )
     backend.post_teardown_cleanup(handle, terminate=to_terminate, purge=False)
     return global_user_state.get_cluster_from_name(cluster_name)
 
@@ -2916,44 +2995,57 @@ def get_clusters(
             logger.info(f'Cluster(s) not found: {bright}{clusters_str}{reset}.')
         records = new_records
 
-    def _update_record_with_credentials_and_resources_str(
-            record: Optional[Dict[str, Any]]) -> None:
+    def _update_records_with_credentials_and_resources_str(
+            records: List[Optional[Dict[str, Any]]]) -> None:
         """Add the credentials to the record.
 
         This is useful for the client side to setup the ssh config of the
         cluster.
         """
-        if record is None:
-            return
-        handle = record['handle']
-        if handle is None:
-            return
-        record['resources_str'] = resources_utils.get_readable_resources_repr(
-            handle, simplify=True)
-        record[
-            'resources_str_full'] = resources_utils.get_readable_resources_repr(
-                handle, simplify=False)
-        credentials = ssh_credential_from_yaml(handle.cluster_yaml,
-                                               handle.docker_user,
-                                               handle.ssh_user)
-
-        if not credentials:
+        records_with_handle = []
+
+        # only act on records that have a handle
+        for record in records:
+            if record is None:
+                continue
+            handle = record['handle']
+            if handle is None:
+                continue
+            record[
+                'resources_str'] = resources_utils.get_readable_resources_repr(
+                    handle, simplify=True)
+            record[
+                'resources_str_full'] = resources_utils.get_readable_resources_repr(
+                    handle, simplify=False)
+            records_with_handle.append(record)
+        if len(records_with_handle) == 0:
             return
-        ssh_private_key_path = credentials.get('ssh_private_key', None)
-        if ssh_private_key_path is not None:
-            if not os.path.exists(os.path.expanduser(ssh_private_key_path)):
-                auth.create_ssh_key_files_from_db(ssh_private_key_path)
-            with open(os.path.expanduser(ssh_private_key_path),
-                      'r',
-                      encoding='utf-8') as f:
-                credentials['ssh_private_key_content'] = f.read()
-        else:
-            private_key_path, _ = auth.get_or_generate_keys()
-            with open(os.path.expanduser(private_key_path),
-                      'r',
-                      encoding='utf-8') as f:
-                credentials['ssh_private_key_content'] = f.read()
-        record['credentials'] = credentials
+
+        handles = [record['handle'] for record in records_with_handle]
+        credentials = ssh_credentials_from_handles(handles)
+        cached_private_keys: Dict[str, str] = {}
+        for record, credential in zip(records_with_handle, credentials):
+            if not credential:
+                continue
+            ssh_private_key_path = credential.get('ssh_private_key', None)
+            if ssh_private_key_path is not None:
+                expanded_private_key_path = os.path.expanduser(
+                    ssh_private_key_path)
+                if not os.path.exists(expanded_private_key_path):
+                    auth.create_ssh_key_files_from_db(ssh_private_key_path)
+            else:
+                private_key_path, _ = auth.get_or_generate_keys()
+                expanded_private_key_path = os.path.expanduser(private_key_path)
+            if expanded_private_key_path in cached_private_keys:
+                credential['ssh_private_key_content'] = cached_private_keys[
+                    expanded_private_key_path]
+            else:
+                with open(expanded_private_key_path, 'r',
+                          encoding='utf-8') as f:
+                    credential['ssh_private_key_content'] = f.read()
+                    cached_private_keys[expanded_private_key_path] = credential[
+                        'ssh_private_key_content']
+            record['credentials'] = credential
 
     def _update_records_with_resources(
             records: List[Optional[Dict[str, Any]]]) -> None:
@@ -2980,9 +3072,7 @@ def get_clusters(
                 if handle.launched_resources.accelerators else None)
 
     # Add auth_config to the records
-    for record in records:
-        _update_record_with_credentials_and_resources_str(record)
-
+    _update_records_with_credentials_and_resources_str(records)
     if refresh == common.StatusRefreshMode.NONE:
         # Add resources to the records
         _update_records_with_resources(records)
@@ -3022,7 +3112,7 @@ def get_clusters(
                 cluster_name,
                 force_refresh_statuses=force_refresh_statuses,
                 acquire_per_cluster_status_lock=True)
-            _update_record_with_credentials_and_resources_str(record)
+            _update_records_with_credentials_and_resources_str([record])
         except (exceptions.ClusterStatusFetchingError,
                 exceptions.CloudUserIdentityError,
                 exceptions.ClusterOwnerIdentityMismatchError) as e:

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/backends/cloud_vm_ray_backend.py

@@ -65,6 +65,7 @@ from sky.utils import context_utils
 from sky.utils import controller_utils
 from sky.utils import directory_utils
 from sky.utils import env_options
+from sky.utils import lock_events
 from sky.utils import locks
 from sky.utils import log_utils
 from sky.utils import message_utils
@@ -2498,7 +2499,12 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
         self.stable_internal_external_ips = stable_internal_external_ips
 
     @context_utils.cancellation_guard
-    @annotations.lru_cache(scope='global')
+    # we expect different request to be acting on different clusters
+    # (= different handles) so we have no real expectation of cache hit
+    # across requests.
+    # Do not change this cache to global scope
+    # without understanding https://github.com/skypilot-org/skypilot/pull/6908
+    @annotations.lru_cache(scope='request', maxsize=10)
     @timeline.event
     def get_command_runners(self,
                             force_cached: bool = False,
@@ -2854,7 +2860,12 @@ class LocalResourcesHandle(CloudVmRayResourceHandle):
         self.is_grpc_enabled = False
 
     @context_utils.cancellation_guard
-    @annotations.lru_cache(scope='global')
+    # we expect different request to be acting on different clusters
+    # (= different handles) so we have no real expectation of cache hit
+    # across requests.
+    # Do not change this cache to global scope
+    # without understanding https://github.com/skypilot-org/skypilot/pull/6908
+    @annotations.lru_cache(scope='request', maxsize=10)
     @timeline.event
     def get_command_runners(self,
                             force_cached: bool = False,
@@ -3112,7 +3123,12 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         retry_until_up: bool = False,
         skip_unnecessary_provisioning: bool = False,
     ) -> Tuple[Optional[CloudVmRayResourceHandle], bool]:
-        with timeline.DistributedLockEvent(lock_id, _CLUSTER_LOCK_TIMEOUT):
+        with lock_events.DistributedLockEvent(lock_id, _CLUSTER_LOCK_TIMEOUT):
+            # Reset spinner message to remove any mention of being blocked
+            # by other requests.
+            rich_utils.force_update_status(
+                ux_utils.spinner_message('Launching'))
+
             # Try to launch the exiting cluster first. If no existing
             # cluster, this function will create a to_provision_config
             # with required resources.

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/backends/wheel_utils.py

@@ -16,6 +16,7 @@ import pathlib
 import re
 import shutil
 import subprocess
+import sys
 import tempfile
 from typing import Optional, Tuple
 
@@ -133,19 +134,45 @@ def _build_sky_wheel() -> pathlib.Path:
         # It is important to normalize the path, otherwise 'pip wheel' would
         # treat the directory as a file and generate an empty wheel.
         norm_path = str(tmp_dir) + os.sep
+        # TODO(#5046): Consider adding native UV support for building wheels.
+        # Use `python -m pip` instead of `pip3` for better compatibility across
+        # different environments (conda, venv, UV, system Python, etc.)
         try:
-            # TODO(suquark): For python>=3.7, 'subprocess.run' supports capture
-            # of the output.
             subprocess.run([
-                'pip3', 'wheel', '--no-deps', norm_path, '--wheel-dir',
+                sys.executable, '-m', 'pip', 'wheel', '--no-deps', norm_path,
+                '--wheel-dir',
                 str(tmp_dir)
             ],
-                           stdout=subprocess.DEVNULL,
-                           stderr=subprocess.PIPE,
-                           check=True)
+                           capture_output=True,
+                           check=True,
+                           text=True)
         except subprocess.CalledProcessError as e:
-            raise RuntimeError('Failed to build pip wheel for SkyPilot. '
-                               f'Error message: {e.stderr.decode()}') from e
+            error_msg = e.stderr
+            if 'No module named pip' in error_msg:
+                # pip module not found - provide helpful suggestions based on
+                # the available package managers
+                if shutil.which('uv'):
+                    msg = ('pip module not found. Since you have UV installed, '
+                           'you can install pip by running:\n'
+                           '  uv pip install pip')
+                elif shutil.which('conda'):
+                    msg = (
+                        'pip module not found. Since you have conda installed, '
+                        'you can install pip by running:\n'
+                        '  conda install pip')
+                else:
+                    msg = ('pip module not found. Please install pip for your '
+                           f'Python environment ({sys.executable}).')
+            else:
+                # Other pip errors
+                msg = f'pip wheel command failed. Error: {error_msg}'
+            raise RuntimeError('Failed to build pip wheel for SkyPilot.\n' +
+                               msg) from e
+        except FileNotFoundError as e:
+            # Python executable not found (extremely rare)
+            raise RuntimeError(
+                f'Failed to build pip wheel for SkyPilot. '
+                f'Python executable not found: {sys.executable}') from e
 
         try:
             wheel_path = next(tmp_dir.glob(_WHEEL_PATTERN))

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/clouds/aws.py

@@ -77,6 +77,103 @@ DEFAULT_SECURITY_GROUP_NAME = f'sky-sg-{common_utils.user_and_hostname_hash()}'
 # Security group to use when user specified ports in their resources.
 USER_PORTS_SECURITY_GROUP_NAME = 'sky-sg-{}'
 
+# GPU instance types that support EFA
+# TODO(hailong): Some CPU instance types also support EFA, may need to support
+# all of them later.
+# TODO(hailong): Add the EFA info in catalog.
+_EFA_INSTANCE_TYPE_PREFIXES = [
+    'g4dn.',
+    'g5.',
+    'g6.',
+    'gr6.',
+    'g6e.',
+    'p4d.',
+    'p4de.',
+    'p5.',
+    'p5e.',
+    'p5en.',
+    'p6-b200.',
+]
+
+# Docker run options for EFA.
+# Refer to https://github.com/ofiwg/libfabric/issues/6437 for updating
+# memlock ulimit
+_EFA_DOCKER_RUN_OPTIONS = [
+    '--cap-add=IPC_LOCK',
+    '--device=/dev/infiniband',
+    '--ulimit memlock=-1:-1',
+]
+
+# AWS EFA image name.
+# Refer to https://docs.aws.amazon.com/dlami/latest/devguide/aws-deep-learning-base-gpu-ami-ubuntu-22-04.html for latest version. # pylint: disable=line-too-long
+# TODO(hailong): may need to update the version later.
+_EFA_IMAGE_NAME = 'Deep Learning Base OSS Nvidia Driver GPU AMI' \
+' (Ubuntu 22.04) 20250808'
+
+
+def _is_efa_instance_type(instance_type: str) -> bool:
+    """Check if the instance type is in EFA supported instance family."""
+    return any(
+        instance_type.startswith(prefix)
+        for prefix in _EFA_INSTANCE_TYPE_PREFIXES)
+
+
+@annotations.lru_cache(scope='global', maxsize=128)
+def _get_efa_image_id(region_name: str) -> Optional[str]:
+    """Get the EFA image id for the given region."""
+    try:
+        client = aws.client('ec2', region_name=region_name)
+        response = client.describe_images(Filters=[{
+            'Name': 'name',
+            'Values': [_EFA_IMAGE_NAME]
+        }])
+        if 'Images' not in response:
+            return None
+        if len(response['Images']) == 0:
+            return None
+        available_images = [
+            img for img in response['Images'] if img['State'] == 'available'
+        ]
+        if len(available_images) == 0:
+            return None
+        sorted_images = sorted(available_images,
+                               key=lambda x: x['CreationDate'],
+                               reverse=True)
+        return sorted_images[0]['ImageId']
+    except (aws.botocore_exceptions().NoCredentialsError,
+            aws.botocore_exceptions().ProfileNotFound,
+            aws.botocore_exceptions().ClientError) as e:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(f'Failed to get EFA image id: {e}') from None
+
+
+@annotations.lru_cache(scope='global', maxsize=128)
+def _get_max_efa_interfaces(instance_type: str, region_name: str) -> int:
+    """Get the maximum number of EFA interfaces for the given instance type."""
+    if not _is_efa_instance_type(instance_type):
+        return 0
+    try:
+        client = aws.client('ec2', region_name=region_name)
+        response = client.describe_instance_types(
+            InstanceTypes=[instance_type],
+            Filters=[{
+                'Name': 'network-info.efa-supported',
+                'Values': ['true']
+            }])
+        if 'InstanceTypes' in response and len(response['InstanceTypes']) > 0:
+            network_info = response['InstanceTypes'][0]['NetworkInfo']
+            if ('EfaInfo' in network_info and
+                    'MaximumEfaInterfaces' in network_info['EfaInfo']):
+                return network_info['EfaInfo']['MaximumEfaInterfaces']
+        return 0
+    except (aws.botocore_exceptions().NoCredentialsError,
+            aws.botocore_exceptions().ProfileNotFound,
+            aws.botocore_exceptions().ClientError) as e:
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(
+                f'Failed to get max EFA interfaces for {instance_type}: {e}'
+            ) from None
+
 
 class AWSIdentityType(enum.Enum):
     """AWS identity type.
@@ -295,8 +392,13 @@ class AWS(clouds.Cloud):
         image_id: Optional[Dict[Optional[str], str]],
         region_name: str,
         instance_type: str,
+        enable_efa: bool,
     ) -> str:
         if image_id is None:
+            if enable_efa:
+                efa_image_id = _get_efa_image_id(region_name)
+                if efa_image_id:
+                    return efa_image_id
             return cls._get_default_ami(region_name, instance_type)
         if None in image_id:
             image_id_str = image_id[None]
@@ -499,12 +601,25 @@ class AWS(clouds.Cloud):
         custom_resources = resources_utils.make_ray_custom_resources_str(
             acc_dict)
 
+        network_tier = (resources.network_tier if resources.network_tier
+                        is not None else resources_utils.NetworkTier.STANDARD)
+        if network_tier == resources_utils.NetworkTier.BEST:
+            max_efa_interfaces = _get_max_efa_interfaces(
+                resources.instance_type, region_name)
+            enable_efa = max_efa_interfaces > 0
+        else:
+            max_efa_interfaces = 0
+            enable_efa = False
+
+        docker_run_options = []
         if resources.extract_docker_image() is not None:
             image_id_to_use = None
+            if enable_efa:
+                docker_run_options = _EFA_DOCKER_RUN_OPTIONS
         else:
             image_id_to_use = resources.image_id
         image_id = self._get_image_id(image_id_to_use, region_name,
-                                      resources.instance_type)
+                                      resources.instance_type, enable_efa)
 
         root_device_name = self.get_image_root_device_name(
             image_id, region_name)
@@ -563,6 +678,8 @@ class AWS(clouds.Cloud):
             'security_group': security_group,
             'security_group_managed_by_skypilot':
                 str(security_group != user_security_group).lower(),
+            'max_efa_interfaces': max_efa_interfaces,
+            'docker_run_options': docker_run_options,
             **AWS._get_disk_specs(resources.disk_tier)
         }
 

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/dashboard/out/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_error-c66a4e8afc46f17b.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6e76f636a048e145.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_error-c66a4e8afc46f17b.js" defer=""></script><script src="/dashboard/_next/static/hYJYFIxp_ZFONR4wTIJqZ/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/hYJYFIxp_ZFONR4wTIJqZ/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"hYJYFIxp_ZFONR4wTIJqZ","assetPrefix":"/dashboard","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

skypilot_nightly-1.0.0.dev20250828/sky/dashboard/out/_next/static/chunks/webpack-6dae1cd599a34def.js → skypilot_nightly-1.0.0.dev20250829/sky/dashboard/out/_next/static/chunks/webpack-6e76f636a048e145.js

@@ -1 +1 @@
-!function(){"use strict";var t,e,n,r,c,o,u,a,i,f={},s={};function d(t){var e=s[t];if(void 0!==e)return e.exports;var n=s[t]={exports:{}},r=!0;try{f[t](n,n.exports,d),r=!1}finally{r&&delete s[t]}return n.exports}d.m=f,t=[],d.O=function(e,n,r,c){if(n){c=c||0;for(var o=t.length;o>0&&t[o-1][2]>c;o--)t[o]=t[o-1];t[o]=[n,r,c];return}for(var u=1/0,o=0;o<t.length;o++){for(var n=t[o][0],r=t[o][1],c=t[o][2],a=!0,i=0;i<n.length;i++)u>=c&&Object.keys(d.O).every(function(t){return d.O[t](n[i])})?n.splice(i--,1):(a=!1,c<u&&(u=c));if(a){t.splice(o--,1);var f=r();void 0!==f&&(e=f)}}return e},d.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return d.d(e,{a:e}),e},n=Object.getPrototypeOf?function(t){return Object.getPrototypeOf(t)}:function(t){return t.__proto__},d.t=function(t,r){if(1&r&&(t=this(t)),8&r||"object"==typeof t&&t&&(4&r&&t.__esModule||16&r&&"function"==typeof t.then))return t;var c=Object.create(null);d.r(c);var o={};e=e||[null,n({}),n([]),n(n)];for(var u=2&r&&t;"object"==typeof u&&!~e.indexOf(u);u=n(u))Object.getOwnPropertyNames(u).forEach(function(e){o[e]=function(){return t[e]}});return o.default=function(){return t},d.d(c,o),c},d.d=function(t,e){for(var n in e)d.o(e,n)&&!d.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},d.f={},d.e=function(t){return Promise.all(Object.keys(d.f).reduce(function(e,n){return d.f[n](t,e),e},[]))},d.u=function(t){return 2350===t?"static/chunks/2350.fab69e61bac57b23.js":3937===t?"static/chunks/3937.210053269f121201.js":7325===t?"static/chunks/7325.b4bc99ce0892dcd5.js":9025===t?"static/chunks/9025.a1bef12d672bb66d.js":9984===t?"static/chunks/9984.7eb6cc51fb460cae.js":9946===t?"static/chunks/9946.3b7b43c217ff70ec.js":4045===t?"static/chunks/4045.b30465273dc5e468.js":4725===t?"static/chunks/4725.10f7a9a5d3ea8208.js":7669===t?"static/chunks/7669.1f5d9a402bf5cc42.js":3785===t?"static/chunks/3785.d5b86f6ebc88e6e6.js":4783===t?"static/chunks/4783.c485f48348349f47.js":"static/chunks/"+t+"-"+({616:"3d59f75e2ccf9321",1121:"8afcf719ea87debc",1141:"943efc7aff0f0c06",1272:"1ef0bf0237faccdb",3015:"6c9c09593b1e67b6",3850:"ff4a9a69d978632b",4676:"9da7fdbde90b5549",5739:"d67458fcb1386c92",6130:"2be46d70a38f1e82",6135:"4b4d5e824b7f9d3c",6601:"06114c982db410b6",6856:"049014c6d43d127b",6989:"01359c57e018caa4",6990:"08b2a1cae076a943",7205:"88191679e7988c57",7411:"b15471acd2cba716",8969:"4a6f1a928fb6d370",9037:"89a84fd7fa31362d"})[t]+".js"},d.miniCssF=function(t){},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(t){if("object"==typeof window)return window}}(),d.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},r={},c="_N_E:",d.l=function(t,e,n,o){if(r[t]){r[t].push(e);return}if(void 0!==n)for(var u,a,i=document.getElementsByTagName("script"),f=0;f<i.length;f++){var s=i[f];if(s.getAttribute("src")==t||s.getAttribute("data-webpack")==c+n){u=s;break}}u||(a=!0,(u=document.createElement("script")).charset="utf-8",u.timeout=120,d.nc&&u.setAttribute("nonce",d.nc),u.setAttribute("data-webpack",c+n),u.src=d.tu(t)),r[t]=[e];var b=function(e,n){u.onerror=u.onload=null,clearTimeout(l);var c=r[t];if(delete r[t],u.parentNode&&u.parentNode.removeChild(u),c&&c.forEach(function(t){return t(n)}),e)return e(n)},l=setTimeout(b.bind(null,void 0,{type:"timeout",target:u}),12e4);u.onerror=b.bind(null,u.onerror),u.onload=b.bind(null,u.onload),a&&document.head.appendChild(u)},d.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},d.tt=function(){return void 0===o&&(o={createScriptURL:function(t){return t}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(o=trustedTypes.createPolicy("nextjs#bundler",o))),o},d.tu=function(t){return d.tt().createScriptURL(t)},d.p="/dashboard/_next/",u={2272:0},d.f.j=function(t,e){var n=d.o(u,t)?u[t]:void 0;if(0!==n){if(n)e.push(n[2]);else if(2272!=t){var r=new Promise(function(e,r){n=u[t]=[e,r]});e.push(n[2]=r);var c=d.p+d.u(t),o=Error();d.l(c,function(e){if(d.o(u,t)&&(0!==(n=u[t])&&(u[t]=void 0),n)){var r=e&&("load"===e.type?"missing":e.type),c=e&&e.target&&e.target.src;o.message="Loading chunk "+t+" failed.\n("+r+": "+c+")",o.name="ChunkLoadError",o.type=r,o.request=c,n[1](o)}},"chunk-"+t,t)}else u[t]=0}},d.O.j=function(t){return 0===u[t]},a=function(t,e){var n,r,c=e[0],o=e[1],a=e[2],i=0;if(c.some(function(t){return 0!==u[t]})){for(n in o)d.o(o,n)&&(d.m[n]=o[n]);if(a)var f=a(d)}for(t&&t(e);i<c.length;i++)r=c[i],d.o(u,r)&&u[r]&&u[r][0](),u[r]=0;return d.O(f)},(i=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(a.bind(null,0)),i.push=a.bind(null,i.push.bind(i)),d.nc=void 0}();
+!function(){"use strict";var t,e,n,r,c,o,u,a,i,f={},s={};function d(t){var e=s[t];if(void 0!==e)return e.exports;var n=s[t]={exports:{}},r=!0;try{f[t](n,n.exports,d),r=!1}finally{r&&delete s[t]}return n.exports}d.m=f,t=[],d.O=function(e,n,r,c){if(n){c=c||0;for(var o=t.length;o>0&&t[o-1][2]>c;o--)t[o]=t[o-1];t[o]=[n,r,c];return}for(var u=1/0,o=0;o<t.length;o++){for(var n=t[o][0],r=t[o][1],c=t[o][2],a=!0,i=0;i<n.length;i++)u>=c&&Object.keys(d.O).every(function(t){return d.O[t](n[i])})?n.splice(i--,1):(a=!1,c<u&&(u=c));if(a){t.splice(o--,1);var f=r();void 0!==f&&(e=f)}}return e},d.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return d.d(e,{a:e}),e},n=Object.getPrototypeOf?function(t){return Object.getPrototypeOf(t)}:function(t){return t.__proto__},d.t=function(t,r){if(1&r&&(t=this(t)),8&r||"object"==typeof t&&t&&(4&r&&t.__esModule||16&r&&"function"==typeof t.then))return t;var c=Object.create(null);d.r(c);var o={};e=e||[null,n({}),n([]),n(n)];for(var u=2&r&&t;"object"==typeof u&&!~e.indexOf(u);u=n(u))Object.getOwnPropertyNames(u).forEach(function(e){o[e]=function(){return t[e]}});return o.default=function(){return t},d.d(c,o),c},d.d=function(t,e){for(var n in e)d.o(e,n)&&!d.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},d.f={},d.e=function(t){return Promise.all(Object.keys(d.f).reduce(function(e,n){return d.f[n](t,e),e},[]))},d.u=function(t){return 2350===t?"static/chunks/2350.fab69e61bac57b23.js":7325===t?"static/chunks/7325.b4bc99ce0892dcd5.js":3937===t?"static/chunks/3937.210053269f121201.js":9025===t?"static/chunks/9025.a1bef12d672bb66d.js":9984===t?"static/chunks/9984.7eb6cc51fb460cae.js":9946===t?"static/chunks/9946.3b7b43c217ff70ec.js":7669===t?"static/chunks/7669.1f5d9a402bf5cc42.js":4045===t?"static/chunks/4045.b30465273dc5e468.js":4725===t?"static/chunks/4725.10f7a9a5d3ea8208.js":3785===t?"static/chunks/3785.d5b86f6ebc88e6e6.js":4783===t?"static/chunks/4783.c485f48348349f47.js":"static/chunks/"+t+"-"+({616:"3d59f75e2ccf9321",1121:"8afcf719ea87debc",1141:"943efc7aff0f0c06",1272:"1ef0bf0237faccdb",3015:"6c9c09593b1e67b6",3850:"ff4a9a69d978632b",4676:"9da7fdbde90b5549",5739:"d67458fcb1386c92",6130:"2be46d70a38f1e82",6135:"4b4d5e824b7f9d3c",6601:"06114c982db410b6",6856:"049014c6d43d127b",6989:"01359c57e018caa4",6990:"08b2a1cae076a943",7205:"88191679e7988c57",7411:"b15471acd2cba716",8969:"4a6f1a928fb6d370",9037:"89a84fd7fa31362d"})[t]+".js"},d.miniCssF=function(t){},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(t){if("object"==typeof window)return window}}(),d.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},r={},c="_N_E:",d.l=function(t,e,n,o){if(r[t]){r[t].push(e);return}if(void 0!==n)for(var u,a,i=document.getElementsByTagName("script"),f=0;f<i.length;f++){var s=i[f];if(s.getAttribute("src")==t||s.getAttribute("data-webpack")==c+n){u=s;break}}u||(a=!0,(u=document.createElement("script")).charset="utf-8",u.timeout=120,d.nc&&u.setAttribute("nonce",d.nc),u.setAttribute("data-webpack",c+n),u.src=d.tu(t)),r[t]=[e];var b=function(e,n){u.onerror=u.onload=null,clearTimeout(l);var c=r[t];if(delete r[t],u.parentNode&&u.parentNode.removeChild(u),c&&c.forEach(function(t){return t(n)}),e)return e(n)},l=setTimeout(b.bind(null,void 0,{type:"timeout",target:u}),12e4);u.onerror=b.bind(null,u.onerror),u.onload=b.bind(null,u.onload),a&&document.head.appendChild(u)},d.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},d.tt=function(){return void 0===o&&(o={createScriptURL:function(t){return t}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(o=trustedTypes.createPolicy("nextjs#bundler",o))),o},d.tu=function(t){return d.tt().createScriptURL(t)},d.p="/dashboard/_next/",u={2272:0},d.f.j=function(t,e){var n=d.o(u,t)?u[t]:void 0;if(0!==n){if(n)e.push(n[2]);else if(2272!=t){var r=new Promise(function(e,r){n=u[t]=[e,r]});e.push(n[2]=r);var c=d.p+d.u(t),o=Error();d.l(c,function(e){if(d.o(u,t)&&(0!==(n=u[t])&&(u[t]=void 0),n)){var r=e&&("load"===e.type?"missing":e.type),c=e&&e.target&&e.target.src;o.message="Loading chunk "+t+" failed.\n("+r+": "+c+")",o.name="ChunkLoadError",o.type=r,o.request=c,n[1](o)}},"chunk-"+t,t)}else u[t]=0}},d.O.j=function(t){return 0===u[t]},a=function(t,e){var n,r,c=e[0],o=e[1],a=e[2],i=0;if(c.some(function(t){return 0!==u[t]})){for(n in o)d.o(o,n)&&(d.m[n]=o[n]);if(a)var f=a(d)}for(t&&t(e);i<c.length;i++)r=c[i],d.o(u,r)&&u[r]&&u[r][0](),u[r]=0;return d.O(f)},(i=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(a.bind(null,0)),i.push=a.bind(null,i.push.bind(i)),d.nc=void 0}();

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250829}/sky/dashboard/out/clusters/[cluster]/[job].html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D/%5Bjob%5D-06afb50d25f7c61f.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]/[job]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6e76f636a048e145.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D/%5Bjob%5D-06afb50d25f7c61f.js" defer=""></script><script src="/dashboard/_next/static/hYJYFIxp_ZFONR4wTIJqZ/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/hYJYFIxp_ZFONR4wTIJqZ/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]/[job]","query":{},"buildId":"hYJYFIxp_ZFONR4wTIJqZ","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>