skypilot-nightly 1.0.0.dev20250828__tar.gz → 1.0.0.dev20250831__tar.gz

{skypilot_nightly-1.0.0.dev20250828/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250831}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250828
+Version: 1.0.0.dev20250831
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -13,6 +13,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -20,6 +22,8 @@ Classifier: Topic :: System :: Distributed Computing
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: wheel<0.46.0
+Requires-Dist: setuptools
+Requires-Dist: pip
 Requires-Dist: cachetools
 Requires-Dist: click<8.2.0,>=7.0
 Requires-Dist: colorama
@@ -59,6 +63,7 @@ Requires-Dist: gitpython
 Requires-Dist: types-paramiko
 Requires-Dist: alembic
 Requires-Dist: aiohttp
+Requires-Dist: anyio
 Provides-Extra: aws
 Requires-Dist: awscli>=1.27.10; extra == "aws"
 Requires-Dist: botocore>=1.29.10; extra == "aws"
@@ -73,7 +78,6 @@ Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "azure"
 Requires-Dist: azure-storage-blob>=12.23.1; extra == "azure"
 Requires-Dist: msgraph-sdk; extra == "azure"
 Requires-Dist: msrestazure; extra == "azure"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "azure"
 Provides-Extra: gcp
 Requires-Dist: google-api-python-client>=2.69.0; extra == "gcp"
 Requires-Dist: google-cloud-storage; extra == "gcp"
@@ -97,7 +101,6 @@ Provides-Extra: scp
 Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "scp"
 Provides-Extra: oci
 Requires-Dist: oci; extra == "oci"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "oci"
 Provides-Extra: kubernetes
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "kubernetes"
 Requires-Dist: websockets; extra == "kubernetes"
@@ -136,66 +139,51 @@ Requires-Dist: sqlalchemy_adapter; extra == "server"
 Requires-Dist: passlib; extra == "server"
 Requires-Dist: pyjwt; extra == "server"
 Requires-Dist: aiohttp; extra == "server"
+Requires-Dist: anyio; extra == "server"
 Requires-Dist: grpcio>=1.63.0; extra == "server"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "server"
 Provides-Extra: all
-Requires-Dist: awscli>=1.27.10; extra == "all"
-Requires-Dist: botocore>=1.29.10; extra == "all"
-Requires-Dist: boto3>=1.26.1; extra == "all"
+Requires-Dist: sqlalchemy_adapter; extra == "all"
 Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: azure-cli>=2.65.0; extra == "all"
-Requires-Dist: azure-core>=1.31.0; extra == "all"
-Requires-Dist: azure-identity>=1.19.0; extra == "all"
-Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
-Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
-Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
 Requires-Dist: msgraph-sdk; extra == "all"
-Requires-Dist: msrestazure; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
-Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
-Requires-Dist: google-cloud-storage; extra == "all"
-Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
-Requires-Dist: ibm-cloud-sdk-core; extra == "all"
-Requires-Dist: ibm-vpc; extra == "all"
 Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
-Requires-Dist: ibm-cos-sdk; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: docker; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
+Requires-Dist: msrestazure; extra == "all"
+Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
+Requires-Dist: pydo>=0.3.0; extra == "all"
+Requires-Dist: python-dateutil; extra == "all"
+Requires-Dist: oci; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
-Requires-Dist: botocore>=1.29.10; extra == "all"
+Requires-Dist: ibm-cloud-sdk-core; extra == "all"
+Requires-Dist: casbin; extra == "all"
+Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
 Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
-Requires-Dist: oci; extra == "all"
-Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
-Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
-Requires-Dist: websockets; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
-Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
+Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
 Requires-Dist: websockets; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
-Requires-Dist: grpcio>=1.63.0; extra == "all"
-Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
-Requires-Dist: runpod>=1.6.1; extra == "all"
-Requires-Dist: cudo-compute>=0.1.10; extra == "all"
-Requires-Dist: pydo>=0.3.0; extra == "all"
 Requires-Dist: azure-core>=1.24.0; extra == "all"
-Requires-Dist: azure-common; extra == "all"
-Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
-Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: nebius>=0.2.47; extra == "all"
-Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: cudo-compute>=0.1.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"
-Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: casbin; extra == "all"
-Requires-Dist: sqlalchemy_adapter; extra == "all"
-Requires-Dist: passlib; extra == "all"
-Requires-Dist: pyjwt; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: anyio; extra == "all"
+Requires-Dist: azure-core>=1.31.0; extra == "all"
+Requires-Dist: ray[default]!=2.6.0,>=2.2.0; extra == "all"
 Requires-Dist: grpcio>=1.63.0; extra == "all"
+Requires-Dist: azure-identity>=1.19.0; extra == "all"
+Requires-Dist: passlib; extra == "all"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
+Requires-Dist: azure-common; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
+Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
+Requires-Dist: pyjwt; extra == "all"
+Requires-Dist: nebius>=0.2.47; extra == "all"
+Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
+Requires-Dist: ibm-vpc; extra == "all"
+Requires-Dist: azure-cli>=2.65.0; extra == "all"
+Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
+Requires-Dist: google-cloud-storage; extra == "all"
+Requires-Dist: ibm-cos-sdk; extra == "all"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{skypilot_nightly-1.0.0.dev20250828/sky/setup_files → skypilot_nightly-1.0.0.dev20250831}/setup.py

@@ -178,6 +178,8 @@ setuptools.setup(
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: OS Independent',
         'Topic :: Software Development :: Libraries :: Python Modules',

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250831}/sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'ff93214498e29e0aa9a73868b73613535f96b8a3'
+_SKYPILOT_COMMIT_SHA = 'f394ebc3a46b5d2cde5afb8a765d97709ea584ed'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250828'
+__version__ = '1.0.0.dev20250831'
 __root_dir__ = directory_utils.get_sky_dir()
 
 

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250831}/sky/adaptors/nebius.py

@@ -62,6 +62,10 @@ def iam_token_path() -> str:
     return '~/.nebius/NEBIUS_IAM_TOKEN.txt'
 
 
+def domain_path() -> str:
+    return '~/.nebius/NEBIUS_DOMAIN.txt'
+
+
 def credentials_path() -> str:
     workspace_path = skypilot_config.get_workspace_cloud('nebius').get(
         'credentials_file_path', None)
@@ -82,6 +86,22 @@ def _get_default_credentials_path() -> str:
     return '~/.nebius/credentials.json'
 
 
+def api_domain() -> Optional[str]:
+    domain_in_ws_config = skypilot_config.get_workspace_cloud('nebius').get(
+        'domain', None)
+    if domain_in_ws_config is not None:
+        return domain_in_ws_config
+    domain_in_config = skypilot_config.get_effective_region_config(
+        cloud='nebius', region=None, keys=('domain',), default_value=None)
+    if domain_in_config is not None:
+        return domain_in_config
+    try:
+        with open(os.path.expanduser(domain_path()), encoding='utf-8') as file:
+            return file.read().strip()
+    except FileNotFoundError:
+        return None
+
+
 DEFAULT_REGION = 'eu-north1'
 
 NEBIUS_PROFILE_NAME = 'nebius'
@@ -215,10 +235,12 @@ def _sdk(token: Optional[str], cred_path: Optional[str]):
     # Exactly one of token or cred_path must be provided
     assert (token is None) != (cred_path is None), (token, cred_path)
     if token is not None:
-        return nebius.sdk.SDK(credentials=token)
+        return nebius.sdk.SDK(credentials=token, domain=api_domain())
     if cred_path is not None:
         return nebius.sdk.SDK(
-            credentials_file_name=os.path.expanduser(cred_path))
+            credentials_file_name=os.path.expanduser(cred_path),
+            domain=api_domain(),
+        )
     raise ValueError('Either token or credentials file path must be provided')
 
 

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250831}/sky/backends/backend_utils.py

@@ -928,19 +928,19 @@ def write_cluster_config(
     # Add kubernetes config fields from ~/.sky/config
     if isinstance(cloud, clouds.Kubernetes):
         cluster_config_overrides = to_provision.cluster_config_overrides
-        kubernetes_utils.combine_pod_config_fields(
-            tmp_yaml_path,
+        with open(tmp_yaml_path, 'r', encoding='utf-8') as f:
+            tmp_yaml_str = f.read()
+        cluster_yaml_obj = yaml_utils.safe_load(tmp_yaml_str)
+        combined_yaml_obj = kubernetes_utils.combine_pod_config_fields_and_metadata(
+            cluster_yaml_obj,
             cluster_config_overrides=cluster_config_overrides,
             cloud=cloud,
             context=region.name)
-        kubernetes_utils.combine_metadata_fields(
-            tmp_yaml_path,
-            cluster_config_overrides=cluster_config_overrides,
-            context=region.name)
-        yaml_obj = yaml_utils.read_yaml(tmp_yaml_path)
-        pod_config: Dict[str, Any] = yaml_obj['available_node_types'][
-            'ray_head_default']['node_config']
+        # Write the updated YAML back to the file
+        yaml_utils.dump_yaml(tmp_yaml_path, combined_yaml_obj)
 
+        pod_config: Dict[str, Any] = combined_yaml_obj['available_node_types'][
+            'ray_head_default']['node_config']
         # Check pod spec only. For high availability controllers, we deploy pvc & deployment for the controller. Read kubernetes-ray.yml.j2 for more details.
         pod_config.pop('deployment_spec', None)
         pod_config.pop('pvc_spec', None)
@@ -1409,6 +1409,62 @@ def ssh_credential_from_yaml(
     return credentials
 
 
+def ssh_credentials_from_handles(
+    handles: List['cloud_vm_ray_backend.CloudVmRayResourceHandle'],
+) -> List[Dict[str, Any]]:
+    """Returns ssh_user, ssh_private_key and ssh_control name.
+    """
+    non_empty_cluster_yaml_paths = [
+        handle.cluster_yaml
+        for handle in handles
+        if handle.cluster_yaml is not None
+    ]
+    cluster_yaml_dicts = global_user_state.get_cluster_yaml_dict_multiple(
+        non_empty_cluster_yaml_paths)
+    cluster_yaml_dicts_to_index = {
+        cluster_yaml_path: cluster_yaml_dict
+        for cluster_yaml_path, cluster_yaml_dict in zip(
+            non_empty_cluster_yaml_paths, cluster_yaml_dicts)
+    }
+
+    credentials_to_return: List[Dict[str, Any]] = []
+    for handle in handles:
+        if handle.cluster_yaml is None:
+            credentials_to_return.append(dict())
+            continue
+        ssh_user = handle.ssh_user
+        docker_user = handle.docker_user
+        config = cluster_yaml_dicts_to_index[handle.cluster_yaml]
+        auth_section = config['auth']
+        if ssh_user is None:
+            ssh_user = auth_section['ssh_user'].strip()
+        ssh_private_key_path = auth_section.get('ssh_private_key')
+        ssh_control_name = config.get('cluster_name', '__default__')
+        ssh_proxy_command = auth_section.get('ssh_proxy_command')
+
+        # Update the ssh_user placeholder in proxy command, if required
+        if (ssh_proxy_command is not None and
+                constants.SKY_SSH_USER_PLACEHOLDER in ssh_proxy_command):
+            ssh_proxy_command = ssh_proxy_command.replace(
+                constants.SKY_SSH_USER_PLACEHOLDER, ssh_user)
+
+        credentials = {
+            'ssh_user': ssh_user,
+            'ssh_private_key': ssh_private_key_path,
+            'ssh_control_name': ssh_control_name,
+            'ssh_proxy_command': ssh_proxy_command,
+        }
+        if docker_user is not None:
+            credentials['docker_user'] = docker_user
+        ssh_provider_module = config['provider']['module']
+        # If we are running ssh command on kubernetes node.
+        if 'kubernetes' in ssh_provider_module:
+            credentials['disable_control_master'] = True
+        credentials_to_return.append(credentials)
+
+    return credentials_to_return
+
+
 def parallel_data_transfer_to_nodes(
         runners: List[command_runner.CommandRunner],
         source: Optional[str],
@@ -2056,7 +2112,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 f'{output}\n', stderr)
         return (*_count_healthy_nodes_from_ray(output), output, stderr)
 
+    ray_status_details: Optional[str] = None
+
     def run_ray_status_to_check_ray_cluster_healthy() -> bool:
+        nonlocal ray_status_details
         try:
             # NOTE: fetching the IPs is very slow as it calls into
             # `ray get head-ip/worker-ips`. Using cached IPs is safe because
@@ -2134,19 +2193,25 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 #   showing up
                 time.sleep(1)
 
+            ray_status_details = (
+                f'{ready_head + ready_workers}/{total_nodes} ready')
             raise RuntimeError(
                 f'Refreshing status ({cluster_name!r}): ray status not showing '
                 f'all nodes ({ready_head + ready_workers}/'
                 f'{total_nodes});\noutput:\n{output}\nstderr:\n{stderr}')
 
         except exceptions.FetchClusterInfoError:
+            ray_status_details = 'failed to get IPs'
             logger.debug(
                 f'Refreshing status ({cluster_name!r}) failed to get IPs.')
         except RuntimeError as e:
+            if ray_status_details is None:
+                ray_status_details = str(e)
             logger.debug(common_utils.format_exception(e))
         except Exception as e:  # pylint: disable=broad-except
             # This can be raised by `external_ssh_ports()`, due to the
             # underlying call to kubernetes API.
+            ray_status_details = str(e)
             logger.debug(f'Refreshing status ({cluster_name!r}) failed: ',
                          exc_info=e)
         return False
@@ -2259,6 +2324,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
     #  (2) Otherwise, we will reset the autostop setting, unless the cluster is
     #      autostopping/autodowning.
     some_nodes_terminated = 0 < len(node_statuses) < handle.launched_nodes
+    # If all nodes are up and ray cluster is health, we would have returned
+    # earlier. So if all_nodes_up is True and we are here, it means the ray
+    # cluster must have been unhealthy.
+    ray_cluster_unhealthy = all_nodes_up
     some_nodes_not_stopped = any(status[0] != status_lib.ClusterStatus.STOPPED
                                  for status in node_statuses)
     is_abnormal = (some_nodes_terminated or some_nodes_not_stopped)
@@ -2269,8 +2338,10 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
 
         if some_nodes_terminated:
             init_reason = 'one or more nodes terminated'
+        elif ray_cluster_unhealthy:
+            init_reason = f'ray cluster is unhealthy ({ray_status_details})'
         elif some_nodes_not_stopped:
-            init_reason = 'some nodes are up and some nodes are stopped'
+            init_reason = 'some but not all nodes are stopped'
         logger.debug('The cluster is abnormal. Setting to INIT status. '
                      f'node_statuses: {node_statuses}')
         if record['autostop'] >= 0:
@@ -2365,7 +2436,8 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
             # Some status reason clears after a certain time (e.g. k8s events
             # are only stored for an hour by default), so it is possible that
             # the previous event has a status reason, but now it does not.
-            init_reason_regex = f'^Cluster is abnormal because {init_reason}.*'
+            init_reason_regex = (f'^Cluster is abnormal because '
+                                 f'{re.escape(init_reason)}.*')
         log_message = f'Cluster is abnormal because {init_reason}'
         if status_reason:
             log_message += f' ({status_reason})'
@@ -2385,10 +2457,17 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
         return global_user_state.get_cluster_from_name(cluster_name)
     # Now is_abnormal is False: either node_statuses is empty or all nodes are
     # STOPPED.
+    verb = 'terminated' if to_terminate else 'stopped'
     backend = backends.CloudVmRayBackend()
     global_user_state.add_cluster_event(
-        cluster_name, None, 'All nodes terminated, cleaning up the cluster.',
-        global_user_state.ClusterEventType.STATUS_CHANGE)
+        cluster_name,
+        None,
+        f'All nodes {verb}, cleaning up the cluster.',
+        global_user_state.ClusterEventType.STATUS_CHANGE,
+        # This won't do anything for a terminated cluster, but it's needed for a
+        # stopped cluster.
+        nop_if_duplicate=True,
+    )
     backend.post_teardown_cleanup(handle, terminate=to_terminate, purge=False)
     return global_user_state.get_cluster_from_name(cluster_name)
 
@@ -2850,6 +2929,7 @@ def get_clusters(
     refresh: common.StatusRefreshMode,
     cluster_names: Optional[Union[str, List[str]]] = None,
     all_users: bool = True,
+    include_credentials: bool = False,
     # Internal only:
     # pylint: disable=invalid-name
     _include_is_managed: bool = False,
@@ -2862,17 +2942,14 @@ def get_clusters(
     of the clusters.
 
     Args:
-        include_controller: Whether to include controllers, e.g. jobs controller
-            or sky serve controller.
         refresh: Whether to refresh the status of the clusters. (Refreshing will
             set the status to STOPPED if the cluster cannot be pinged.)
-        cloud_filter: Sets which clouds to filer through from the global user
-            state. Supports three values, 'all' for all clouds, 'public' for
-            public clouds only, and 'local' for only local clouds.
         cluster_names: If provided, only return records for the given cluster
             names.
         all_users: If True, return clusters from all users. If False, only
             return clusters from the current user.
+        include_credentials: If True, include cluster ssh credentials in the
+            return value.
         _include_is_managed: Whether to force include clusters created by the
             controller.
 
@@ -2916,54 +2993,68 @@ def get_clusters(
             logger.info(f'Cluster(s) not found: {bright}{clusters_str}{reset}.')
         records = new_records
 
-    def _update_record_with_credentials_and_resources_str(
-            record: Optional[Dict[str, Any]]) -> None:
+    def _get_records_with_handle(
+            records: List[Optional[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+        """Filter for records that have a handle"""
+        return [
+            record for record in records
+            if record is not None and record['handle'] is not None
+        ]
+
+    def _update_records_with_resources_str(
+            records: List[Optional[Dict[str, Any]]]) -> None:
+        """Add resource str to record"""
+        for record in _get_records_with_handle(records):
+            handle = record['handle']
+            record[
+                'resources_str'] = resources_utils.get_readable_resources_repr(
+                    handle, simplify=True)
+            record[
+                'resources_str_full'] = resources_utils.get_readable_resources_repr(
+                    handle, simplify=False)
+
+    def _update_records_with_credentials(
+            records: List[Optional[Dict[str, Any]]]) -> None:
         """Add the credentials to the record.
 
         This is useful for the client side to setup the ssh config of the
         cluster.
         """
-        if record is None:
-            return
-        handle = record['handle']
-        if handle is None:
+        records_with_handle = _get_records_with_handle(records)
+        if len(records_with_handle) == 0:
             return
-        record['resources_str'] = resources_utils.get_readable_resources_repr(
-            handle, simplify=True)
-        record[
-            'resources_str_full'] = resources_utils.get_readable_resources_repr(
-                handle, simplify=False)
-        credentials = ssh_credential_from_yaml(handle.cluster_yaml,
-                                               handle.docker_user,
-                                               handle.ssh_user)
-
-        if not credentials:
-            return
-        ssh_private_key_path = credentials.get('ssh_private_key', None)
-        if ssh_private_key_path is not None:
-            if not os.path.exists(os.path.expanduser(ssh_private_key_path)):
-                auth.create_ssh_key_files_from_db(ssh_private_key_path)
-            with open(os.path.expanduser(ssh_private_key_path),
-                      'r',
-                      encoding='utf-8') as f:
-                credentials['ssh_private_key_content'] = f.read()
-        else:
-            private_key_path, _ = auth.get_or_generate_keys()
-            with open(os.path.expanduser(private_key_path),
-                      'r',
-                      encoding='utf-8') as f:
-                credentials['ssh_private_key_content'] = f.read()
-        record['credentials'] = credentials
+
+        handles = [record['handle'] for record in records_with_handle]
+        credentials = ssh_credentials_from_handles(handles)
+        cached_private_keys: Dict[str, str] = {}
+        for record, credential in zip(records_with_handle, credentials):
+            if not credential:
+                continue
+            ssh_private_key_path = credential.get('ssh_private_key', None)
+            if ssh_private_key_path is not None:
+                expanded_private_key_path = os.path.expanduser(
+                    ssh_private_key_path)
+                if not os.path.exists(expanded_private_key_path):
+                    auth.create_ssh_key_files_from_db(ssh_private_key_path)
+            else:
+                private_key_path, _ = auth.get_or_generate_keys()
+                expanded_private_key_path = os.path.expanduser(private_key_path)
+            if expanded_private_key_path in cached_private_keys:
+                credential['ssh_private_key_content'] = cached_private_keys[
+                    expanded_private_key_path]
+            else:
+                with open(expanded_private_key_path, 'r',
+                          encoding='utf-8') as f:
+                    credential['ssh_private_key_content'] = f.read()
+                    cached_private_keys[expanded_private_key_path] = credential[
+                        'ssh_private_key_content']
+            record['credentials'] = credential
 
     def _update_records_with_resources(
             records: List[Optional[Dict[str, Any]]]) -> None:
         """Add the resources to the record."""
-        for record in records:
-            if record is None:
-                continue
+        for record in _get_records_with_handle(records):
             handle = record['handle']
-            if handle is None:
-                continue
             record['nodes'] = handle.launched_nodes
             if handle.launched_resources is None:
                 continue
@@ -2980,9 +3071,9 @@ def get_clusters(
                 if handle.launched_resources.accelerators else None)
 
     # Add auth_config to the records
-    for record in records:
-        _update_record_with_credentials_and_resources_str(record)
-
+    _update_records_with_resources_str(records)
+    if include_credentials:
+        _update_records_with_credentials(records)
     if refresh == common.StatusRefreshMode.NONE:
         # Add resources to the records
         _update_records_with_resources(records)
@@ -3022,7 +3113,9 @@ def get_clusters(
                 cluster_name,
                 force_refresh_statuses=force_refresh_statuses,
                 acquire_per_cluster_status_lock=True)
-            _update_record_with_credentials_and_resources_str(record)
+            _update_records_with_resources_str([record])
+            if include_credentials:
+                _update_records_with_credentials([record])
         except (exceptions.ClusterStatusFetchingError,
                 exceptions.CloudUserIdentityError,
                 exceptions.ClusterOwnerIdentityMismatchError) as e:

{skypilot_nightly-1.0.0.dev20250828 → skypilot_nightly-1.0.0.dev20250831}/sky/backends/cloud_vm_ray_backend.py

@@ -65,6 +65,7 @@ from sky.utils import context_utils
 from sky.utils import controller_utils
 from sky.utils import directory_utils
 from sky.utils import env_options
+from sky.utils import lock_events
 from sky.utils import locks
 from sky.utils import log_utils
 from sky.utils import message_utils
@@ -2498,7 +2499,12 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
         self.stable_internal_external_ips = stable_internal_external_ips
 
     @context_utils.cancellation_guard
-    @annotations.lru_cache(scope='global')
+    # we expect different request to be acting on different clusters
+    # (= different handles) so we have no real expectation of cache hit
+    # across requests.
+    # Do not change this cache to global scope
+    # without understanding https://github.com/skypilot-org/skypilot/pull/6908
+    @annotations.lru_cache(scope='request', maxsize=10)
     @timeline.event
     def get_command_runners(self,
                             force_cached: bool = False,
@@ -2854,7 +2860,12 @@ class LocalResourcesHandle(CloudVmRayResourceHandle):
         self.is_grpc_enabled = False
 
     @context_utils.cancellation_guard
-    @annotations.lru_cache(scope='global')
+    # we expect different request to be acting on different clusters
+    # (= different handles) so we have no real expectation of cache hit
+    # across requests.
+    # Do not change this cache to global scope
+    # without understanding https://github.com/skypilot-org/skypilot/pull/6908
+    @annotations.lru_cache(scope='request', maxsize=10)
     @timeline.event
     def get_command_runners(self,
                             force_cached: bool = False,
@@ -3112,7 +3123,12 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         retry_until_up: bool = False,
         skip_unnecessary_provisioning: bool = False,
     ) -> Tuple[Optional[CloudVmRayResourceHandle], bool]:
-        with timeline.DistributedLockEvent(lock_id, _CLUSTER_LOCK_TIMEOUT):
+        with lock_events.DistributedLockEvent(lock_id, _CLUSTER_LOCK_TIMEOUT):
+            # Reset spinner message to remove any mention of being blocked
+            # by other requests.
+            rich_utils.force_update_status(
+                ux_utils.spinner_message('Launching'))
+
             # Try to launch the exiting cluster first. If no existing
             # cluster, this function will create a to_provision_config
             # with required resources.
@@ -5141,6 +5157,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             # Take a random resource in order to get resource info that applies
             # to all resources.
             one_task_resource = list(task.resources)[0]
+
             # Assume resources share the same ports.
             for resource in task.resources:
                 assert resource.ports == one_task_resource.ports
@@ -5181,6 +5198,42 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             if one_task_resource.docker_login_config is not None:
                 to_provision = to_provision.copy(
                     _docker_login_config=one_task_resource.docker_login_config)
+
+            # cluster_config_overrides should be the same for all resources.
+            for resource in task.resources:
+                assert (resource.cluster_config_overrides ==
+                        one_task_resource.cluster_config_overrides)
+            if isinstance(to_provision.cloud, clouds.Kubernetes):
+                # Warn users if the Kubernetes pod config is different
+                # from the existing cluster.
+                cluster_yaml_str = global_user_state.get_cluster_yaml_str(
+                    cluster_name)
+                actual_cluster_yaml_obj = yaml_utils.safe_load(cluster_yaml_str)
+                desired_cluster_yaml_obj = (
+                    kubernetes_utils.combine_pod_config_fields_and_metadata(
+                        actual_cluster_yaml_obj,
+                        cluster_config_overrides=one_task_resource.
+                        cluster_config_overrides,
+                        cloud=to_provision.cloud,
+                        context=to_provision.region))
+
+                def _get_pod_config(yaml_obj: Dict[str, Any]) -> Dict[str, Any]:
+                    return (yaml_obj.get('available_node_types',
+                                         {}).get('ray_head_default',
+                                                 {}).get('node_config', {}))
+
+                if _get_pod_config(desired_cluster_yaml_obj) != _get_pod_config(
+                        actual_cluster_yaml_obj):
+                    # pylint: disable=line-too-long
+                    logger.warning(
+                        f'{colorama.Fore.YELLOW}WARNING: Kubernetes pod config mismatch detected. Task requires different '
+                        f'pod config than the existing cluster. The existing '
+                        f'cluster will be used with its current pod config.'
+                        f'To apply use your task\'s new pod config:\n'
+                        f'  • Use a new cluster'
+                        f'  • Or restart this cluster: sky down {cluster_name}; sky launch -c {cluster_name} ...'
+                        f'{colorama.Style.RESET_ALL}')
+
             return RetryingVmProvisioner.ToProvisionConfig(
                 cluster_name,
                 to_provision,