skypilot-nightly 1.0.0.dev20250528__py3-none-any.whl → 1.0.0.dev20250530__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '19ec9254037c83057b1e3f51f5dc17045ec42b53'
+_SKYPILOT_COMMIT_SHA = '86ea13eb24c6357ea885539fd3196d88295f5943'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250528'
+__version__ = '1.0.0.dev20250530'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/nebius.py

@@ -1,17 +1,44 @@
 """Nebius cloud adaptor."""
 import os
 import threading
+from typing import List, Optional
 
+from sky import sky_logging
+from sky import skypilot_config
 from sky.adaptors import common
 from sky.utils import annotations
 from sky.utils import ux_utils
 
-NEBIUS_TENANT_ID_FILENAME = 'NEBIUS_TENANT_ID.txt'
-NEBIUS_IAM_TOKEN_FILENAME = 'NEBIUS_IAM_TOKEN.txt'
-NEBIUS_CREDENTIALS_FILENAME = 'credentials.json'
-NEBIUS_TENANT_ID_PATH = '~/.nebius/' + NEBIUS_TENANT_ID_FILENAME
-NEBIUS_IAM_TOKEN_PATH = '~/.nebius/' + NEBIUS_IAM_TOKEN_FILENAME
-NEBIUS_CREDENTIALS_PATH = '~/.nebius/' + NEBIUS_CREDENTIALS_FILENAME
+logger = sky_logging.init_logger(__name__)
+
+
+def tenant_id_path() -> str:
+    return '~/.nebius/NEBIUS_TENANT_ID.txt'
+
+
+def iam_token_path() -> str:
+    return '~/.nebius/NEBIUS_IAM_TOKEN.txt'
+
+
+def credentials_path() -> str:
+    workspace_path = skypilot_config.get_workspace_cloud('nebius').get(
+        'credentials_file_path', None)
+    if workspace_path is not None:
+        return workspace_path
+    return _get_default_credentials_path()
+
+
+def _get_workspace_credentials_path() -> Optional[str]:
+    """Get credentials path if explicitly set in workspace config."""
+    workspace_cred_path = skypilot_config.get_workspace_cloud('nebius').get(
+        'credentials_file_path', None)
+    return workspace_cred_path
+
+
+def _get_default_credentials_path() -> str:
+    """Get the default credentials path."""
+    return '~/.nebius/credentials.json'
+
 
 DEFAULT_REGION = 'eu-north1'
 
@@ -74,39 +101,77 @@ def vpc():
     return vpc_v1
 
 
-@annotations.lru_cache(scope='request')
 def get_iam_token():
     try:
-        with open(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH),
+        with open(os.path.expanduser(iam_token_path()),
                   encoding='utf-8') as file:
             return file.read().strip()
     except FileNotFoundError:
         return None
 
 
-@annotations.lru_cache(scope='request')
 def is_token_or_cred_file_exist():
-    return (os.path.exists(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH)) or
-            os.path.exists(os.path.expanduser(NEBIUS_CREDENTIALS_PATH)))
+    return (os.path.exists(os.path.expanduser(iam_token_path())) or
+            os.path.exists(os.path.expanduser(credentials_path())))
 
 
-@annotations.lru_cache(scope='request')
 def get_tenant_id():
+    tenant_id_in_ws_config = skypilot_config.get_workspace_cloud('nebius').get(
+        'tenant_id', None)
+    if tenant_id_in_ws_config is not None:
+        return tenant_id_in_ws_config
+    tenant_id_in_config = skypilot_config.get_nested(('nebius', 'tenant_id'),
+                                                     None)
+    if tenant_id_in_config is not None:
+        return tenant_id_in_config
     try:
-        with open(os.path.expanduser(NEBIUS_TENANT_ID_PATH),
+        with open(os.path.expanduser(tenant_id_path()),
                   encoding='utf-8') as file:
             return file.read().strip()
     except FileNotFoundError:
         return None
 
 
-@annotations.lru_cache(scope='request')
 def sdk():
+    """Create the Nebius SDK with the correct credentials.
+
+    The order of priority is:
+    1. Credentials file specified in workspace config, if set
+    2. IAM token file, if set
+    3. Default credentials path
+    """
+    # 1. Check if credentials path is set in workspace config (highest priority)
+    workspace_cred_path = _get_workspace_credentials_path()
+    if workspace_cred_path is not None:
+        # Check if token is also available and warn
+        token = get_iam_token()
+        if token is not None:
+            logger.warning(
+                f'Both workspace credentials file ({workspace_cred_path}) and '
+                f'IAM token file ({iam_token_path()}) are available. Using '
+                'workspace credentials file.')
+        return _sdk(None, workspace_cred_path)
+
+    # 2. Check for IAM token file (second priority)
     token = get_iam_token()
+    if token is not None:
+        return _sdk(token, None)
+
+    # 3. Fall back to default credentials path (lowest priority)
+    default_cred_path = _get_default_credentials_path()
+    return _sdk(None, default_cred_path)
+
+
+@annotations.lru_cache(scope='request')
+def _sdk(token: Optional[str], cred_path: Optional[str]):
+    # Exactly one of token or cred_path must be provided
+    assert (token is None) != (cred_path is None), (token, cred_path)
     if token is not None:
         return nebius.sdk.SDK(credentials=token)
-    return nebius.sdk.SDK(
-        credentials_file_name=os.path.expanduser(NEBIUS_CREDENTIALS_PATH))
+    if cred_path is not None:
+        return nebius.sdk.SDK(
+            credentials_file_name=os.path.expanduser(cred_path))
+    raise ValueError('Either token or credentials file path must be provided')
 
 
 def get_nebius_credentials(boto3_session):
@@ -184,3 +249,21 @@ def botocore_exceptions():
     # pylint: disable=import-outside-toplevel
     from botocore import exceptions
     return exceptions
+
+
+def get_credential_file_paths() -> List[str]:
+    """Get the list of credential file paths based on current configuration."""
+    paths = {
+        # Always include tenant ID and IAM token paths
+        tenant_id_path(),
+        iam_token_path(),
+    }
+
+    # Add workspace-specific credentials path if set
+    workspace_cred_path = _get_workspace_credentials_path()
+    if workspace_cred_path is not None:
+        paths.add(workspace_cred_path)
+    # Always add default path in case it's needed for fallback
+    paths.add(_get_default_credentials_path())
+
+    return list(paths)

sky/authentication.py

@@ -26,7 +26,7 @@ import socket
 import subprocess
 import sys
 import typing
-from typing import Any, Dict, Tuple
+from typing import Any, Dict, Optional, Tuple
 import uuid
 
 import colorama
@@ -34,6 +34,7 @@ import filelock
 
 from sky import clouds
 from sky import exceptions
+from sky import global_user_state
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
@@ -72,8 +73,10 @@ else:
     yaml = adaptors_common.LazyImport('yaml')
 
 
-def get_ssh_key_and_lock_path() -> Tuple[str, str, str]:
-    user_hash = common_utils.get_user_hash()
+def get_ssh_key_and_lock_path(
+        user_hash: Optional[str] = None) -> Tuple[str, str, str]:
+    if user_hash is None:
+        user_hash = common_utils.get_user_hash()
     user_ssh_key_prefix = _SSH_KEY_PATH_PREFIX.format(user_hash=user_hash)
 
     os.makedirs(os.path.expanduser(user_ssh_key_prefix),
@@ -129,9 +132,12 @@ def _save_key_pair(private_key_path: str, public_key_path: str,
               opener=functools.partial(os.open, mode=0o644)) as f:
         f.write(public_key)
 
+    global_user_state.set_ssh_keys(common_utils.get_user_hash(), public_key,
+                                   private_key)
+
 
 def get_or_generate_keys() -> Tuple[str, str]:
-    """Returns the aboslute private and public key paths."""
+    """Returns the absolute private and public key paths."""
     private_key_path, public_key_path, lock_path = get_ssh_key_and_lock_path()
     private_key_path = os.path.expanduser(private_key_path)
     public_key_path = os.path.expanduser(public_key_path)
@@ -144,15 +150,56 @@ def get_or_generate_keys() -> Tuple[str, str]:
     os.makedirs(lock_dir, exist_ok=True, mode=0o700)
     with filelock.FileLock(lock_path, timeout=10):
         if not os.path.exists(private_key_path):
-            public_key, private_key = _generate_rsa_key_pair()
-            _save_key_pair(private_key_path, public_key_path, private_key,
-                           public_key)
+            ssh_public_key, ssh_private_key, exists = (
+                global_user_state.get_ssh_keys(common_utils.get_user_hash()))
+            if not exists:
+                ssh_public_key, ssh_private_key = _generate_rsa_key_pair()
+            _save_key_pair(private_key_path, public_key_path, ssh_private_key,
+                           ssh_public_key)
     assert os.path.exists(public_key_path), (
         'Private key found, but associated public key '
         f'{public_key_path} does not exist.')
     return private_key_path, public_key_path
 
 
+def create_ssh_key_files_from_db(private_key_path: Optional[str] = None):
+    if private_key_path is None:
+        user_hash = common_utils.get_user_hash()
+    else:
+        # Assume private key path is in the format of
+        # ~/.sky/clients/<user_hash>/ssh/sky-key
+        separated_path = os.path.normpath(private_key_path).split(os.path.sep)
+        assert separated_path[-1] == 'sky-key'
+        assert separated_path[-2] == 'ssh'
+        user_hash = separated_path[-3]
+
+    private_key_path_generated, public_key_path, lock_path = (
+        get_ssh_key_and_lock_path(user_hash))
+    assert private_key_path == os.path.expanduser(private_key_path_generated), (
+        f'Private key path {private_key_path} does not '
+        f'match the generated path {private_key_path_generated}')
+    private_key_path = os.path.expanduser(private_key_path)
+    public_key_path = os.path.expanduser(public_key_path)
+    lock_path = os.path.expanduser(lock_path)
+
+    lock_dir = os.path.dirname(lock_path)
+    # We should have the folder ~/.sky/generated/ssh to have 0o700 permission,
+    # as the ssh configs will be written to this folder as well in
+    # backend_utils.SSHConfigHelper
+    os.makedirs(lock_dir, exist_ok=True, mode=0o700)
+    with filelock.FileLock(lock_path, timeout=10):
+        if not os.path.exists(private_key_path):
+            ssh_public_key, ssh_private_key, exists = (
+                global_user_state.get_ssh_keys(user_hash))
+            if not exists:
+                raise RuntimeError(f'SSH keys not found for user {user_hash}')
+            _save_key_pair(private_key_path, public_key_path, ssh_private_key,
+                           ssh_public_key)
+    assert os.path.exists(public_key_path), (
+        'Private key found, but associated public key '
+        f'{public_key_path} does not exist.')
+
+
 def configure_ssh_info(config: Dict[str, Any]) -> Dict[str, Any]:
     _, public_key_path = get_or_generate_keys()
     with open(public_key_path, 'r', encoding='utf-8') as f:

sky/backends/backend_utils.py

@@ -213,7 +213,7 @@ def _get_yaml_path_from_cluster_name(cluster_name: str,
 # Add retry for the file mounts optimization, as the underlying cp command may
 # experience transient errors, #4758.
 @common_utils.retry
-def _optimize_file_mounts(yaml_path: str) -> None:
+def _optimize_file_mounts(tmp_yaml_path: str) -> None:
     """Optimize file mounts in the given ray yaml file.
 
     Runtime files handling:
@@ -227,7 +227,7 @@ def _optimize_file_mounts(yaml_path: str) -> None:
         subprocess.CalledProcessError: If the file mounts are failed to be
             copied.
     """
-    yaml_config = common_utils.read_yaml(yaml_path)
+    yaml_config = common_utils.read_yaml(tmp_yaml_path)
 
     file_mounts = yaml_config.get('file_mounts', {})
     # Remove the file mounts added by the newline.
@@ -311,7 +311,7 @@ def _optimize_file_mounts(yaml_path: str) -> None:
             shell=True,
             check=True)
 
-    common_utils.dump_yaml(yaml_path, yaml_config)
+    common_utils.dump_yaml(tmp_yaml_path, yaml_config)
 
 
 def path_size_megabytes(path: str) -> int:
@@ -842,9 +842,8 @@ def write_cluster_config(
     _add_auth_to_cluster_config(cloud, tmp_yaml_path)
 
     # Restore the old yaml content for backward compatibility.
-    if os.path.exists(yaml_path) and keep_launch_fields_in_existing_config:
-        with open(yaml_path, 'r', encoding='utf-8') as f:
-            old_yaml_content = f.read()
+    old_yaml_content = global_user_state.get_cluster_yaml_str(yaml_path)
+    if old_yaml_content is not None and keep_launch_fields_in_existing_config:
         with open(tmp_yaml_path, 'r', encoding='utf-8') as f:
             new_yaml_content = f.read()
         restored_yaml_content = _replace_yaml_dicts(
@@ -881,18 +880,29 @@ def write_cluster_config(
     # compatibility should go before this call.
     _optimize_file_mounts(tmp_yaml_path)
 
-    # Rename the tmp file to the final YAML path.
-    os.rename(tmp_yaml_path, yaml_path)
-    usage_lib.messages.usage.update_ray_yaml(yaml_path)
+    # commit the final yaml to the database
+    global_user_state.set_cluster_yaml(
+        cluster_name,
+        open(tmp_yaml_path, 'r', encoding='utf-8').read())
+
+    usage_lib.messages.usage.update_ray_yaml(tmp_yaml_path)
+
+    # Remove the tmp file.
+    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+        debug_yaml_path = yaml_path + '.debug'
+        os.rename(tmp_yaml_path, debug_yaml_path)
+    else:
+        os.remove(tmp_yaml_path)
+
     return config_dict
 
 
-def _add_auth_to_cluster_config(cloud: clouds.Cloud, cluster_config_file: str):
+def _add_auth_to_cluster_config(cloud: clouds.Cloud, tmp_yaml_path: str):
     """Adds SSH key info to the cluster config.
 
     This function's output removes comments included in the jinja2 template.
     """
-    config = common_utils.read_yaml(cluster_config_file)
+    config = common_utils.read_yaml(tmp_yaml_path)
     # Check the availability of the cloud type.
     if isinstance(cloud, (
             clouds.AWS,
@@ -922,7 +932,7 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, cluster_config_file: str):
         config = auth.setup_fluidstack_authentication(config)
     else:
         assert False, cloud
-    common_utils.dump_yaml(cluster_config_file, config)
+    common_utils.dump_yaml(tmp_yaml_path, config)
 
 
 def get_timestamp_from_run_timestamp(run_timestamp: str) -> float:
@@ -980,7 +990,7 @@ def _count_healthy_nodes_from_ray(output: str,
 
 
 @timeline.event
-def _deterministic_cluster_yaml_hash(yaml_path: str) -> str:
+def _deterministic_cluster_yaml_hash(tmp_yaml_path: str) -> str:
     """Hash the cluster yaml and contents of file mounts to a unique string.
 
     Two invocations of this function should return the same string if and only
@@ -1024,7 +1034,7 @@ def _deterministic_cluster_yaml_hash(yaml_path: str) -> str:
     """
 
     # Load the yaml contents so that we can directly remove keys.
-    yaml_config = common_utils.read_yaml(yaml_path)
+    yaml_config = common_utils.read_yaml(tmp_yaml_path)
     for key_list in _RAY_YAML_KEYS_TO_REMOVE_FOR_HASH:
         dict_to_remove_from = yaml_config
         found_key = True
@@ -1053,7 +1063,7 @@ def _deterministic_cluster_yaml_hash(yaml_path: str) -> str:
         file_mounts.pop('')
 
     for dst, src in sorted(file_mounts.items()):
-        if src == yaml_path:
+        if src == tmp_yaml_path:
             # Skip the yaml file itself. We have already hashed a modified
             # version of it. The file may include fields we don't want to hash.
             continue
@@ -1148,7 +1158,7 @@ def wait_until_ray_cluster_ready(
         logger.error(common_utils.format_exception(e))
         return False, None  # failed
 
-    config = common_utils.read_yaml(cluster_config_file)
+    config = global_user_state.get_cluster_yaml_dict(cluster_config_file)
 
     docker_user = None
     if 'docker' in config:
@@ -1248,11 +1258,11 @@ def ssh_credential_from_yaml(
     """
     if cluster_yaml is None:
         return dict()
-    config = common_utils.read_yaml(cluster_yaml)
+    config = global_user_state.get_cluster_yaml_dict(cluster_yaml)
     auth_section = config['auth']
     if ssh_user is None:
         ssh_user = auth_section['ssh_user'].strip()
-    ssh_private_key = auth_section.get('ssh_private_key')
+    ssh_private_key_path = auth_section.get('ssh_private_key')
     ssh_control_name = config.get('cluster_name', '__default__')
     ssh_proxy_command = auth_section.get('ssh_proxy_command')
 
@@ -1261,9 +1271,10 @@ def ssh_credential_from_yaml(
             constants.SKY_SSH_USER_PLACEHOLDER in ssh_proxy_command):
         ssh_proxy_command = ssh_proxy_command.replace(
             constants.SKY_SSH_USER_PLACEHOLDER, ssh_user)
+
     credentials = {
         'ssh_user': ssh_user,
-        'ssh_private_key': ssh_private_key,
+        'ssh_private_key': ssh_private_key_path,
         'ssh_control_name': ssh_control_name,
         'ssh_proxy_command': ssh_proxy_command,
     }
@@ -1436,7 +1447,7 @@ def get_node_ips(cluster_yaml: str,
         exceptions.FetchClusterInfoError: if we failed to get the IPs. e.reason is
             HEAD or WORKER.
     """
-    ray_config = common_utils.read_yaml(cluster_yaml)
+    ray_config = global_user_state.get_cluster_yaml_dict(cluster_yaml)
     # Use the new provisioner for AWS.
     provider_name = cluster_utils.get_provider_name(ray_config)
     cloud = registry.CLOUD_REGISTRY.from_str(provider_name)
@@ -1652,7 +1663,7 @@ def _query_cluster_status_via_cloud_api(
     # Use region and zone from the cluster config, instead of the
     # handle.launched_resources, because the latter may not be set
     # correctly yet.
-    ray_config = common_utils.read_yaml(handle.cluster_yaml)
+    ray_config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
     provider_config = ray_config['provider']
 
     # Query the cloud provider.
@@ -2599,6 +2610,8 @@ def get_clusters(
             return
         ssh_private_key_path = credentials.get('ssh_private_key', None)
         if ssh_private_key_path is not None:
+            if not os.path.exists(os.path.expanduser(ssh_private_key_path)):
+                auth.create_ssh_key_files_from_db(ssh_private_key_path)
             with open(os.path.expanduser(ssh_private_key_path),
                       'r',
                       encoding='utf-8') as f:
@@ -2999,7 +3012,7 @@ def get_endpoints(cluster: str,
             raise ValueError('Querying endpoints is not supported '
                              f'for {cluster!r} on {cloud}.') from None
 
-    config = common_utils.read_yaml(handle.cluster_yaml)
+    config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
     port_details = provision_lib.query_ports(repr(cloud),
                                              handle.cluster_name_on_cloud,
                                              handle.launched_resources.ports,

sky/backends/cloud_vm_ray_backend.py

@@ -1235,7 +1235,8 @@ class RetryingVmProvisioner(object):
                 assert isinstance(handle, CloudVmRayResourceHandle), (
                     'handle should be CloudVmRayResourceHandle (found: '
                     f'{type(handle)}) {cluster_name!r}')
-                config = common_utils.read_yaml(handle.cluster_yaml)
+                config = global_user_state.get_cluster_yaml_dict(
+                    handle.cluster_yaml)
                 # This is for the case when the zone field is not set in the
                 # launched resources in a previous launch (e.g., ctrl-c during
                 # launch and multi-node cluster before PR #1700).
@@ -1935,7 +1936,8 @@ class RetryingVmProvisioner(object):
             # ready to ensure cluster will not scale up after preemption (spot).
             # Skip for non-spot as this takes extra time to provision (~1min).
             if use_spot:
-                ray_config = common_utils.read_yaml(cluster_config_file)
+                ray_config = global_user_state.get_cluster_yaml_dict(
+                    cluster_config_file)
                 ray_config['upscaling_speed'] = 0
                 common_utils.dump_yaml(cluster_config_file, ray_config)
                 start = time.time()
@@ -2270,7 +2272,7 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
         # Directly load the `use_internal_ips` flag from the cluster yaml
         # instead of `skypilot_config` as the latter can be changed after the
         # cluster is UP.
-        return common_utils.read_yaml(self.cluster_yaml).get(
+        return global_user_state.get_cluster_yaml_dict(self.cluster_yaml).get(
             'provider', {}).get('use_internal_ips', False)
 
     def update_ssh_ports(self, max_attempts: int = 1) -> None:
@@ -2299,7 +2301,8 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
                 # It is possible that the cluster yaml is not available when
                 # the handle is unpickled for service replicas from the
                 # controller with older version.
-                config = common_utils.read_yaml(self.cluster_yaml)
+                config = global_user_state.get_cluster_yaml_dict(
+                    self.cluster_yaml)
             try:
                 cluster_info = provision_lib.get_cluster_info(
                     provider_name,
@@ -2634,7 +2637,7 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
             # pylint: disable=import-outside-toplevel
             launched_resources = state['launched_resources']
             if isinstance(launched_resources.cloud, clouds.Kubernetes):
-                yaml_config = common_utils.read_yaml(
+                yaml_config = global_user_state.get_cluster_yaml_dict(
                     os.path.expanduser(state['_cluster_yaml']))
                 context = kubernetes_utils.get_context_from_config(
                     yaml_config['provider'])
@@ -3044,8 +3047,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             ssh_port_list = handle.external_ssh_ports()
             assert ip_list is not None, handle
             assert ssh_port_list is not None, handle
-
-            config = common_utils.read_yaml(cluster_config_file)
+            config = global_user_state.get_cluster_yaml_dict(
+                cluster_config_file)
             if 'docker' in config:
                 handle.setup_docker_user(cluster_config_file)
 
@@ -3113,7 +3116,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         cloud = handle.launched_resources.cloud
         logger.debug(
             f'Opening ports {handle.launched_resources.ports} for {cloud}')
-        config = common_utils.read_yaml(handle.cluster_yaml)
+        config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
         provider_config = config['provider']
         provision_lib.open_ports(repr(cloud), handle.cluster_name_on_cloud,
                                  handle.launched_resources.ports,
@@ -3178,6 +3181,11 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                             'Launching - Opening new ports')):
                     self._open_ports(handle)
 
+        # Capture task YAML and command
+        task_config = None
+        if task is not None:
+            task_config = task.to_yaml_config()
+
         with timeline.Event('backend.provision.post_process'):
             global_user_state.add_or_update_cluster(
                 handle.cluster_name,
@@ -3185,6 +3193,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 set(task.resources),
                 ready=True,
                 config_hash=config_hash,
+                task_config=task_config,
             )
             usage_lib.messages.usage.update_final_cluster_status(
                 status_lib.ClusterStatus.UP)
@@ -3507,9 +3516,11 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 # Add the managed job to job queue database.
                 managed_job_codegen = managed_jobs.ManagedJobCodeGen()
                 managed_job_code = managed_job_codegen.set_pending(
-                    job_id, managed_job_dag,
+                    job_id,
+                    managed_job_dag,
                     skypilot_config.get_active_workspace(
-                        force_user_workspace=True))
+                        force_user_workspace=True),
+                    entrypoint=common_utils.get_current_command())
                 # Set the managed job to PENDING state to make sure that this
                 # managed job appears in the `sky jobs queue`, even if it needs
                 # to wait to be submitted.
@@ -4195,7 +4206,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         log_abs_path = os.path.abspath(log_path)
         launched_resources = handle.launched_resources.assert_launchable()
         cloud = launched_resources.cloud
-        config = common_utils.read_yaml(handle.cluster_yaml)
+        config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
         cluster_name = handle.cluster_name
         cluster_name_on_cloud = handle.cluster_name_on_cloud
 
@@ -4255,7 +4266,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             from sky.adaptors import ibm
             from sky.skylet.providers.ibm.vpc_provider import IBMVPCProvider
 
-            config_provider = common_utils.read_yaml(
+            config_provider = global_user_state.get_cluster_yaml_dict(
                 handle.cluster_yaml)['provider']
             region = config_provider['region']
             search_client = ibm.search_client()
@@ -4409,7 +4420,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                     launched_resources = (
                         handle.launched_resources.assert_launchable())
                     cloud = launched_resources.cloud
-                    config = common_utils.read_yaml(handle.cluster_yaml)
+                    config = global_user_state.get_cluster_yaml_dict(
+                        handle.cluster_yaml)
                     cloud.check_features_are_supported(
                         launched_resources,
                         {clouds.CloudImplementationFeatures.OPEN_PORTS})
@@ -4448,7 +4460,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             # https://github.com/skypilot-org/skypilot/pull/4443#discussion_r1872798032
             attempts = 0
             while True:
-                config = common_utils.read_yaml(handle.cluster_yaml)
+                config = global_user_state.get_cluster_yaml_dict(
+                    handle.cluster_yaml)
 
                 logger.debug(f'instance statuses attempt {attempts + 1}')
                 node_status_dict = provision_lib.query_instances(
@@ -4504,9 +4517,11 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
 
     def remove_cluster_config(self, handle: CloudVmRayResourceHandle) -> None:
         """Remove the YAML config of a cluster."""
+        cluster_yaml_path = handle.cluster_yaml
         handle.cluster_yaml = None
         global_user_state.update_cluster_handle(handle.cluster_name, handle)
-        common_utils.remove_file_if_exists(handle.cluster_yaml)
+        global_user_state.remove_cluster_yaml(handle.cluster_name)
+        common_utils.remove_file_if_exists(cluster_yaml_path)
 
     def set_autostop(self,
                      handle: CloudVmRayResourceHandle,

sky/check.py

@@ -257,7 +257,7 @@ def check_capabilities(
         # Check all workspaces
         workspaces_to_check = available_workspaces
 
-        hide_per_cloud_details_flag = (not verbose and
+        hide_per_cloud_details_flag = (not verbose and clouds is None and
                                        len(workspaces_to_check) > 1)
 
         for ws_name in workspaces_to_check: