skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/utils/locks.py

@@ -11,15 +11,19 @@ import time
 from typing import Any, Optional
 
 import filelock
+import psycopg2
 import sqlalchemy
 
 from sky import global_user_state
-from sky.skylet import constants
+from sky.skylet import runtime_utils
 from sky.utils import common_utils
 from sky.utils.db import db_utils
 
 logger = logging.getLogger(__name__)
 
+# The directory for file locks.
+SKY_LOCKS_DIR = runtime_utils.get_runtime_dir_path('.sky/locks')
+
 
 class LockTimeout(RuntimeError):
     """Raised when a lock acquisition times out."""
@@ -126,9 +130,8 @@ class FileLock(DistributedLock):
             poll_interval: Interval in seconds to poll for lock acquisition.
         """
         super().__init__(lock_id, timeout, poll_interval)
-        os.makedirs(constants.SKY_LOCKS_DIR, exist_ok=True)
-        self.lock_path = os.path.join(constants.SKY_LOCKS_DIR,
-                                      f'.{lock_id}.lock')
+        os.makedirs(SKY_LOCKS_DIR, exist_ok=True)
+        self.lock_path = os.path.join(SKY_LOCKS_DIR, f'.{lock_id}.lock')
         if timeout is None:
             timeout = -1
         self._filelock: filelock.FileLock = filelock.FileLock(self.lock_path,
@@ -154,7 +157,7 @@ class FileLock(DistributedLock):
         common_utils.remove_file_if_exists(self.lock_path)
 
     def is_locked(self) -> bool:
-        return self._filelock.is_locked()
+        return self._filelock.is_locked
 
 
 class PostgresLock(DistributedLock):
@@ -162,15 +165,20 @@ class PostgresLock(DistributedLock):
 
     Uses PostgreSQL advisory locks to implement distributed locking
     that works across multiple machines sharing the same database.
-    Reference:
-    https://www.postgresql.org/docs/current/explicit-locking.html
-    #ADVISORY-LOCKS
+    Supports both exclusive and shared lock modes.
+
+    References:
+    # pylint: disable=line-too-long
+    - https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS
+    - https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS
+    # TODO(cooperc): re-enable pylint line-too-long
     """
 
     def __init__(self,
                  lock_id: str,
                  timeout: Optional[float] = None,
-                 poll_interval: float = 1):
+                 poll_interval: float = 1,
+                 shared_lock: bool = False):
         """Initialize the postgres lock.
 
         Args:
@@ -178,10 +186,13 @@ class PostgresLock(DistributedLock):
             timeout: Maximum time to wait for lock acquisition.
             poll_interval: Interval in seconds to poll for lock acquisition,
                 default to 1 second to avoid storming the database.
+            shared_lock: Whether to use shared advisory lock or exclusive
+                advisory lock (default).
         """
         super().__init__(lock_id, timeout, poll_interval)
         # Convert string lock_id to integer for postgres advisory locks
         self._lock_key = self._string_to_lock_key(lock_id)
+        self._shared_lock = shared_lock
         self._acquired = False
         self._connection: Optional[sqlalchemy.pool.PoolProxiedConnection] = None
 
@@ -197,6 +208,7 @@ class PostgresLock(DistributedLock):
         if engine.dialect.name != db_utils.SQLAlchemyDialect.POSTGRESQL.value:
             raise ValueError('PostgresLock requires PostgreSQL database. '
                              f'Current dialect: {engine.dialect.name}')
+        # Borrow a dedicated connection from the pool.
         return engine.raw_connection()
 
     def acquire(self, blocking: bool = True) -> AcquireReturnProxy:
@@ -209,33 +221,37 @@ class PostgresLock(DistributedLock):
 
         start_time = time.time()
 
+        if self._shared_lock:
+            lock_func = 'pg_try_advisory_lock_shared'
+        else:
+            lock_func = 'pg_try_advisory_lock'
+
         try:
             while True:
-                cursor.execute('SELECT pg_try_advisory_lock(%s)',
-                               (self._lock_key,))
+                cursor.execute(f'SELECT {lock_func}(%s)', (self._lock_key,))
                 result = cursor.fetchone()[0]
 
                 if result:
                     self._acquired = True
                     return AcquireReturnProxy(self)
 
+                mode_str = ('shared' if self._shared_lock else 'exclusive')
                 if not blocking:
                     raise LockTimeout(
-                        f'Failed to immediately acquire postgres lock '
-                        f'{self.lock_id}')
+                        f'Failed to immediately acquire {mode_str} '
+                        f'postgres lock {self.lock_id}')
 
                 if (self.timeout is not None and
                         time.time() - start_time > self.timeout):
                     raise LockTimeout(
-                        f'Failed to acquire postgres lock {self.lock_id} '
-                        f'within {self.timeout} seconds')
+                        f'Failed to acquire {mode_str} postgres lock '
+                        f'{self.lock_id} within {self.timeout} '
+                        f'seconds')
 
                 time.sleep(self.poll_interval)
 
         except Exception:
-            if self._connection:
-                self._connection.close()
-                self._connection = None
+            self._close_connection()
             raise
 
     def release(self) -> None:
@@ -243,32 +259,94 @@ class PostgresLock(DistributedLock):
         if not self._acquired or not self._connection:
             return
 
+        connection_lost = False
         try:
             cursor = self._connection.cursor()
-            cursor.execute('SELECT pg_advisory_unlock(%s)', (self._lock_key,))
+            if self._shared_lock:
+                unlock_func = 'pg_advisory_unlock_shared'
+            else:
+                unlock_func = 'pg_advisory_unlock'
+            cursor.execute(f'SELECT {unlock_func}(%s)', (self._lock_key,))
             self._connection.commit()
             self._acquired = False
+        except psycopg2.OperationalError as e:
+            # Lost connection to the database, likely the lock is force unlocked
+            # by other routines.
+            logger.debug(f'Failed to release postgres lock {self.lock_id}: {e}')
+            connection_lost = True
         finally:
-            if self._connection:
-                self._connection.close()
-                self._connection = None
+            # Invalidate if connection was lost to prevent SQLAlchemy from
+            # trying to reset a dead connection
+            self._close_connection(invalidate=connection_lost)
 
     def force_unlock(self) -> None:
         """Force unlock the postgres advisory lock."""
         try:
-            if not self._connection:
+            # The lock is held by current routine, gracefully unlock it
+            if self._acquired:
+                self.release()
+                return
+
+            # The lock is held by another routine, force unlock it.
+            if self._connection is None:
                 self._connection = self._get_connection()
             cursor = self._connection.cursor()
-            cursor.execute('SELECT pg_advisory_unlock(%s)', (self._lock_key,))
-            self._connection.commit()
+            if self._shared_lock:
+                unlock_func = 'pg_advisory_unlock_shared'
+            else:
+                unlock_func = 'pg_advisory_unlock'
+
+            cursor.execute(f'SELECT {unlock_func}(%s)', (self._lock_key,))
+            result = cursor.fetchone()[0]
+            if result:
+                # The lock is held by current routine and unlock succeed
+                self._connection.commit()
+                self._acquired = False
+                return
+            cursor.execute(
+                ('SELECT pid FROM pg_locks WHERE locktype = \'advisory\' '
+                 'AND ((classid::bigint << 32) | objid::bigint) = %s'),
+                (self._lock_key,))
+            rows = cursor.fetchall()
+            if rows:
+                # There can be multiple PIDs holding the lock, it is not enough
+                # to only kill some of them. For example, if pid 1 is holding a
+                # shared lock, and pid 2 is waiting to grab an exclusive lock,
+                # killing pid 1 will transfer the lock to pid 2, so the lock
+                # will still not be released.
+                for row in rows:
+                    cursor.execute('SELECT pg_terminate_backend(%s)', (row[0],))
+                self._connection.commit()
+                return
         except Exception as e:
             raise RuntimeError(
                 f'Failed to force unlock postgres lock {self.lock_id}: {e}'
             ) from e
         finally:
-            if self._connection:
-                self._connection.close()
-                self._connection = None
+            self._close_connection()
+
+    def _close_connection(self, invalidate: bool = False) -> None:
+        """Close the postgres connection.
+
+        Args:
+            invalidate: If True, invalidate connection instead of closing it.
+                Use this when the connection might be broken (e.g., after
+                pg_terminate_backend) to prevent SQLAlchemy from trying to
+                reset it (which would result in an error being logged).
+        """
+        if self._connection:
+            try:
+                if invalidate:
+                    self._connection.invalidate()
+                else:
+                    self._connection.close()
+            except Exception as e:  # pylint: disable=broad-except
+                if invalidate:
+                    logger.debug(
+                        f'Failed to invalidate postgres connection: {e}')
+                else:
+                    logger.debug(f'Failed to close postgres connection: {e}')
+            self._connection = None
 
     def is_locked(self) -> bool:
         """Check if the postgres advisory lock is acquired."""
@@ -278,7 +356,8 @@ class PostgresLock(DistributedLock):
 def get_lock(lock_id: str,
              timeout: Optional[float] = None,
              lock_type: Optional[str] = None,
-             poll_interval: Optional[float] = None) -> DistributedLock:
+             poll_interval: Optional[float] = None,
+             shared_lock: bool = False) -> DistributedLock:
     """Create a distributed lock instance.
 
     Args:
@@ -287,6 +366,9 @@ def get_lock(lock_id: str,
                  None means wait indefinitely.
         lock_type: Type of lock to create ('filelock' or 'postgres').
                    If None, auto-detect based on database configuration.
+        poll_interval: Interval in seconds to poll for lock acquisition.
+        shared_lock: Whether to use shared lock or exclusive lock (default).
+                     NOTE: Only applicable for PostgresLock.
 
     Returns:
         DistributedLock instance.
@@ -296,9 +378,24 @@ def get_lock(lock_id: str,
 
     if lock_type == 'postgres':
         if poll_interval is None:
-            return PostgresLock(lock_id, timeout)
-        return PostgresLock(lock_id, timeout, poll_interval)
+            return PostgresLock(lock_id, timeout, shared_lock=shared_lock)
+        return PostgresLock(lock_id,
+                            timeout,
+                            poll_interval,
+                            shared_lock=shared_lock)
     elif lock_type == 'filelock':
+        # The filelock library we use does not support shared locks.
+        # It explicitly uses fcntl.LOCK_EX on Unix systems,
+        # whereas fcntl.LOCK_SH is needed for shared locks.
+
+        # This should be fine as it should not introduce correctness issues,
+        # just that concurrency is reduced and so is performance, because
+        # read-only operations can't run at the same time, each of them need
+        # to wait to exclusively hold the lock.
+
+        # But given that we recommend users to use Postgres in production,
+        # the impact of this should be limited to local API server mostly.
+        del shared_lock
         if poll_interval is None:
             return FileLock(lock_id, timeout)
         return FileLock(lock_id, timeout, poll_interval)

sky/utils/log_utils.py

@@ -198,325 +198,6 @@ class SkyLocalUpLineProcessor(LineProcessor):
         self.status_display.stop()
 
 
-class SkyRemoteUpLineProcessor(LineProcessor):
-    """A processor for deploy_remote_cluster.py log lines."""
-
-    def __init__(self, log_path: str, is_local: bool):
-        self.log_path = log_path
-        self.is_local = is_local
-
-    def __enter__(self) -> None:
-        # TODO(romilb): Use ux_utils.INDENT_SYMBOL to be consistent with other
-        #  messages.
-        status = rich_utils.safe_status(
-            ux_utils.spinner_message('Creating remote cluster',
-                                     log_path=self.log_path,
-                                     is_local=self.is_local))
-        self.status_display = status
-        self.status_display.start()
-
-    def process_line(self, log_line: str) -> None:
-        # Pre-flight checks
-        if 'SSH connection successful' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}SSH connection established.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # Kubernetes installation steps
-        if 'Deploying Kubernetes on head node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Creating remote cluster - '
-                    'deploying Kubernetes on head node',
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-        if 'K3s deployed on head node.' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}'
-                        '✔ K3s successfully deployed on head node.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # Worker nodes
-        if 'Deploying Kubernetes on worker node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Creating remote cluster - '
-                    'deploying Kubernetes on worker nodes',
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-        if 'Kubernetes deployed on worker node' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}'
-                        '✔ K3s successfully deployed on worker node.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # Cluster configuration
-        if 'Configuring local kubectl to connect to the cluster...' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Creating remote cluster - '
-                    'configuring local kubectl',
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-        if 'kubectl configured to connect to the cluster.' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}'
-                        '✔ kubectl configured for the remote cluster.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # GPU operator installation
-        if 'Installing Nvidia GPU Operator...' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Creating remote cluster - '
-                    'installing Nvidia GPU Operator',
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-        if 'GPU Operator installed.' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}'
-                        '✔ Nvidia GPU Operator installed successfully.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # Cleanup steps
-        if 'Cleaning up head node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Cleaning up head node',
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-        if 'Cleaning up node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Cleaning up worker node',
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-        if 'cleaned up successfully' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}'
-                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
-
-        # Final status
-        if 'Cluster deployment completed.' in log_line:
-            logger.info(f'{colorama.Fore.GREEN}✔ Remote k3s is running.'
-                        f'{colorama.Style.RESET_ALL}')
-
-    def __exit__(self, except_type: Optional[Type[BaseException]],
-                 except_value: Optional[BaseException],
-                 traceback: Optional[types.TracebackType]) -> None:
-        del except_type, except_value, traceback  # unused
-        self.status_display.stop()
-
-
-class SkySSHUpLineProcessor(LineProcessor):
-    """A processor for deploy_remote_cluster.py log lines for SSH clusters"""
-
-    def __init__(self, log_path: str, is_local: bool):
-        self.log_path = log_path
-        self.is_local = is_local
-        self.current_cluster: Optional[str] = None
-        self.is_cleanup_mode = False
-
-    def __enter__(self) -> None:
-        status = rich_utils.safe_status(
-            ux_utils.spinner_message('Preparing to set up SSH Node Pools',
-                                     log_path=self.log_path,
-                                     is_local=self.is_local))
-        self.status_display = status
-        self.status_display.start()
-
-    def process_line(self, log_line: str) -> None:
-        # Detect cleanup mode
-        if 'SKYPILOT_CLEANUP_MODE:' in log_line:
-            self.is_cleanup_mode = True
-            if self.current_cluster:
-                self.status_display.update(
-                    ux_utils.spinner_message(
-                        f'Cleaning up Node Pool: \\[{self.current_cluster}]',
-                        log_path=self.log_path,
-                        is_local=self.is_local))
-
-        # Cluster detection message
-        if 'SKYPILOT_CLUSTER_INFO:' in log_line:
-            clusters_part = log_line.split('SKYPILOT_CLUSTER_INFO:',
-                                           1)[1].strip()
-            if clusters_part.startswith('Found'):
-                logger.info(f'{colorama.Style.RESET_ALL}'
-                            f'{colorama.Fore.CYAN}{clusters_part}'
-                            f'{colorama.Style.RESET_ALL}')
-
-        # Current cluster being operated on
-        if 'SKYPILOT_CURRENT_CLUSTER:' in log_line:
-            self.current_cluster = log_line.split('SKYPILOT_CURRENT_CLUSTER:',
-                                                  1)[1].strip()
-
-            if self.is_cleanup_mode:
-                self.status_display.update(
-                    ux_utils.spinner_message(
-                        f'Cleaning up Node Pool: {self.current_cluster}',
-                        log_path=self.log_path,
-                        is_local=self.is_local))
-                logger.info(f'{colorama.Fore.CYAN}\nCleaning up Node Pool: '
-                            f'{self.current_cluster}{colorama.Style.RESET_ALL}')
-            else:
-                self.status_display.update(
-                    ux_utils.spinner_message(
-                        f'Deploying SkyPilot \\[{self.current_cluster}]',
-                        log_path=self.log_path,
-                        is_local=self.is_local))
-                logger.info(f'{colorama.Style.RESET_ALL}'
-                            f'{colorama.Fore.CYAN}\nSetting up Node Pool: '
-                            f'{self.current_cluster}{colorama.Style.RESET_ALL}')
-
-        # Handle cluster completion marker
-        if 'SKYPILOT_CLUSTER_COMPLETED:' in log_line:
-            if self.is_cleanup_mode:
-                logger.info(
-                    f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Fore.GREEN}'
-                    f'✔ Node Pool {self.current_cluster} cleaned up '
-                    f'successfully.{colorama.Style.RESET_ALL}')
-            else:
-                logger.info(
-                    f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Fore.GREEN}'
-                    f'✔ Node Pool {self.current_cluster} deployed successfully.'
-                    f'{colorama.Style.RESET_ALL}')
-
-        # Pre-flight checks
-        if 'Checking SSH connection to head node' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Style.DIM}'
-                        'Checking SSH connection to head node...'
-                        f'{colorama.Style.RESET_ALL}')
-
-        if log_line.startswith('SSH connection successful'):
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                        '✔ SSH connection established to head node '
-                        f'{node_name}.{colorama.Style.RESET_ALL}')
-
-        # Kubernetes installation steps
-        if 'Deploying Kubernetes on head node' in log_line:
-            current_cluster_str = f' \\[{self.current_cluster}]' if (
-                self.current_cluster) else ''
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Deploying SkyPilot runtime on head node'
-                    f'{current_cluster_str}',
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-
-        if 'K3s deployed on head node' in log_line:
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(
-                f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                f'✔ SkyPilot runtime successfully deployed on head node '
-                f'{node_name}.{colorama.Style.RESET_ALL}')
-
-        # Worker nodes
-        if 'Deploying Kubernetes on worker node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message(
-                    'Deploying SkyPilot runtime on worker nodes' +
-                    (f' \\[{self.current_cluster}]'
-                     if self.current_cluster else ''),
-                    log_path=self.log_path,
-                    is_local=self.is_local))
-
-        if 'Kubernetes deployed on worker node' in log_line:
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(
-                f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                '✔ SkyPilot runtime successfully deployed on worker node '
-                f'{node_name}.{colorama.Style.RESET_ALL}')
-
-        if 'Failed to deploy K3s on worker node' in log_line:
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.RED}'
-                        '✗ Failed to deploy K3s on worker node '
-                        f'{node_name}.{colorama.Style.RESET_ALL}')
-
-        # Cluster configuration
-        if 'Configuring local kubectl to connect to the cluster...' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Setting up SkyPilot configuration' +
-                                         (f' \\[{self.current_cluster}]'
-                                          if self.current_cluster else ''),
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-
-        if 'kubectl configured to connect to the cluster.' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                        '✔ SkyPilot configuration complete.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # GPU operator installation
-        if 'Installing Nvidia GPU Operator...' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Configuring Nvidia GPUs' +
-                                         (f' \\[{self.current_cluster}]'
-                                          if self.current_cluster else ''),
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-
-        if 'GPU Operator installed.' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                        '✔ Nvidia GPUs configured successfully.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        # Cleanup steps
-        if 'Cleaning up head node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Cleaning up head node' +
-                                         (f' \\[{self.current_cluster}]'
-                                          if self.current_cluster else ''),
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-
-        if 'Cleaning up worker node' in log_line:
-            self.status_display.update(
-                ux_utils.spinner_message('Cleaning up worker nodes' +
-                                         (f' \\[{self.current_cluster}]'
-                                          if self.current_cluster else ''),
-                                         log_path=self.log_path,
-                                         is_local=self.is_local))
-
-        # Handle node cleanup success messages
-        if 'Node' in log_line and 'cleaned up successfully' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
-
-        if 'Node' in log_line and 'Failed to clean up' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.RED}'
-                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
-
-        if 'Failed to clean up worker node' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.RED}'
-                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
-
-        # Final status for the cluster deployment
-        if 'Cluster deployment completed.' in log_line:
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.GREEN}'
-                        '✔ SkyPilot runtime is up.'
-                        f'{colorama.Style.RESET_ALL}')
-
-        if 'Failed to deploy Kubernetes on the following nodes:' in log_line:
-            logger.info(log_line.strip())
-
-        if 'already exists in history. ' in log_line:
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.YELLOW}'
-                        '✔ SkyPilot runtime already deployed on worker node '
-                        f'{node_name}. Skipping.{colorama.Style.RESET_ALL}')
-
-        if 'Failed to setup TCP forwarding on head node' in log_line:
-            node_name = log_line.split('(')[-1].split(')')[0]
-            logger.info(
-                f'{ux_utils.INDENT_SYMBOL}{colorama.Fore.RED}'
-                f'✗ Failed to setup TCP forwarding on head node {node_name}.'
-                f'{colorama.Style.RESET_ALL}')
-
-        if 'Error in deploying SSH Target' in log_line:
-            logger.info(f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Fore.RED}'
-                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
-
-    def __exit__(self, except_type: Optional[Type[BaseException]],
-                 except_value: Optional[BaseException],
-                 traceback: Optional[types.TracebackType]) -> None:
-        del except_type, except_value, traceback  # unused
-        self.status_display.stop()
-
-
 def create_table(field_names: List[str], **kwargs) -> prettytable.PrettyTable:
     """Creates table with default style."""
     border = kwargs.pop('border', False)

sky/utils/resource_checker.py

@@ -140,7 +140,7 @@ def _check_active_resources(resource_operations: List[Tuple[str, str]],
 
 def check_users_workspaces_active_resources(
         user_ids: List[str],
-        workspace_names: List[str]) -> Tuple[str, List[str]]:
+        workspace_names: List[str]) -> Tuple[str, List[str], Dict[str, str]]:
     """Check if all the active clusters or managed jobs in workspaces
        belong to the user_ids. If not, return the error message.
 
@@ -151,6 +151,7 @@ def check_users_workspaces_active_resources(
     Returns:
         resource_error_summary: str
         missed_users_names: List[str]
+        missed_user_dict: Dict[str, str]
     """
     all_clusters, all_managed_jobs = _get_active_resources_for_workspaces(
         workspace_names)
@@ -187,14 +188,14 @@ def check_users_workspaces_active_resources(
     if resource_errors:
         resource_error_summary = ' and '.join(resource_errors)
     missed_users_names = []
+    missed_user_dict = {}
     if missed_users:
         all_users = global_user_state.get_all_users()
-        missed_users_names = [
-            user.name if user.name else user.id
-            for user in all_users
-            if user.id in missed_users
-        ]
-    return resource_error_summary, missed_users_names
+        for user in all_users:
+            if user.id in missed_users:
+                missed_users_names.append(user.name if user.name else user.id)
+                missed_user_dict[user.id] = user.name if user.name else user.id
+    return resource_error_summary, missed_users_names, missed_user_dict
 
 
 def _get_active_resources_for_workspaces(
@@ -276,9 +277,11 @@ def _get_active_resources(
         # pylint: disable=import-outside-toplevel
         from sky.jobs.server import core as managed_jobs_core
         try:
-            filtered_jobs, _, _, _ = managed_jobs_core.queue(refresh=False,
-                                                             skip_finished=True,
-                                                             all_users=True)
+            filtered_jobs, _, _, _ = managed_jobs_core.queue_v2(
+                refresh=False,
+                skip_finished=True,
+                all_users=True,
+                fields=['job_id', 'user_hash', 'workspace'])
             return filtered_jobs
         except exceptions.ClusterNotUpError:
             logger.warning('All jobs should be finished.')