skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/utils/kubernetes/{deploy_remote_cluster.py → deploy_ssh_node_pools.py}

@@ -1,6 +1,5 @@
 """SSH-based Kubernetes Cluster Deployment Script"""
-# Refer to https://docs.skypilot.co/en/latest/reservations/existing-machines.html for details on how to use this script. # pylint: disable=line-too-long
-import argparse
+# pylint: disable=line-too-long
 import base64
 import concurrent.futures as cf
 import os
@@ -11,10 +10,13 @@ import shutil
 import subprocess
 import sys
 import tempfile
-from typing import List, Set
+from typing import List, Optional, Set
 
+import colorama
 import yaml
 
+from sky import sky_logging
+from sky.utils import rich_utils
 from sky.utils import ux_utils
 from sky.utils.kubernetes import ssh_utils
 
@@ -24,6 +26,9 @@ GREEN = '\033[0;32m'
 YELLOW = '\033[1;33m'
 WARNING_YELLOW = '\x1b[33m'
 NC = '\033[0m'  # No color
+DIM = colorama.Style.DIM
+CYAN = colorama.Fore.CYAN
+RESET_ALL = colorama.Style.RESET_ALL
 
 DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
 SSH_CONFIG_PATH = os.path.expanduser('~/.ssh/config')
@@ -32,69 +37,10 @@ NODE_POOLS_INFO_DIR = os.path.expanduser('~/.sky/ssh_node_pools_info')
 # Get the directory of this script
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
+logger = sky_logging.init_logger(__name__)
 
-def parse_args():
-    parser = argparse.ArgumentParser(
-        description='Deploy a Kubernetes cluster on remote machines.')
-    parser.add_argument(
-        '--infra', help='Name of the cluster in ssh_node_pools.yaml to use')
-    parser.add_argument(
-        '--ssh-node-pools-file',
-        dest='ssh_node_pools_file',
-        default=ssh_utils.DEFAULT_SSH_NODE_POOLS_PATH,
-        help=
-        f'Path to SSH node pools YAML file (default: {ssh_utils.DEFAULT_SSH_NODE_POOLS_PATH})'
-    )
-    parser.add_argument(
-        '--kubeconfig-path',
-        dest='kubeconfig_path',
-        default=DEFAULT_KUBECONFIG_PATH,
-        help=
-        f'Path to save the kubeconfig file (default: {DEFAULT_KUBECONFIG_PATH})'
-    )
-    parser.add_argument(
-        '--use-ssh-config',
-        dest='use_ssh_config',
-        action='store_true',
-        help='Use SSH config for host settings instead of explicit parameters')
-    #TODO(romilb): The `sky local up --ips` command is deprecated and these args are now captured in the ssh_node_pools.yaml file.
-    # Remove these args after 0.11.0 release.
-    parser.add_argument(
-        '--ips-file',
-        dest='ips_file',
-        help=
-        '[Deprecated, use --ssh-node-pools-file instead] File containing IP addresses or SSH host entries (one per line)'
-    )
-    parser.add_argument(
-        '--user',
-        help=
-        '[Deprecated, use --ssh-node-pools-file instead] Username to use for SSH (overridden by SSH config if host exists there)'
-    )
-    parser.add_argument(
-        '--ssh-key',
-        dest='ssh_key',
-        help=
-        '[Deprecated, use --ssh-node-pools-file instead] Path to SSH private key (overridden by SSH config if host exists there)'
-    )
-    parser.add_argument(
-        '--context-name',
-        dest='context_name',
-        default='default',
-        help=
-        '[Deprecated, use --ssh-node-pools-file instead] Kubernetes context name'
-    )
-    parser.add_argument('--cleanup',
-                        action='store_true',
-                        help='Clean up the cluster')
-    parser.add_argument(
-        '--password',
-        help='[Deprecated, use --ssh-node-pools-file instead] Password for sudo'
-    )
-
-    return parser.parse_args()
 
-
-def run_command(cmd, shell=False):
+def run_command(cmd, shell=False, silent=False):
     """Run a local command and return the output."""
     process = subprocess.run(cmd,
                              shell=shell,
@@ -102,9 +48,10 @@ def run_command(cmd, shell=False):
                              text=True,
                              check=False)
     if process.returncode != 0:
-        print(f'{RED}Error executing command: {cmd}{NC}')
-        print(f'STDOUT: {process.stdout}')
-        print(f'STDERR: {process.stderr}')
+        if not silent:
+            logger.error(f'{RED}Error executing command: {cmd}{NC}\n'
+                         f'STDOUT: {process.stdout}\n'
+                         f'STDERR: {process.stderr}')
         return None
     return process.stdout.strip()
 
@@ -132,8 +79,12 @@ def run_remote(node,
                connect_timeout=30,
                use_ssh_config=False,
                print_output=False,
-               use_shell=False):
-    """Run a command on a remote machine via SSH."""
+               use_shell=False,
+               silent=False):
+    """Run a command on a remote machine via SSH.
+
+    silent is used for gpu checking (will show error logs when no gpus are found)"""
+    ssh_cmd: List[str]
     if use_ssh_config:
         # Use SSH config for connection parameters
         ssh_cmd = ['ssh', node, cmd]
@@ -153,20 +104,19 @@ def run_remote(node,
         ssh_cmd.append(f'{user}@{node}' if user else node)
         ssh_cmd.append(cmd)
 
-    if use_shell:
-        ssh_cmd = ' '.join(ssh_cmd)
-
-    process = subprocess.run(ssh_cmd,
+    subprocess_cmd = ' '.join(ssh_cmd) if use_shell else ssh_cmd
+    process = subprocess.run(subprocess_cmd,
                              capture_output=True,
                              text=True,
                              check=False,
                              shell=use_shell)
     if process.returncode != 0:
-        print(f'{RED}Error executing command {cmd} on {node}:{NC}')
-        print(f'STDERR: {process.stderr}')
+        if not silent:
+            logger.error(f'{RED}Error executing command {cmd} on {node}:{NC} '
+                         f'{process.stderr}')
         return None
     if print_output:
-        print(process.stdout)
+        logger.info(process.stdout)
     return process.stdout.strip()
 
 
@@ -191,12 +141,17 @@ export SUDO_ASKPASS=$ASKPASS_SCRIPT
 
 def progress_message(message):
     """Show a progress message."""
-    print(f'{YELLOW}➜ {message}{NC}')
+    logger.info(f'{YELLOW}➜ {message}{NC}')
 
 
 def success_message(message):
     """Show a success message."""
-    print(f'{GREEN}✔ {message}{NC}')
+    logger.info(f'{GREEN}✔ {message}{NC}')
+
+
+def force_update_status(message):
+    """Force update rich spinner status."""
+    rich_utils.force_update_status(ux_utils.spinner_message(message))
 
 
 def cleanup_server_node(node,
@@ -205,7 +160,7 @@ def cleanup_server_node(node,
                         askpass_block,
                         use_ssh_config=False):
     """Uninstall k3s and clean up the state on a server node."""
-    print(f'{YELLOW}Cleaning up head node {node}...{NC}')
+    force_update_status(f'Cleaning up head node ({node})...')
     cmd = f"""
         {askpass_block}
         echo 'Uninstalling k3s...' &&
@@ -214,7 +169,7 @@ def cleanup_server_node(node,
     """
     result = run_remote(node, cmd, user, ssh_key, use_ssh_config=use_ssh_config)
     if result is None:
-        print(f'{RED}Failed to clean up head node ({node}).{NC}')
+        logger.error(f'{RED}Failed to clean up head node ({node}).{NC}')
     else:
         success_message(f'Node {node} cleaned up successfully.')
 
@@ -225,7 +180,7 @@ def cleanup_agent_node(node,
                        askpass_block,
                        use_ssh_config=False):
     """Uninstall k3s and clean up the state on an agent node."""
-    print(f'{YELLOW}Cleaning up worker node {node}...{NC}')
+    force_update_status(f'Cleaning up worker node ({node})...')
     cmd = f"""
         {askpass_block}
         echo 'Uninstalling k3s...' &&
@@ -234,7 +189,7 @@ def cleanup_agent_node(node,
     """
     result = run_remote(node, cmd, user, ssh_key, use_ssh_config=use_ssh_config)
     if result is None:
-        print(f'{RED}Failed to clean up worker node ({node}).{NC}')
+        logger.error(f'{RED}Failed to clean up worker node ({node}).{NC}')
     else:
         success_message(f'Node {node} cleaned up successfully.')
 
@@ -248,6 +203,7 @@ def start_agent_node(node,
                      use_ssh_config=False):
     """Start a k3s agent node.
     Returns: if the start is successful, and if the node has a GPU."""
+    logger.info(f'Deploying worker node ({node}).')
     cmd = f"""
             {askpass_block}
             curl -sfL https://get.k3s.io | K3S_NODE_NAME={node} INSTALL_K3S_EXEC='agent --node-label skypilot-ip={node}' \
@@ -255,12 +211,14 @@ def start_agent_node(node,
         """
     result = run_remote(node, cmd, user, ssh_key, use_ssh_config=use_ssh_config)
     if result is None:
-        print(f'{RED}Failed to deploy K3s on worker node ({node}).{NC}')
+        logger.error(
+            f'{RED}✗ Failed to deploy K3s on worker node ({node}).{NC}')
         return node, False, False
-    success_message(f'Kubernetes deployed on worker node ({node}).')
+    success_message(
+        f'SkyPilot runtime successfully deployed on worker node ({node}).')
     # Check if worker node has a GPU
     if check_gpu(node, user, ssh_key, use_ssh_config=use_ssh_config):
-        print(f'{YELLOW}GPU detected on worker node ({node}).{NC}')
+        logger.info(f'{YELLOW}GPU detected on worker node ({node}).{NC}')
         return node, True, True
     return node, True, False
 
@@ -268,7 +226,12 @@ def start_agent_node(node,
 def check_gpu(node, user, ssh_key, use_ssh_config=False):
     """Check if a node has a GPU."""
     cmd = 'command -v nvidia-smi &> /dev/null && nvidia-smi --query-gpu=gpu_name --format=csv,noheader &> /dev/null'
-    result = run_remote(node, cmd, user, ssh_key, use_ssh_config=use_ssh_config)
+    result = run_remote(node,
+                        cmd,
+                        user,
+                        ssh_key,
+                        use_ssh_config=use_ssh_config,
+                        silent=True)
     return result is not None
 
 
@@ -399,7 +362,7 @@ def setup_kubectl_ssh_tunnel(head_node,
     has_cert_files = os.path.isfile(client_cert_file) and os.path.isfile(
         client_key_file)
     if has_cert_files:
-        print(
+        logger.info(
             f'{GREEN}Client certificate data extracted and will be used for authentication{NC}'
         )
 
@@ -426,22 +389,22 @@ def setup_kubectl_ssh_tunnel(head_node,
     success_message(
         f'SSH tunnel configured through kubectl credential plugin on port {port}'
     )
-    print(
+    logger.info(
         f'{GREEN}Your kubectl connection is now tunneled through SSH (port {port}).{NC}'
     )
-    print(
+    logger.info(
         f'{GREEN}This tunnel will be automatically established when needed.{NC}'
     )
-    print(
+    logger.info(
         f'{GREEN}Credential TTL set to {ttl_seconds}s to ensure tunnel health is checked frequently.{NC}'
     )
 
     return port
 
 
-def cleanup_kubectl_ssh_tunnel(context_name):
+def cleanup_kubectl_ssh_tunnel(cluster_name, context_name):
     """Clean up the SSH tunnel for a specific context"""
-    progress_message(f'Cleaning up SSH tunnel for context {context_name}...')
+    progress_message(f'Cleaning up SSH tunnel for `{cluster_name}`...')
 
     # Path to cleanup script
     cleanup_script = os.path.join(SCRIPT_DIR, 'cleanup-tunnel.sh')
@@ -456,201 +419,148 @@ def cleanup_kubectl_ssh_tunnel(context_name):
                        stderr=subprocess.DEVNULL,
                        check=False)
 
-        success_message(f'SSH tunnel for context {context_name} cleaned up')
+        success_message(f'SSH tunnel for `{cluster_name}` cleaned up.')
     else:
-        print(f'{YELLOW}Cleanup script not found: {cleanup_script}{NC}')
+        logger.error(f'{YELLOW}Cleanup script not found: {cleanup_script}{NC}')
 
 
-def main():
-    args = parse_args()
+def deploy_clusters(
+        infra: Optional[str],
+        ssh_node_pools_file: str = ssh_utils.DEFAULT_SSH_NODE_POOLS_PATH,
+        kubeconfig_path: Optional[str] = None,
+        cleanup: bool = True):
 
-    kubeconfig_path = os.path.expanduser(args.kubeconfig_path)
-    global_use_ssh_config = args.use_ssh_config
+    kubeconfig_path = kubeconfig_path or DEFAULT_KUBECONFIG_PATH
+    kubeconfig_path = os.path.expanduser(kubeconfig_path)
 
     failed_clusters = []
     successful_clusters = []
 
-    # Print cleanup mode marker if applicable
-    if args.cleanup:
-        print('SKYPILOT_CLEANUP_MODE: Cleanup mode activated')
-
-    # Check if using YAML configuration or command line arguments
-    if args.ips_file:
-        # Using command line arguments - legacy mode
-        if args.ssh_key and not os.path.isfile(
-                args.ssh_key) and not global_use_ssh_config:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(f'SSH key not found: {args.ssh_key}')
-
-        if not os.path.isfile(args.ips_file):
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(f'IPs file not found: {args.ips_file}')
-
-        with open(args.ips_file, 'r', encoding='utf-8') as f:
-            hosts = [line.strip() for line in f if line.strip()]
-
-        if not hosts:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(
-                    'Hosts file is empty or not formatted correctly.')
-
-        head_node = hosts[0]
-        worker_nodes = hosts[1:]
-        ssh_user = args.user if not global_use_ssh_config else ''
-        ssh_key = args.ssh_key if not global_use_ssh_config else ''
-        context_name = args.context_name
-        password = args.password
-
-        # Check if hosts are in SSH config
-        head_use_ssh_config = global_use_ssh_config or ssh_utils.check_host_in_ssh_config(
-            head_node)
-        worker_use_ssh_config = [
-            global_use_ssh_config or ssh_utils.check_host_in_ssh_config(node)
-            for node in worker_nodes
-        ]
+    # Using YAML configuration
+    targets = ssh_utils.load_ssh_targets(ssh_node_pools_file)
+    clusters_config = ssh_utils.get_cluster_config(
+        targets, infra, file_path=ssh_node_pools_file)
+
+    # Print information about clusters being processed
+    num_clusters = len(clusters_config)
+    cluster_names = list(clusters_config.keys())
+    cluster_info = f'Found {num_clusters} Node Pool{"s" if num_clusters > 1 else ""}: {", ".join(cluster_names)}'
+    logger.info(f'{colorama.Fore.CYAN}{cluster_info}{colorama.Style.RESET_ALL}')
+
+    # Process each cluster
+    for cluster_name, cluster_config in clusters_config.items():
+        try:
+            action = 'Cleaning up' if cleanup else 'Deploying'
+            force_update_status(f'{action} Node Pool: {cluster_name}')
+            hosts_info = ssh_utils.prepare_hosts_info(cluster_name,
+                                                      cluster_config)
+
+            if not hosts_info:
+                logger.warning(
+                    f'{RED}Error: No valid hosts found for cluster {cluster_name!r}. Skipping.{NC}'
+                )
+                continue
 
-        # Single cluster deployment for legacy mode
-        deploy_cluster(head_node, worker_nodes, ssh_user, ssh_key, context_name,
-                       password, head_use_ssh_config, worker_use_ssh_config,
-                       kubeconfig_path, args.cleanup)
-    else:
-        # Using YAML configuration
-        targets = ssh_utils.load_ssh_targets(args.ssh_node_pools_file)
-        clusters_config = ssh_utils.get_cluster_config(
-            targets, args.infra, file_path=args.ssh_node_pools_file)
-
-        # Print information about clusters being processed
-        num_clusters = len(clusters_config)
-        cluster_names = list(clusters_config.keys())
-        cluster_info = f'Found {num_clusters} Node Pool{"s" if num_clusters > 1 else ""}: {", ".join(cluster_names)}'
-        print(f'SKYPILOT_CLUSTER_INFO: {cluster_info}')
-
-        # Process each cluster
-        for cluster_name, cluster_config in clusters_config.items():
-            try:
-                print(f'SKYPILOT_CURRENT_CLUSTER: {cluster_name}')
-                print(
-                    f'{YELLOW}==== Deploying cluster: {cluster_name} ====${NC}')
-                hosts_info = ssh_utils.prepare_hosts_info(
-                    cluster_name, cluster_config)
-
-                if not hosts_info:
-                    print(
-                        f'{RED}Error: No valid hosts found for cluster {cluster_name!r}. Skipping.{NC}'
-                    )
-                    continue
-
-                # Generate a unique context name for each cluster
-                context_name = args.context_name
-                if context_name == 'default':
-                    context_name = 'ssh-' + cluster_name
-
-                # Check cluster history
-                os.makedirs(NODE_POOLS_INFO_DIR, exist_ok=True)
-                history_yaml_file = os.path.join(
-                    NODE_POOLS_INFO_DIR, f'{context_name}-history.yaml')
-
-                history = None
-                if os.path.exists(history_yaml_file):
-                    print(
-                        f'{YELLOW}Loading history from {history_yaml_file}{NC}')
-                    with open(history_yaml_file, 'r', encoding='utf-8') as f:
-                        history = yaml.safe_load(f)
-                else:
-                    print(f'{YELLOW}No history found for {context_name}.{NC}')
-
-                history_workers_info = None
-                history_worker_nodes = None
-                history_use_ssh_config = None
-                # Do not support changing anything besides hosts for now
-                if history is not None:
-                    for key in ['user', 'identity_file', 'password']:
-                        if not args.cleanup and history.get(
-                                key) != cluster_config.get(key):
-                            raise ValueError(
-                                f'Cluster configuration has changed for field {key!r}. '
-                                f'Previous value: {history.get(key)}, '
-                                f'Current value: {cluster_config.get(key)}')
-                    history_hosts_info = ssh_utils.prepare_hosts_info(
-                        cluster_name, history)
-                    if not args.cleanup and history_hosts_info[0] != hosts_info[
-                            0]:
+            context_name = f'ssh-{cluster_name}'
+
+            # Check cluster history
+            os.makedirs(NODE_POOLS_INFO_DIR, exist_ok=True)
+            history_yaml_file = os.path.join(NODE_POOLS_INFO_DIR,
+                                             f'{context_name}-history.yaml')
+
+            history = None
+            if os.path.exists(history_yaml_file):
+                logger.debug(f'Loading history from {history_yaml_file}')
+                with open(history_yaml_file, 'r', encoding='utf-8') as f:
+                    history = yaml.safe_load(f)
+            else:
+                logger.debug(f'No history found for {context_name}.')
+
+            history_workers_info = None
+            history_worker_nodes = None
+            history_use_ssh_config = None
+            # Do not support changing anything besides hosts for now
+            if history is not None:
+                for key in ['user', 'identity_file', 'password']:
+                    if not cleanup and history.get(key) != cluster_config.get(
+                            key):
                         raise ValueError(
-                            f'Cluster configuration has changed for master node. '
-                            f'Previous value: {history_hosts_info[0]}, '
-                            f'Current value: {hosts_info[0]}')
-                    history_workers_info = history_hosts_info[1:] if len(
-                        history_hosts_info) > 1 else []
-                    history_worker_nodes = [
-                        h['ip'] for h in history_workers_info
-                    ]
-                    history_use_ssh_config = [
-                        h.get('use_ssh_config', False)
-                        for h in history_workers_info
-                    ]
-
-                # Use the first host as the head node and the rest as worker nodes
-                head_host = hosts_info[0]
-                worker_hosts = hosts_info[1:] if len(hosts_info) > 1 else []
-
-                head_node = head_host['ip']
-                worker_nodes = [h['ip'] for h in worker_hosts]
-                ssh_user = head_host['user']
-                ssh_key = head_host['identity_file']
-                head_use_ssh_config = global_use_ssh_config or head_host.get(
-                    'use_ssh_config', False)
-                worker_use_ssh_config = [
-                    global_use_ssh_config or h.get('use_ssh_config', False)
-                    for h in worker_hosts
+                            f'Cluster configuration has changed for field {key!r}. '
+                            f'Previous value: {history.get(key)}, '
+                            f'Current value: {cluster_config.get(key)}')
+                history_hosts_info = ssh_utils.prepare_hosts_info(
+                    cluster_name, history)
+                if not cleanup and history_hosts_info[0] != hosts_info[0]:
+                    raise ValueError(
+                        f'Cluster configuration has changed for master node. '
+                        f'Previous value: {history_hosts_info[0]}, '
+                        f'Current value: {hosts_info[0]}')
+                history_workers_info = history_hosts_info[1:] if len(
+                    history_hosts_info) > 1 else []
+                history_worker_nodes = [h['ip'] for h in history_workers_info]
+                history_use_ssh_config = [
+                    h.get('use_ssh_config', False) for h in history_workers_info
                 ]
-                password = head_host['password']
-
-                # Deploy this cluster
-                unsuccessful_workers = deploy_cluster(
-                    head_node,
-                    worker_nodes,
-                    ssh_user,
-                    ssh_key,
-                    context_name,
-                    password,
-                    head_use_ssh_config,
-                    worker_use_ssh_config,
-                    kubeconfig_path,
-                    args.cleanup,
-                    worker_hosts=worker_hosts,
-                    history_worker_nodes=history_worker_nodes,
-                    history_workers_info=history_workers_info,
-                    history_use_ssh_config=history_use_ssh_config)
-
-                if not args.cleanup:
-                    successful_hosts = []
-                    for host in cluster_config['hosts']:
-                        if isinstance(host, str):
-                            host_node = host
-                        else:
-                            host_node = host['ip']
-                        if host_node not in unsuccessful_workers:
-                            successful_hosts.append(host)
-                    cluster_config['hosts'] = successful_hosts
-                    with open(history_yaml_file, 'w', encoding='utf-8') as f:
-                        print(
-                            f'{YELLOW}Writing history to {history_yaml_file}{NC}'
-                        )
-                        yaml.dump(cluster_config, f)
-
-                print(
-                    f'{GREEN}==== Completed deployment for cluster: {cluster_name} ====${NC}'
-                )
-                successful_clusters.append(cluster_name)
-            except Exception as e:  # pylint: disable=broad-except
-                reason = str(e)
-                failed_clusters.append((cluster_name, reason))
-                print(
-                    f'{RED}Error deploying SSH Node Pool {cluster_name}: {reason}{NC}'
-                )  # Print for internal logging
+
+            # Use the first host as the head node and the rest as worker nodes
+            head_host = hosts_info[0]
+            worker_hosts = hosts_info[1:] if len(hosts_info) > 1 else []
+
+            head_node = head_host['ip']
+            worker_nodes = [h['ip'] for h in worker_hosts]
+            ssh_user = head_host['user']
+            ssh_key = head_host['identity_file']
+            head_use_ssh_config = head_host.get('use_ssh_config', False)
+            worker_use_ssh_config = [
+                h.get('use_ssh_config', False) for h in worker_hosts
+            ]
+            password = head_host['password']
+
+            # Deploy this cluster
+            unsuccessful_workers = deploy_cluster(
+                cluster_name,
+                head_node,
+                worker_nodes,
+                ssh_user,
+                ssh_key,
+                context_name,
+                password,
+                head_use_ssh_config,
+                worker_use_ssh_config,
+                kubeconfig_path,
+                cleanup,
+                worker_hosts=worker_hosts,
+                history_worker_nodes=history_worker_nodes,
+                history_workers_info=history_workers_info,
+                history_use_ssh_config=history_use_ssh_config)
+
+            if not cleanup:
+                successful_hosts = []
+                for host in cluster_config['hosts']:
+                    if isinstance(host, str):
+                        host_node = host
+                    else:
+                        host_node = host['ip']
+                    if host_node not in unsuccessful_workers:
+                        successful_hosts.append(host)
+                cluster_config['hosts'] = successful_hosts
+                with open(history_yaml_file, 'w', encoding='utf-8') as f:
+                    logger.debug(f'Writing history to {history_yaml_file}')
+                    yaml.dump(cluster_config, f)
+
+            action = 'cleanup' if cleanup else 'deployment'
+            logger.info(
+                f'{colorama.Fore.CYAN}Completed {action} for cluster: {cluster_name}{colorama.Style.RESET_ALL}'
+            )
+            successful_clusters.append(cluster_name)
+        except Exception as e:  # pylint: disable=broad-except
+            reason = str(e)
+            failed_clusters.append((cluster_name, reason))
+            logger.debug(
+                f'Error deploying SSH Node Pool `{cluster_name}`: {reason}')
 
     if failed_clusters:
-        action = 'clean' if args.cleanup else 'deploy'
+        action = 'clean' if cleanup else 'deploy'
         msg = f'{GREEN}Successfully {action}ed {len(successful_clusters)} cluster(s) ({", ".join(successful_clusters)}). {NC}'
         msg += f'{RED}Failed to {action} {len(failed_clusters)} cluster(s): {NC}'
         for cluster_name, reason in failed_clusters:
@@ -658,7 +568,8 @@ def main():
         raise RuntimeError(msg)
 
 
-def deploy_cluster(head_node,
+def deploy_cluster(cluster_name,
+                   head_node,
                    worker_nodes,
                    ssh_user,
                    ssh_key,
@@ -691,15 +602,15 @@ def deploy_cluster(head_node,
     k3s_token = 'mytoken'  # Any string can be used as the token
 
     # Pre-flight checks
-    print(f'{YELLOW}Checking SSH connection to head node...{NC}')
-    result = run_remote(
-        head_node,
-        f'echo \'SSH connection successful ({head_node})\'',
-        ssh_user,
-        ssh_key,
-        use_ssh_config=head_use_ssh_config,
-        # For SkySSHUpLineProcessor
-        print_output=True)
+    logger.info(f'Checking SSH connection to head node ({head_node})...')
+    result = run_remote(head_node,
+                        f'echo \'SSH connection successful ({head_node})\'',
+                        ssh_user,
+                        ssh_key,
+                        use_ssh_config=head_use_ssh_config)
+    if result.startswith('SSH connection successful'):
+        success_message(f'SSH connection established to head node {head_node}.')
+
     if not cleanup and result is None:
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(
@@ -720,9 +631,9 @@ def deploy_cluster(head_node,
                 history_worker_nodes, history_workers_info,
                 history_use_ssh_config):
             if worker_hosts is not None and history_info not in worker_hosts:
-                print(
-                    f'{YELLOW}Worker node {history_node} not found in YAML config. '
-                    f'Removing from history...{NC}')
+                logger.debug(
+                    f'Worker node {history_node} not found in YAML config. '
+                    'Removing from history...')
                 worker_nodes_to_cleanup.append(
                     dict(
                         node=history_node,
@@ -758,8 +669,6 @@ def deploy_cluster(head_node,
                     use_ssh_config=use_ssh_config,
                 ))
 
-        print(f'{YELLOW}Starting cleanup...{NC}')
-
         # Clean up head node
         cleanup_server_node(head_node,
                             ssh_user,
@@ -767,23 +676,20 @@ def deploy_cluster(head_node,
                             askpass_block,
                             use_ssh_config=head_use_ssh_config)
     # Clean up worker nodes
+    force_update_status(f'Cleaning up worker nodes [{cluster_name}]')
     with cf.ThreadPoolExecutor() as executor:
         executor.map(lambda kwargs: cleanup_agent_node(**kwargs),
                      worker_nodes_to_cleanup)
 
     with cf.ThreadPoolExecutor() as executor:
-
-        def run_cleanup_cmd(cmd):
-            print('Cleaning up worker nodes:', cmd)
-            run_command(cmd, shell=True)
-
-        executor.map(run_cleanup_cmd, remove_worker_cmds)
+        executor.map(lambda cmd: run_command(cmd, shell=True),
+                     remove_worker_cmds)
 
     if cleanup:
 
         # Remove the context from local kubeconfig if it exists
         if os.path.isfile(kubeconfig_path):
-            progress_message(
+            logger.debug(
                 f'Removing context {context_name!r} from local kubeconfig...')
             run_command(['kubectl', 'config', 'delete-context', context_name],
                         shell=False)
@@ -806,7 +712,7 @@ def deploy_cluster(head_node,
                 run_command(['kubectl', 'config', 'unset', 'current-context'],
                             shell=False)
 
-            success_message(
+            logger.debug(
                 f'Context {context_name!r} removed from local kubeconfig.')
 
         for file in [history_yaml_file, cert_file_path, key_file_path]:
@@ -815,16 +721,12 @@ def deploy_cluster(head_node,
 
         # Clean up SSH tunnel after clean up kubeconfig, because the kubectl
         # will restart the ssh tunnel if it's not running.
-        cleanup_kubectl_ssh_tunnel(context_name)
-
-        print(f'{GREEN}Cleanup completed successfully.{NC}')
-
-        # Print completion marker for current cluster
-        print(f'{GREEN}SKYPILOT_CLUSTER_COMPLETED: {NC}')
+        cleanup_kubectl_ssh_tunnel(cluster_name, context_name)
 
+        success_message(f'Node Pool `{cluster_name}` cleaned up successfully.')
         return []
 
-    print(f'{YELLOW}Checking TCP Forwarding Options...{NC}')
+    logger.debug('Checking TCP Forwarding Options...')
     cmd = (
         'if [ "$(sudo sshd -T | grep allowtcpforwarding)" = "allowtcpforwarding yes" ]; then '
         f'echo "TCP Forwarding already enabled on head node ({head_node})."; '
@@ -833,15 +735,12 @@ def deploy_cluster(head_node,
         '/etc/ssh/sshd_config && sudo systemctl restart sshd && '
         f'echo "Successfully enabled TCP Forwarding on head node ({head_node})."; '
         'fi')
-    result = run_remote(
-        head_node,
-        shlex.quote(cmd),
-        ssh_user,
-        ssh_key,
-        use_ssh_config=head_use_ssh_config,
-        # For SkySSHUpLineProcessor
-        print_output=True,
-        use_shell=True)
+    result = run_remote(head_node,
+                        shlex.quote(cmd),
+                        ssh_user,
+                        ssh_key,
+                        use_ssh_config=head_use_ssh_config,
+                        use_shell=True)
     if result is None:
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(
@@ -851,7 +750,7 @@ def deploy_cluster(head_node,
     # Get effective IP for master node if using SSH config - needed for workers to connect
     if head_use_ssh_config:
         effective_master_ip = get_effective_host_ip(head_node)
-        print(
+        logger.info(
             f'{GREEN}Resolved head node {head_node} to {effective_master_ip} from SSH config{NC}'
         )
     else:
@@ -860,7 +759,8 @@ def deploy_cluster(head_node,
     # Step 1: Install k3s on the head node
     # Check if head node has a GPU
     install_gpu = False
-    progress_message(f'Deploying Kubernetes on head node ({head_node})...')
+    force_update_status(
+        f'Deploying SkyPilot runtime on head node ({head_node}).')
     cmd = f"""
         {askpass_block}
         curl -sfL https://get.k3s.io | K3S_TOKEN={k3s_token} K3S_NODE_NAME={head_node} sudo -E -A sh - &&
@@ -889,7 +789,8 @@ def deploy_cluster(head_node,
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(
                 f'Failed to deploy K3s on head node ({head_node}).')
-    success_message(f'K3s deployed on head node ({head_node}).')
+    success_message(
+        f'SkyPilot runtime successfully deployed on head node ({head_node}).')
 
     # Check if head node has a GPU
     install_gpu = False
@@ -897,7 +798,7 @@ def deploy_cluster(head_node,
                  ssh_user,
                  ssh_key,
                  use_ssh_config=head_use_ssh_config):
-        print(f'{YELLOW}GPU detected on head node ({head_node}).{NC}')
+        logger.info(f'{YELLOW}GPU detected on head node ({head_node}).{NC}')
         install_gpu = True
 
     # Fetch the head node's internal IP (this will be passed to worker nodes)
@@ -910,21 +811,20 @@ def deploy_cluster(head_node,
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(f'Failed to SSH to head node ({head_node}). '
                                f'Please check the SSH configuration.')
-    print(f'{GREEN}Master node internal IP: {master_addr}{NC}')
+    logger.debug(f'Master node internal IP: {master_addr}')
 
     # Step 2: Install k3s on worker nodes and join them to the master node
     def deploy_worker(args):
         (i, node, worker_hosts, history_workers_info, ssh_user, ssh_key,
          askpass_block, worker_use_ssh_config, master_addr, k3s_token) = args
-        progress_message(f'Deploying Kubernetes on worker node ({node})...')
 
         # If using YAML config with specific worker info
         if worker_hosts and i < len(worker_hosts):
             if history_workers_info is not None and worker_hosts[
                     i] in history_workers_info:
-                print(
-                    f'{YELLOW}Worker node ({node}) already exists in history. '
-                    f'Skipping...{NC}')
+                logger.info(
+                    f'{colorama.Style.DIM}✔ SkyPilot runtime already deployed on worker node {node}. '
+                    f'Skipping...{colorama.Style.RESET_ALL}')
                 return node, True, False
             worker_user = worker_hosts[i]['user']
             worker_key = worker_hosts[i]['identity_file']
@@ -948,6 +848,8 @@ def deploy_cluster(head_node,
     unsuccessful_workers = []
 
     # Deploy workers in parallel using thread pool
+    force_update_status(
+        f'Deploying SkyPilot runtime on worker nodes [{cluster_name}]')
     with cf.ThreadPoolExecutor() as executor:
         futures = []
         for i, node in enumerate(worker_nodes):
@@ -964,7 +866,7 @@ def deploy_cluster(head_node,
                 unsuccessful_workers.append(node)
 
     # Step 3: Configure local kubectl to connect to the cluster
-    progress_message('Configuring local kubectl to connect to the cluster...')
+    force_update_status(f'Setting up SkyPilot configuration [{cluster_name}]')
 
     # Create temporary directory for kubeconfig operations
     with tempfile.TemporaryDirectory() as temp_dir:
@@ -1054,8 +956,8 @@ def deploy_cluster(head_node,
                         has_end = '-----END CERTIFICATE-----' in cert_pem
 
                         if not has_begin or not has_end:
-                            print(
-                                f'{YELLOW}Warning: Certificate data missing PEM markers, attempting to fix...{NC}'
+                            logger.debug(
+                                'Warning: Certificate data missing PEM markers, attempting to fix...'
                             )
                             # Add PEM markers if missing
                             if not has_begin:
@@ -1070,8 +972,8 @@ def deploy_cluster(head_node,
 
                         # Verify the file was written correctly
                         if os.path.getsize(cert_file_path) > 0:
-                            print(
-                                f'{GREEN}Successfully saved certificate data ({len(cert_pem)} bytes){NC}'
+                            logger.debug(
+                                f'Successfully saved certificate data ({len(cert_pem)} bytes)'
                             )
 
                             # Quick validation of PEM format
@@ -1086,13 +988,14 @@ def deploy_cluster(head_node,
                             if not first_line.startswith(
                                     '-----BEGIN') or not last_line.startswith(
                                         '-----END'):
-                                print(
-                                    f'{YELLOW}Warning: Certificate may not be in proper PEM format{NC}'
+                                logger.debug(
+                                    'Warning: Certificate may not be in proper PEM format'
                                 )
                         else:
-                            print(f'{RED}Error: Certificate file is empty{NC}')
+                            logger.error(
+                                f'{RED}Error: Certificate file is empty{NC}')
                     except Exception as e:  # pylint: disable=broad-except
-                        print(
+                        logger.error(
                             f'{RED}Error processing certificate data: {e}{NC}')
 
                 if client_key_data:
@@ -1134,8 +1037,8 @@ def deploy_cluster(head_node,
                             ])
 
                             if not has_begin or not has_end:
-                                print(
-                                    f'{YELLOW}Warning: Key data missing PEM markers, attempting to fix...{NC}'
+                                logger.debug(
+                                    'Warning: Key data missing PEM markers, attempting to fix...'
                                 )
                                 # Add PEM markers if missing
                                 if not has_begin:
@@ -1154,8 +1057,8 @@ def deploy_cluster(head_node,
 
                         # Verify the file was written correctly
                         if os.path.getsize(key_file_path) > 0:
-                            print(
-                                f'{GREEN}Successfully saved key data ({len(key_pem)} bytes){NC}'
+                            logger.debug(
+                                f'Successfully saved key data ({len(key_pem)} bytes)'
                             )
 
                             # Quick validation of PEM format
@@ -1170,22 +1073,25 @@ def deploy_cluster(head_node,
                             if not first_line.startswith(
                                     '-----BEGIN') or not last_line.startswith(
                                         '-----END'):
-                                print(
-                                    f'{YELLOW}Warning: Key may not be in proper PEM format{NC}'
+                                logger.debug(
+                                    'Warning: Key may not be in proper PEM format'
                                 )
                         else:
-                            print(f'{RED}Error: Key file is empty{NC}')
+                            logger.error(f'{RED}Error: Key file is empty{NC}')
                     except Exception as e:  # pylint: disable=broad-except
-                        print(f'{RED}Error processing key data: {e}{NC}')
+                        logger.error(f'{RED}Error processing key data: {e}{NC}')
 
         # First check if context name exists and delete it if it does
         # TODO(romilb): Should we throw an error here instead?
         run_command(['kubectl', 'config', 'delete-context', context_name],
-                    shell=False)
+                    shell=False,
+                    silent=True)
         run_command(['kubectl', 'config', 'delete-cluster', context_name],
-                    shell=False)
+                    shell=False,
+                    silent=True)
         run_command(['kubectl', 'config', 'delete-user', context_name],
-                    shell=False)
+                    shell=False,
+                    silent=True)
 
         # Merge the configurations using kubectl
         merged_config = os.path.join(temp_dir, 'merged_config')
@@ -1210,17 +1116,12 @@ def deploy_cluster(head_node,
                              context_name,
                              use_ssh_config=head_use_ssh_config)
 
-    success_message(f'kubectl configured with new context \'{context_name}\'.')
-
-    print(
-        f'Cluster deployment completed. Kubeconfig saved to {kubeconfig_path}')
-    print('You can now run \'kubectl get nodes\' to verify the setup.')
+    logger.debug(f'kubectl configured with new context \'{context_name}\'.')
+    success_message(f'SkyPilot runtime is up [{cluster_name}].')
 
     # Install GPU operator if a GPU was detected on any node
     if install_gpu:
-        print(
-            f'{YELLOW}GPU detected in the cluster. Installing Nvidia GPU Operator...{NC}'
-        )
+        force_update_status(f'Configuring NVIDIA GPUs [{cluster_name}]')
         cmd = f"""
             {askpass_block}
             curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 &&
@@ -1240,7 +1141,7 @@ def deploy_cluster(head_node,
             while ! kubectl describe nodes --kubeconfig ~/.kube/config | grep -q 'nvidia.com/gpu:' || ! kubectl describe nodes --kubeconfig ~/.kube/config | grep -q 'nvidia.com/gpu.product'; do
                 echo 'Waiting for GPU operator...'
                 sleep 5
-            done 
+            done
             echo 'GPU operator installed successfully.'
         """
         result = run_remote(head_node,
@@ -1249,51 +1150,28 @@ def deploy_cluster(head_node,
                             ssh_key,
                             use_ssh_config=head_use_ssh_config)
         if result is None:
-            print(f'{RED}Failed to install GPU Operator.{NC}')
+            logger.error(f'{RED}Failed to install GPU Operator.{NC}')
         else:
             success_message('GPU Operator installed.')
     else:
-        print(
-            f'{YELLOW}No GPUs detected. Skipping GPU Operator installation.{NC}'
-        )
-
-    # Configure SkyPilot
-    progress_message('Configuring SkyPilot...')
+        logger.debug('No GPUs detected. Skipping GPU Operator installation.')
 
     # The env var KUBECONFIG ensures sky check uses the right kubeconfig
     os.environ['KUBECONFIG'] = kubeconfig_path
-    run_command(['sky', 'check', 'kubernetes'], shell=False)
+    run_command(['sky', 'check', 'ssh'], shell=False)
 
     success_message('SkyPilot configured successfully.')
 
-    # Display final success message
-    print(
-        f'{GREEN}==== 🎉 Kubernetes cluster deployment completed successfully 🎉 ====${NC}'
-    )
-    print(
-        'You can now interact with your Kubernetes cluster through SkyPilot: ')
-    print('  • List available GPUs: sky show-gpus --cloud kubernetes')
-    print(
-        '  • Launch a GPU development pod: sky launch -c devbox --cloud kubernetes'
-    )
-    print(
-        '  • Connect to pod with VSCode: code --remote ssh-remote+devbox "/home"'
-    )
-    # Print completion marker for current cluster
-    print(f'{GREEN}SKYPILOT_CLUSTER_COMPLETED: {NC}')
-
     if unsuccessful_workers:
         quoted_unsuccessful_workers = [
             f'"{worker}"' for worker in unsuccessful_workers
         ]
 
-        print(
+        logger.info(
             f'{WARNING_YELLOW}Failed to deploy Kubernetes on the following nodes: '
             f'{", ".join(quoted_unsuccessful_workers)}. Please check '
             f'the logs for more details.{NC}')
+    else:
+        success_message(f'Node Pool `{cluster_name}` deployed successfully.')
 
     return unsuccessful_workers
-
-
-if __name__ == '__main__':
-    main()