skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -1,9 +1,14 @@
 """Utility functions for deploying Kubernetes clusters."""
 import os
+import random
 import shlex
 import subprocess
+import sys
 import tempfile
-from typing import List, Optional
+import textwrap
+from typing import List, Optional, Tuple
+
+import colorama
 
 from sky import check as sky_check
 from sky import sky_logging
@@ -19,6 +24,151 @@ from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
 
+# Default path for Kubernetes configuration file
+DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
+DEFAULT_LOCAL_CLUSTER_NAME = 'skypilot'
+LOCAL_CLUSTER_PORT_RANGE = 100
+LOCAL_CLUSTER_INTERNAL_PORT_START = 30000
+LOCAL_CLUSTER_INTERNAL_PORT_END = 30099
+
+
+def check_ssh_cluster_dependencies(
+        raise_error: bool = True) -> Optional[List[str]]:
+    """Checks if the dependencies for ssh cluster are installed.
+
+    Args:
+        raise_error: set to true when the dependency needs to be present.
+            set to false for `sky check`, where reason strings are compiled
+            at the end.
+
+    Returns: the reasons list if there are missing dependencies.
+    """
+    # error message
+    jq_message = ('`jq` is required to setup ssh cluster.')
+
+    # save
+    reasons = []
+    required_binaries = []
+
+    # Ensure jq is installed
+    try:
+        subprocess.run(['jq', '--version'],
+                       stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL,
+                       check=True)
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        required_binaries.append('jq')
+        reasons.append(jq_message)
+
+    if required_binaries:
+        reasons.extend([
+            'On Debian/Ubuntu, install the missing dependenc(ies) with:',
+            f'  $ sudo apt install {" ".join(required_binaries)}',
+            'On MacOS, install with: ',
+            f'  $ brew install {" ".join(required_binaries)}',
+        ])
+        if raise_error:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError('\n'.join(reasons))
+        return reasons
+    return None
+
+
+def deploy_ssh_cluster(cleanup: bool = False,
+                       infra: Optional[str] = None,
+                       kubeconfig_path: Optional[str] = None):
+    """Deploy a Kubernetes cluster on SSH targets.
+
+    This function reads ~/.sky/ssh_node_pools.yaml and uses it to deploy a
+    Kubernetes cluster on the specified machines.
+
+    Args:
+        cleanup: Whether to clean up the cluster instead of deploying.
+        infra: Name of the cluster in ssh_node_pools.yaml to use.
+            If None, the first cluster in the file will be used.
+        kubeconfig_path: Path to save the Kubernetes configuration file.
+            If None, the default ~/.kube/config will be used.
+    """
+    check_ssh_cluster_dependencies()
+
+    # Prepare command to call deploy_remote_cluster.py script
+    # TODO(romilb): We should move this to a native python method/class call
+    #  instead of invoking a script with subprocess.
+    path_to_package = os.path.dirname(__file__)
+    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.py')
+    cwd = os.path.dirname(os.path.abspath(up_script_path))
+
+    deploy_command = [sys.executable, up_script_path]
+
+    if cleanup:
+        deploy_command.append('--cleanup')
+
+    if infra:
+        deploy_command.extend(['--infra', infra])
+
+    # Use the default kubeconfig path if none is provided
+    kubeconfig_path = kubeconfig_path or DEFAULT_KUBECONFIG_PATH
+    deploy_command.extend(['--kubeconfig-path', kubeconfig_path])
+
+    # Setup logging paths
+    run_timestamp = sky_logging.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'ssh_up.log')
+
+    if cleanup:
+        msg_str = 'Cleaning up SSH Node Pools...'
+    else:
+        msg_str = 'Initializing deployment to SSH Node Pools...'
+
+    # Create environment with PYTHONUNBUFFERED=1 to ensure unbuffered output
+    env = os.environ.copy()
+    env['PYTHONUNBUFFERED'] = '1'
+
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(msg_str, log_path=log_path,
+                                     is_local=True)):
+        returncode, _, stderr = log_lib.run_with_log(
+            cmd=deploy_command,
+            log_path=log_path,
+            require_outputs=True,
+            stream_logs=False,
+            line_processor=log_utils.SkySSHUpLineProcessor(log_path=log_path,
+                                                           is_local=False),
+            cwd=cwd,
+            env=env)
+
+    if returncode == 0:
+        success = True
+    else:
+        with ux_utils.print_exception_no_traceback():
+            log_hint = ux_utils.log_path_hint(log_path, is_local=False)
+            raise RuntimeError('Failed to deploy SkyPilot on some Node Pools. '
+                               f'{log_hint}'
+                               f'\nError: {stderr}')
+
+    if success:
+        # Add an empty line to separate the deployment logs from the final
+        # message
+        logger.info('')
+        if cleanup:
+            logger.info(
+                ux_utils.finishing_message(
+                    '🎉 SSH Node Pools cleaned up successfully.',
+                    log_path=log_path,
+                    is_local=True))
+        else:
+            logger.info(
+                ux_utils.finishing_message(
+                    '🎉 SSH Node Pools set up successfully. ',
+                    follow_up_message=(
+                        f'Run `{colorama.Style.BRIGHT}'
+                        f'sky check ssh'
+                        f'{colorama.Style.RESET_ALL}` to verify access, '
+                        f'`{colorama.Style.BRIGHT}sky launch --infra ssh'
+                        f'{colorama.Style.RESET_ALL}` to launch a cluster. '),
+                    log_path=log_path,
+                    is_local=True))
+
 
 def deploy_remote_cluster(ip_list: List[str],
                           ssh_user: str,
@@ -28,7 +178,7 @@ def deploy_remote_cluster(ip_list: List[str],
                           password: Optional[str] = None):
     success = False
     path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.sh')
+    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.py')
     # Get directory of script and run it from there
     cwd = os.path.dirname(os.path.abspath(up_script_path))
 
@@ -44,17 +194,18 @@ def deploy_remote_cluster(ip_list: List[str],
         key_file.flush()
         os.chmod(key_file.name, 0o600)
 
-        deploy_command = (f'{up_script_path} {ip_file.name} '
-                          f'{ssh_user} {key_file.name}')
+        # Use the legacy mode command line arguments for backward compatibility
+        deploy_command = [
+            sys.executable, up_script_path, '--ips-file', ip_file.name,
+            '--user', ssh_user, '--ssh-key', key_file.name
+        ]
+
         if context_name is not None:
-            deploy_command += f' {context_name}'
+            deploy_command.extend(['--context-name', context_name])
         if password is not None:
-            deploy_command += f' --password {password}'
+            deploy_command.extend(['--password', password])
         if cleanup:
-            deploy_command += ' --cleanup'
-
-        # Convert the command to a format suitable for subprocess
-        deploy_command = shlex.split(deploy_command)
+            deploy_command.append('--cleanup')
 
         # Setup logging paths
         run_timestamp = sky_logging.get_run_timestamp()
@@ -65,6 +216,11 @@ def deploy_remote_cluster(ip_list: List[str],
             msg_str = 'Cleaning up remote cluster...'
         else:
             msg_str = 'Deploying remote cluster...'
+
+        # Create environment with PYTHONUNBUFFERED=1 to ensure unbuffered output
+        env = os.environ.copy()
+        env['PYTHONUNBUFFERED'] = '1'
+
         with rich_utils.safe_status(
                 ux_utils.spinner_message(msg_str,
                                          log_path=log_path,
@@ -76,7 +232,8 @@ def deploy_remote_cluster(ip_list: List[str],
                 stream_logs=False,
                 line_processor=log_utils.SkyRemoteUpLineProcessor(
                     log_path=log_path, is_local=True),
-                cwd=cwd)
+                cwd=cwd,
+                env=env)
         if returncode == 0:
             success = True
         else:
@@ -101,7 +258,93 @@ def deploy_remote_cluster(ip_list: List[str],
                         is_local=True))
 
 
-def deploy_local_cluster(gpus: bool):
+def generate_kind_config(port_start: int,
+                         num_nodes: int = 1,
+                         gpus: bool = False) -> str:
+    """Generate a kind cluster config with ports mapped from host to container
+
+    Port range will be [port_start, port_start + LOCAL_CLUSTER_PORT_RANGE)
+    Internally, this will map to ports 30000 - 30099
+
+    Args:
+        path: Path to generate the config file at
+        port_start: Port range start for mappings
+        num_nodes: Number of nodes in the cluster
+        gpus: If true, initialize kind cluster with GPU support
+
+    Returns:
+        The kind cluster config
+    """
+    internal_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+    internal_end = LOCAL_CLUSTER_INTERNAL_PORT_END
+
+    config = textwrap.dedent(f"""
+    apiVersion: kind.x-k8s.io/v1alpha4
+    kind: Cluster
+    kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          "service-node-port-range": {internal_start}-{internal_end}
+    nodes:
+    - role: control-plane
+      kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    """)
+    if gpus:
+        config += textwrap.indent(
+            textwrap.dedent("""
+        extraMounts:
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
+    config += textwrap.indent(textwrap.dedent("""
+      extraPortMappings:"""), ' ' * 2)
+    for offset in range(LOCAL_CLUSTER_PORT_RANGE):
+        config += textwrap.indent(
+            textwrap.dedent(f"""
+        - containerPort: {internal_start + offset}
+          hostPort: {port_start + offset}
+          listenAddress: "0.0.0.0"
+          protocol: tcp
+        """), ' ' * 2)
+    if num_nodes > 1:
+        config += '- role: worker\n' * (num_nodes - 1)
+    return config
+
+
+def _get_port_range(name: str, port_start: Optional[int]) -> Tuple[int, int]:
+    is_default = name == DEFAULT_LOCAL_CLUSTER_NAME
+    if port_start is None:
+        if is_default:
+            port_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+        else:
+            port_start = random.randint(301, 399) * 100
+    port_end = port_start + LOCAL_CLUSTER_PORT_RANGE - 1
+
+    port_range = f'Current port range: {port_start}-{port_end}'
+    if is_default and port_start != LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Default local cluster `skypilot` should have '
+                         f'port range from 30000 to 30099. {port_range}.')
+    if not is_default and port_start == LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Port range 30000 to 30099 is reserved for '
+                         f'default local cluster `skypilot`. {port_range}.')
+    if port_start % 100 != 0:
+        raise ValueError('Local cluster port start must be a multiple of 100. '
+                         f'{port_range}.')
+
+    return port_start, port_end
+
+
+def deploy_local_cluster(name: Optional[str], port_start: Optional[int],
+                         gpus: bool):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    port_start, port_end = _get_port_range(name, port_start)
+    context_name = f'kind-{name}'
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -111,41 +354,52 @@ def deploy_local_cluster(gpus: bool):
     # Check if ~/.kube/config exists:
     if os.path.exists(os.path.expanduser('~/.kube/config')):
         curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        skypilot_context = 'kind-skypilot'
-        if curr_context is not None and curr_context != skypilot_context:
+        if curr_context is not None and curr_context != context_name:
             logger.info(
                 f'Current context in kube config: {curr_context}'
-                '\nWill automatically switch to kind-skypilot after the local '
-                'cluster is created.')
-    message_str = 'Creating local cluster{}...'
-    message_str = message_str.format((' with GPU support (this may take up '
-                                      'to 15 minutes)') if gpus else '')
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
+                f'\nWill automatically switch to {context_name} after the '
+                'local cluster is created.')
+    message_str = 'Creating local cluster {}{}...'
+    message_str = message_str.format(
+        name,
+        ' with GPU support (this may take up to 15 minutes)' if gpus else '')
 
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-    run_command = up_script_path + ' --gpus' if gpus else up_script_path
-    run_command = shlex.split(run_command)
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.yaml',
+                                     delete=True) as f:
+        # Choose random port range to use on the host machine.
+        # Port range is port_start - port_start + 99 (exactly 100 ports).
+        logger.debug(f'Using host port range {port_start}-{port_end}')
+        f.write(generate_kind_config(port_start, gpus=gpus))
+        f.flush()
 
-    # Setup logging paths
-    run_timestamp = sky_logging.get_run_timestamp()
-    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                            'local_up.log')
-    logger.info(message_str)
+        path_to_package = os.path.dirname(__file__)
+        up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
 
-    with rich_utils.safe_status(
-            ux_utils.spinner_message(message_str,
-                                     log_path=log_path,
-                                     is_local=True)):
-        returncode, _, stderr = log_lib.run_with_log(
-            cmd=run_command,
-            log_path=log_path,
-            require_outputs=True,
-            stream_logs=False,
-            line_processor=log_utils.SkyLocalUpLineProcessor(log_path=log_path,
-                                                             is_local=True),
-            cwd=cwd)
+        # Get directory of script and run it from there
+        cwd = os.path.dirname(os.path.abspath(up_script_path))
+        run_command = f'{up_script_path} {name} {f.name}'
+        if gpus:
+            run_command += ' --gpus'
+        run_command = shlex.split(run_command)
+
+        # Setup logging paths
+        run_timestamp = sky_logging.get_run_timestamp()
+        log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                                'local_up.log')
+        logger.info(message_str)
+
+        with rich_utils.safe_status(
+                ux_utils.spinner_message(message_str,
+                                         log_path=log_path,
+                                         is_local=True)):
+            returncode, _, stderr = log_lib.run_with_log(
+                cmd=run_command,
+                log_path=log_path,
+                require_outputs=True,
+                stream_logs=False,
+                line_processor=log_utils.SkyLocalUpLineProcessor(
+                    log_path=log_path, is_local=True),
+                cwd=cwd)
 
     # Kind always writes to stderr even if it succeeds.
     # If the failure happens after the cluster is created, we need
@@ -158,11 +412,11 @@ def deploy_local_cluster(gpus: bool):
     elif returncode == 100:
         logger.info(
             ux_utils.finishing_message(
-                'Local cluster already exists.\n',
+                f'Local cluster {name} already exists.\n',
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=
-                'If you want to delete it instead, run: sky local down'))
+                'If you want to delete it instead, run: `sky local down --name {name}`'))  # pylint: disable=line-too-long
     else:
         with ux_utils.print_exception_no_traceback():
             log_hint = ux_utils.log_path_hint(log_path, is_local=True)
@@ -188,7 +442,7 @@ def deploy_local_cluster(gpus: bool):
         if gpus:
             # Get GPU model by querying the node labels
             label_name_escaped = 'skypilot.co/accelerator'.replace('.', '\\.')
-            gpu_type_cmd = f'kubectl get node skypilot-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
+            gpu_type_cmd = f'kubectl get node {name}-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
             try:
                 # Run the command and capture the output
                 gpu_count_output = subprocess.check_output(gpu_type_cmd,
@@ -224,8 +478,10 @@ def deploy_local_cluster(gpus: bool):
                         'This may cause issues with running tasks.')
         logger.info(
             ux_utils.finishing_message(
-                message=(f'Local Kubernetes cluster created successfully with '
-                         f'{num_cpus} CPUs{gpu_message}.'),
+                message=(
+                    f'Local Kubernetes cluster {name} created successfully '
+                    f'with {num_cpus} CPUs{gpu_message} on host port range '
+                    f'{port_start}-{port_end}.'),
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=(
@@ -233,3 +489,54 @@ def deploy_local_cluster(gpus: bool):
                     'Hint: To change the number of CPUs, change your docker '
                     'runtime settings. See https://kind.sigs.k8s.io/docs/user/quick-start/#settings-for-docker-desktop for more info.'  # pylint: disable=line-too-long
                     f'{gpu_hint}')))
+
+
+def teardown_local_cluster(name: Optional[str] = None):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    cluster_removed = False
+
+    path_to_package = os.path.dirname(__file__)
+    down_script_path = os.path.join(path_to_package, 'delete_cluster.sh')
+
+    cwd = os.path.dirname(os.path.abspath(down_script_path))
+    run_command = f'{down_script_path} {name}'
+    run_command = shlex.split(run_command)
+
+    # Setup logging paths
+    run_timestamp = sky_logging.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_down.log')
+
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(f'Removing local cluster {name}',
+                                     log_path=log_path,
+                                     is_local=True)):
+
+        returncode, stdout, stderr = log_lib.run_with_log(cmd=run_command,
+                                                          log_path=log_path,
+                                                          require_outputs=True,
+                                                          stream_logs=False,
+                                                          cwd=cwd)
+        stderr = stderr.replace('No kind clusters found.\n', '')
+
+        if returncode == 0:
+            cluster_removed = True
+        elif returncode == 100:
+            logger.info(
+                ux_utils.error_message(f'Local cluster {name} does not exist.'))
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(f'Failed to down local cluster {name}. '
+                                   f'Stdout: {stdout}'
+                                   f'\nError: {stderr}')
+    if cluster_removed:
+        # Run sky check
+        with rich_utils.safe_status(
+                ux_utils.spinner_message('Running sky check...')):
+            sky_check.check_capability(sky_cloud.CloudCapability.COMPUTE,
+                                       clouds=['kubernetes'],
+                                       quiet=True)
+        logger.info(
+            ux_utils.finishing_message(f'Local cluster {name} removed.',
+                                       log_path=log_path,
+                                       is_local=True))