skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -1,9 +1,13 @@
 """Utility functions for deploying Kubernetes clusters."""
 import os
+import random
 import shlex
 import subprocess
 import tempfile
-from typing import List, Optional
+import textwrap
+from typing import List, Optional, Tuple
+
+import colorama
 
 from sky import check as sky_check
 from sky import sky_logging
@@ -16,92 +20,194 @@ from sky.utils import log_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
+from sky.utils.kubernetes import deploy_ssh_node_pools
 
 logger = sky_logging.init_logger(__name__)
 
+# Default path for Kubernetes configuration file
+DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
+DEFAULT_LOCAL_CLUSTER_NAME = 'skypilot'
+LOCAL_CLUSTER_PORT_RANGE = 100
+LOCAL_CLUSTER_INTERNAL_PORT_START = 30000
+LOCAL_CLUSTER_INTERNAL_PORT_END = 30099
 
-def deploy_remote_cluster(ip_list: List[str],
-                          ssh_user: str,
-                          ssh_key: str,
-                          cleanup: bool,
-                          context_name: Optional[str] = None,
-                          password: Optional[str] = None):
-    success = False
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.sh')
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-
-    # Create temporary files for the IPs and SSH key
-    with tempfile.NamedTemporaryFile(mode='w') as ip_file, \
-         tempfile.NamedTemporaryFile(mode='w') as key_file:
-
-        # Write IPs and SSH key to temporary files
-        ip_file.write('\n'.join(ip_list))
-        ip_file.flush()
-
-        key_file.write(ssh_key)
-        key_file.flush()
-        os.chmod(key_file.name, 0o600)
-
-        deploy_command = (f'{up_script_path} {ip_file.name} '
-                          f'{ssh_user} {key_file.name}')
-        if context_name is not None:
-            deploy_command += f' {context_name}'
-        if password is not None:
-            deploy_command += f' --password {password}'
-        if cleanup:
-            deploy_command += ' --cleanup'
-
-        # Convert the command to a format suitable for subprocess
-        deploy_command = shlex.split(deploy_command)
 
-        # Setup logging paths
-        run_timestamp = sky_logging.get_run_timestamp()
-        log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                                'local_up.log')
+def check_ssh_cluster_dependencies(
+        raise_error: bool = True) -> Optional[List[str]]:
+    """Checks if the dependencies for ssh cluster are installed.
 
-        if cleanup:
-            msg_str = 'Cleaning up remote cluster...'
-        else:
-            msg_str = 'Deploying remote cluster...'
-        with rich_utils.safe_status(
-                ux_utils.spinner_message(msg_str,
-                                         log_path=log_path,
-                                         is_local=True)):
-            returncode, _, stderr = log_lib.run_with_log(
-                cmd=deploy_command,
-                log_path=log_path,
-                require_outputs=True,
-                stream_logs=False,
-                line_processor=log_utils.SkyRemoteUpLineProcessor(
-                    log_path=log_path, is_local=True),
-                cwd=cwd)
-        if returncode == 0:
-            success = True
-        else:
+    Args:
+        raise_error: set to true when the dependency needs to be present.
+            set to false for `sky check`, where reason strings are compiled
+            at the end.
+
+    Returns: the reasons list if there are missing dependencies.
+    """
+    # error message
+    jq_message = ('`jq` is required to setup ssh cluster.')
+
+    # save
+    reasons = []
+    required_binaries = []
+
+    # Ensure jq is installed
+    try:
+        subprocess.run(['jq', '--version'],
+                       stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL,
+                       check=True)
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        required_binaries.append('jq')
+        reasons.append(jq_message)
+
+    if required_binaries:
+        reasons.extend([
+            'On Debian/Ubuntu, install the missing dependenc(ies) with:',
+            f'  $ sudo apt install {" ".join(required_binaries)}',
+            'On MacOS, install with: ',
+            f'  $ brew install {" ".join(required_binaries)}',
+        ])
+        if raise_error:
             with ux_utils.print_exception_no_traceback():
-                log_hint = ux_utils.log_path_hint(log_path, is_local=True)
-                raise RuntimeError('Failed to deploy remote cluster. '
-                                   f'Full log: {log_hint}'
-                                   f'\nError: {stderr}')
+                raise RuntimeError('\n'.join(reasons))
+        return reasons
+    return None
+
+
+def deploy_ssh_cluster(cleanup: bool = False,
+                       infra: Optional[str] = None,
+                       kubeconfig_path: Optional[str] = None):
+    """Deploy a Kubernetes cluster on SSH targets.
+
+    This function reads ~/.sky/ssh_node_pools.yaml and uses it to deploy a
+    Kubernetes cluster on the specified machines.
+
+    Args:
+        cleanup: Whether to clean up the cluster instead of deploying.
+        infra: Name of the cluster in ssh_node_pools.yaml to use.
+            If None, the first cluster in the file will be used.
+        kubeconfig_path: Path to save the Kubernetes configuration file.
+            If None, the default ~/.kube/config will be used.
+    """
+    check_ssh_cluster_dependencies()
+
+    action = 'Cleanup' if cleanup else 'Deployment'
+    msg_str = f'Initializing SSH Node Pools {action}...'
+
+    with rich_utils.safe_status(ux_utils.spinner_message(msg_str)):
+        try:
+            deploy_ssh_node_pools.deploy_clusters(
+                infra=infra, cleanup=cleanup, kubeconfig_path=kubeconfig_path)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(str(e))
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(
+                    'Failed to deploy SkyPilot on some Node Pools.') from e
+
+    logger.info('')
+    if cleanup:
+        logger.info(
+            ux_utils.finishing_message(
+                '🎉 SSH Node Pools cleaned up successfully.'))
+    else:
+        logger.info(
+            ux_utils.finishing_message(
+                '🎉 SSH Node Pools set up successfully. ',
+                follow_up_message=(
+                    f'Run `{colorama.Style.BRIGHT}'
+                    f'sky check ssh'
+                    f'{colorama.Style.RESET_ALL}` to verify access, '
+                    f'`{colorama.Style.BRIGHT}sky launch --infra ssh'
+                    f'{colorama.Style.RESET_ALL}` to launch a cluster.')))
+
+
+def generate_kind_config(port_start: int,
+                         num_nodes: int = 1,
+                         gpus: bool = False) -> str:
+    """Generate a kind cluster config with ports mapped from host to container
+
+    Port range will be [port_start, port_start + LOCAL_CLUSTER_PORT_RANGE)
+    Internally, this will map to ports 30000 - 30099
+
+    Args:
+        path: Path to generate the config file at
+        port_start: Port range start for mappings
+        num_nodes: Number of nodes in the cluster
+        gpus: If true, initialize kind cluster with GPU support
+
+    Returns:
+        The kind cluster config
+    """
+    internal_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+    internal_end = LOCAL_CLUSTER_INTERNAL_PORT_END
+
+    config = textwrap.dedent(f"""
+    apiVersion: kind.x-k8s.io/v1alpha4
+    kind: Cluster
+    kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          "service-node-port-range": {internal_start}-{internal_end}
+    nodes:
+    - role: control-plane
+      kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    """)
+    if gpus:
+        config += textwrap.indent(
+            textwrap.dedent("""
+        extraMounts:
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
+    config += textwrap.indent(textwrap.dedent("""
+      extraPortMappings:"""), ' ' * 2)
+    for offset in range(LOCAL_CLUSTER_PORT_RANGE):
+        config += textwrap.indent(
+            textwrap.dedent(f"""
+        - containerPort: {internal_start + offset}
+          hostPort: {port_start + offset}
+          listenAddress: "0.0.0.0"
+          protocol: tcp
+        """), ' ' * 2)
+    if num_nodes > 1:
+        config += '- role: worker\n' * (num_nodes - 1)
+    return config
+
+
+def _get_port_range(name: str, port_start: Optional[int]) -> Tuple[int, int]:
+    is_default = name == DEFAULT_LOCAL_CLUSTER_NAME
+    if port_start is None:
+        if is_default:
+            port_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+        else:
+            port_start = random.randint(301, 399) * 100
+    port_end = port_start + LOCAL_CLUSTER_PORT_RANGE - 1
+
+    port_range = f'Current port range: {port_start}-{port_end}'
+    if is_default and port_start != LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Default local cluster `skypilot` should have '
+                         f'port range from 30000 to 30099. {port_range}.')
+    if not is_default and port_start == LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Port range 30000 to 30099 is reserved for '
+                         f'default local cluster `skypilot`. {port_range}.')
+    if port_start % 100 != 0:
+        raise ValueError('Local cluster port start must be a multiple of 100. '
+                         f'{port_range}.')
+
+    return port_start, port_end
 
-        if success:
-            if cleanup:
-                logger.info(
-                    ux_utils.finishing_message(
-                        '🎉 Remote cluster cleaned up successfully.',
-                        log_path=log_path,
-                        is_local=True))
-            else:
-                logger.info(
-                    ux_utils.finishing_message(
-                        '🎉 Remote cluster deployed successfully.',
-                        log_path=log_path,
-                        is_local=True))
-
-
-def deploy_local_cluster(gpus: bool):
+
+def deploy_local_cluster(name: Optional[str], port_start: Optional[int],
+                         gpus: bool):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    port_start, port_end = _get_port_range(name, port_start)
+    context_name = f'kind-{name}'
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -111,41 +217,52 @@ def deploy_local_cluster(gpus: bool):
     # Check if ~/.kube/config exists:
     if os.path.exists(os.path.expanduser('~/.kube/config')):
         curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        skypilot_context = 'kind-skypilot'
-        if curr_context is not None and curr_context != skypilot_context:
+        if curr_context is not None and curr_context != context_name:
             logger.info(
                 f'Current context in kube config: {curr_context}'
-                '\nWill automatically switch to kind-skypilot after the local '
-                'cluster is created.')
-    message_str = 'Creating local cluster{}...'
-    message_str = message_str.format((' with GPU support (this may take up '
-                                      'to 15 minutes)') if gpus else '')
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
+                f'\nWill automatically switch to {context_name} after the '
+                'local cluster is created.')
+    message_str = 'Creating local cluster {}{}...'
+    message_str = message_str.format(
+        name,
+        ' with GPU support (this may take up to 15 minutes)' if gpus else '')
 
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-    run_command = up_script_path + ' --gpus' if gpus else up_script_path
-    run_command = shlex.split(run_command)
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.yaml',
+                                     delete=True) as f:
+        # Choose random port range to use on the host machine.
+        # Port range is port_start - port_start + 99 (exactly 100 ports).
+        logger.debug(f'Using host port range {port_start}-{port_end}')
+        f.write(generate_kind_config(port_start, gpus=gpus))
+        f.flush()
 
-    # Setup logging paths
-    run_timestamp = sky_logging.get_run_timestamp()
-    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                            'local_up.log')
-    logger.info(message_str)
+        path_to_package = os.path.dirname(__file__)
+        up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
 
-    with rich_utils.safe_status(
-            ux_utils.spinner_message(message_str,
-                                     log_path=log_path,
-                                     is_local=True)):
-        returncode, _, stderr = log_lib.run_with_log(
-            cmd=run_command,
-            log_path=log_path,
-            require_outputs=True,
-            stream_logs=False,
-            line_processor=log_utils.SkyLocalUpLineProcessor(log_path=log_path,
-                                                             is_local=True),
-            cwd=cwd)
+        # Get directory of script and run it from there
+        cwd = os.path.dirname(os.path.abspath(up_script_path))
+        run_command = f'{up_script_path} {name} {f.name}'
+        if gpus:
+            run_command += ' --gpus'
+        run_command = shlex.split(run_command)
+
+        # Setup logging paths
+        run_timestamp = sky_logging.get_run_timestamp()
+        log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                                'local_up.log')
+        logger.info(message_str)
+
+        with rich_utils.safe_status(
+                ux_utils.spinner_message(message_str,
+                                         log_path=log_path,
+                                         is_local=True)):
+            returncode, _, stderr = log_lib.run_with_log(
+                cmd=run_command,
+                log_path=log_path,
+                require_outputs=True,
+                stream_logs=False,
+                line_processor=log_utils.SkyLocalUpLineProcessor(
+                    log_path=log_path, is_local=True),
+                cwd=cwd)
 
     # Kind always writes to stderr even if it succeeds.
     # If the failure happens after the cluster is created, we need
@@ -158,11 +275,11 @@ def deploy_local_cluster(gpus: bool):
     elif returncode == 100:
         logger.info(
             ux_utils.finishing_message(
-                'Local cluster already exists.\n',
+                f'Local cluster {name} already exists.\n',
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=
-                'If you want to delete it instead, run: sky local down'))
+                'If you want to delete it instead, run: `sky local down --name {name}`'))  # pylint: disable=line-too-long
     else:
         with ux_utils.print_exception_no_traceback():
             log_hint = ux_utils.log_path_hint(log_path, is_local=True)
@@ -188,7 +305,7 @@ def deploy_local_cluster(gpus: bool):
         if gpus:
             # Get GPU model by querying the node labels
             label_name_escaped = 'skypilot.co/accelerator'.replace('.', '\\.')
-            gpu_type_cmd = f'kubectl get node skypilot-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
+            gpu_type_cmd = f'kubectl get node {name}-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
             try:
                 # Run the command and capture the output
                 gpu_count_output = subprocess.check_output(gpu_type_cmd,
@@ -224,8 +341,10 @@ def deploy_local_cluster(gpus: bool):
                         'This may cause issues with running tasks.')
         logger.info(
             ux_utils.finishing_message(
-                message=(f'Local Kubernetes cluster created successfully with '
-                         f'{num_cpus} CPUs{gpu_message}.'),
+                message=(
+                    f'Local Kubernetes cluster {name} created successfully '
+                    f'with {num_cpus} CPUs{gpu_message} on host port range '
+                    f'{port_start}-{port_end}.'),
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=(
@@ -233,3 +352,54 @@ def deploy_local_cluster(gpus: bool):
                     'Hint: To change the number of CPUs, change your docker '
                     'runtime settings. See https://kind.sigs.k8s.io/docs/user/quick-start/#settings-for-docker-desktop for more info.'  # pylint: disable=line-too-long
                     f'{gpu_hint}')))
+
+
+def teardown_local_cluster(name: Optional[str] = None):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    cluster_removed = False
+
+    path_to_package = os.path.dirname(__file__)
+    down_script_path = os.path.join(path_to_package, 'delete_cluster.sh')
+
+    cwd = os.path.dirname(os.path.abspath(down_script_path))
+    run_command = f'{down_script_path} {name}'
+    run_command = shlex.split(run_command)
+
+    # Setup logging paths
+    run_timestamp = sky_logging.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_down.log')
+
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(f'Removing local cluster {name}',
+                                     log_path=log_path,
+                                     is_local=True)):
+
+        returncode, stdout, stderr = log_lib.run_with_log(cmd=run_command,
+                                                          log_path=log_path,
+                                                          require_outputs=True,
+                                                          stream_logs=False,
+                                                          cwd=cwd)
+        stderr = stderr.replace('No kind clusters found.\n', '')
+
+        if returncode == 0:
+            cluster_removed = True
+        elif returncode == 100:
+            logger.info(
+                ux_utils.error_message(f'Local cluster {name} does not exist.'))
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(f'Failed to down local cluster {name}. '
+                                   f'Stdout: {stdout}'
+                                   f'\nError: {stderr}')
+    if cluster_removed:
+        # Run sky check
+        with rich_utils.safe_status(
+                ux_utils.spinner_message('Running sky check...')):
+            sky_check.check_capability(sky_cloud.CloudCapability.COMPUTE,
+                                       clouds=['kubernetes'],
+                                       quiet=True)
+        logger.info(
+            ux_utils.finishing_message(f'Local cluster {name} removed.',
+                                       log_path=log_path,
+                                       is_local=True))

sky/utils/kubernetes/rsync_helper.sh

@@ -48,8 +48,16 @@ fi
 
 if [ -z "$context" ] || [ "$context_lower" = "none" ]; then
     # If context is none, it means we are using incluster auth. In this case,
-    # use need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
-    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --kubeconfig=/dev/null -- "$@"
+    # we need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --kubeconfig=/dev/null --"
 else
-    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --context="$context" -- "$@"
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --context=\"$context\" --"
 fi
+
+# Execute command on remote pod, waiting for rsync to be available first.
+# The waiting happens on the remote pod, not locally, which is more efficient
+# and reliable than polling from the local machine.
+# We wrap the command in a bash script that waits for rsync, then execs the original command.
+# Timeout after MAX_WAIT_TIME_SECONDS seconds.
+MAX_WAIT_TIME_SECONDS=300
+eval "${kubectl_cmd_base% --} -i -- bash -c 'count=0; max_count=$MAX_WAIT_TIME_SECONDS*2; until which rsync >/dev/null 2>&1; do if [ \$count -ge \$max_count ]; then echo \"Error when trying to rsync files to kubernetes cluster. Package installation may have failed.\" >&2; exit 1; fi; sleep 0.5; count=\$((count+1)); done; exec \"\$@\"' -- \"\$@\""