skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -1,10 +1,11 @@
 """Utility functions for deploying Kubernetes clusters."""
 import os
+import random
 import shlex
 import subprocess
-import sys
 import tempfile
-from typing import List, Optional
+import textwrap
+from typing import List, Optional, Tuple
 
 import colorama
 
@@ -19,11 +20,16 @@ from sky.utils import log_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
+from sky.utils.kubernetes import deploy_ssh_node_pools
 
 logger = sky_logging.init_logger(__name__)
 
 # Default path for Kubernetes configuration file
 DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
+DEFAULT_LOCAL_CLUSTER_NAME = 'skypilot'
+LOCAL_CLUSTER_PORT_RANGE = 100
+LOCAL_CLUSTER_INTERNAL_PORT_START = 30000
+LOCAL_CLUSTER_INTERNAL_PORT_END = 30099
 
 
 def check_ssh_cluster_dependencies(
@@ -85,218 +91,178 @@ def deploy_ssh_cluster(cleanup: bool = False,
     """
     check_ssh_cluster_dependencies()
 
-    # Prepare command to call deploy_remote_cluster.py script
-    # TODO(romilb): We should move this to a native python method/class call
-    #  instead of invoking a script with subprocess.
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.py')
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
+    action = 'Cleanup' if cleanup else 'Deployment'
+    msg_str = f'Initializing SSH Node Pools {action}...'
 
-    deploy_command = [sys.executable, up_script_path]
+    with rich_utils.safe_status(ux_utils.spinner_message(msg_str)):
+        try:
+            deploy_ssh_node_pools.deploy_clusters(
+                infra=infra, cleanup=cleanup, kubeconfig_path=kubeconfig_path)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(str(e))
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(
+                    'Failed to deploy SkyPilot on some Node Pools.') from e
 
+    logger.info('')
     if cleanup:
-        deploy_command.append('--cleanup')
-
-    if infra:
-        deploy_command.extend(['--infra', infra])
+        logger.info(
+            ux_utils.finishing_message(
+                '🎉 SSH Node Pools cleaned up successfully.'))
+    else:
+        logger.info(
+            ux_utils.finishing_message(
+                '🎉 SSH Node Pools set up successfully. ',
+                follow_up_message=(
+                    f'Run `{colorama.Style.BRIGHT}'
+                    f'sky check ssh'
+                    f'{colorama.Style.RESET_ALL}` to verify access, '
+                    f'`{colorama.Style.BRIGHT}sky launch --infra ssh'
+                    f'{colorama.Style.RESET_ALL}` to launch a cluster.')))
 
-    # Use the default kubeconfig path if none is provided
-    kubeconfig_path = kubeconfig_path or DEFAULT_KUBECONFIG_PATH
-    deploy_command.extend(['--kubeconfig-path', kubeconfig_path])
 
-    # Setup logging paths
-    run_timestamp = sky_logging.get_run_timestamp()
-    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                            'ssh_up.log')
+def generate_kind_config(port_start: int,
+                         num_nodes: int = 1,
+                         gpus: bool = False) -> str:
+    """Generate a kind cluster config with ports mapped from host to container
 
-    if cleanup:
-        msg_str = 'Cleaning up SSH Node Pools...'
-    else:
-        msg_str = 'Initializing deployment to SSH Node Pools...'
+    Port range will be [port_start, port_start + LOCAL_CLUSTER_PORT_RANGE)
+    Internally, this will map to ports 30000 - 30099
 
-    # Create environment with PYTHONUNBUFFERED=1 to ensure unbuffered output
-    env = os.environ.copy()
-    env['PYTHONUNBUFFERED'] = '1'
+    Args:
+        path: Path to generate the config file at
+        port_start: Port range start for mappings
+        num_nodes: Number of nodes in the cluster
+        gpus: If true, initialize kind cluster with GPU support
 
-    with rich_utils.safe_status(
-            ux_utils.spinner_message(msg_str, log_path=log_path,
-                                     is_local=True)):
-        returncode, _, stderr = log_lib.run_with_log(
-            cmd=deploy_command,
-            log_path=log_path,
-            require_outputs=True,
-            stream_logs=False,
-            line_processor=log_utils.SkySSHUpLineProcessor(log_path=log_path,
-                                                           is_local=False),
-            cwd=cwd,
-            env=env)
+    Returns:
+        The kind cluster config
+    """
+    internal_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+    internal_end = LOCAL_CLUSTER_INTERNAL_PORT_END
+
+    config = textwrap.dedent(f"""
+    apiVersion: kind.x-k8s.io/v1alpha4
+    kind: Cluster
+    kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          "service-node-port-range": {internal_start}-{internal_end}
+    nodes:
+    - role: control-plane
+      kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    """)
+    if gpus:
+        config += textwrap.indent(
+            textwrap.dedent("""
+        extraMounts:
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
+    config += textwrap.indent(textwrap.dedent("""
+      extraPortMappings:"""), ' ' * 2)
+    for offset in range(LOCAL_CLUSTER_PORT_RANGE):
+        config += textwrap.indent(
+            textwrap.dedent(f"""
+        - containerPort: {internal_start + offset}
+          hostPort: {port_start + offset}
+          listenAddress: "0.0.0.0"
+          protocol: tcp
+        """), ' ' * 2)
+    if num_nodes > 1:
+        config += '- role: worker\n' * (num_nodes - 1)
+    return config
+
+
+def _get_port_range(name: str, port_start: Optional[int]) -> Tuple[int, int]:
+    is_default = name == DEFAULT_LOCAL_CLUSTER_NAME
+    if port_start is None:
+        if is_default:
+            port_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+        else:
+            port_start = random.randint(301, 399) * 100
+    port_end = port_start + LOCAL_CLUSTER_PORT_RANGE - 1
+
+    port_range = f'Current port range: {port_start}-{port_end}'
+    if is_default and port_start != LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Default local cluster `skypilot` should have '
+                         f'port range from 30000 to 30099. {port_range}.')
+    if not is_default and port_start == LOCAL_CLUSTER_INTERNAL_PORT_START:
+        raise ValueError('Port range 30000 to 30099 is reserved for '
+                         f'default local cluster `skypilot`. {port_range}.')
+    if port_start % 100 != 0:
+        raise ValueError('Local cluster port start must be a multiple of 100. '
+                         f'{port_range}.')
+
+    return port_start, port_end
+
+
+def deploy_local_cluster(name: Optional[str], port_start: Optional[int],
+                         gpus: bool):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    port_start, port_end = _get_port_range(name, port_start)
+    context_name = f'kind-{name}'
+    cluster_created = False
 
-    if returncode == 0:
-        success = True
-    else:
-        with ux_utils.print_exception_no_traceback():
-            log_hint = ux_utils.log_path_hint(log_path, is_local=False)
-            raise RuntimeError('Failed to deploy SkyPilot on some Node Pools. '
-                               f'{log_hint}'
-                               f'\nError: {stderr}')
+    # Check if GPUs are available on the host
+    local_gpus_available = backend_utils.check_local_gpus()
+    gpus = gpus and local_gpus_available
 
-    if success:
-        # Add an empty line to separate the deployment logs from the final
-        # message
-        logger.info('')
-        if cleanup:
-            logger.info(
-                ux_utils.finishing_message(
-                    '🎉 SSH Node Pools cleaned up successfully.',
-                    log_path=log_path,
-                    is_local=True))
-        else:
+    # Check if ~/.kube/config exists:
+    if os.path.exists(os.path.expanduser('~/.kube/config')):
+        curr_context = kubernetes_utils.get_current_kube_config_context_name()
+        if curr_context is not None and curr_context != context_name:
             logger.info(
-                ux_utils.finishing_message(
-                    '🎉 SSH Node Pools set up successfully. ',
-                    follow_up_message=(
-                        f'Run `{colorama.Style.BRIGHT}'
-                        f'sky check ssh'
-                        f'{colorama.Style.RESET_ALL}` to verify access, '
-                        f'`{colorama.Style.BRIGHT}sky launch --infra ssh'
-                        f'{colorama.Style.RESET_ALL}` to launch a cluster. '),
-                    log_path=log_path,
-                    is_local=True))
-
-
-def deploy_remote_cluster(ip_list: List[str],
-                          ssh_user: str,
-                          ssh_key: str,
-                          cleanup: bool,
-                          context_name: Optional[str] = None,
-                          password: Optional[str] = None):
-    success = False
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.py')
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-
-    # Create temporary files for the IPs and SSH key
-    with tempfile.NamedTemporaryFile(mode='w') as ip_file, \
-         tempfile.NamedTemporaryFile(mode='w') as key_file:
-
-        # Write IPs and SSH key to temporary files
-        ip_file.write('\n'.join(ip_list))
-        ip_file.flush()
-
-        key_file.write(ssh_key)
-        key_file.flush()
-        os.chmod(key_file.name, 0o600)
-
-        # Use the legacy mode command line arguments for backward compatibility
-        deploy_command = [
-            sys.executable, up_script_path, '--ips-file', ip_file.name,
-            '--user', ssh_user, '--ssh-key', key_file.name
-        ]
-
-        if context_name is not None:
-            deploy_command.extend(['--context-name', context_name])
-        if password is not None:
-            deploy_command.extend(['--password', password])
-        if cleanup:
-            deploy_command.append('--cleanup')
+                f'Current context in kube config: {curr_context}'
+                f'\nWill automatically switch to {context_name} after the '
+                'local cluster is created.')
+    message_str = 'Creating local cluster {}{}...'
+    message_str = message_str.format(
+        name,
+        ' with GPU support (this may take up to 15 minutes)' if gpus else '')
+
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.yaml',
+                                     delete=True) as f:
+        # Choose random port range to use on the host machine.
+        # Port range is port_start - port_start + 99 (exactly 100 ports).
+        logger.debug(f'Using host port range {port_start}-{port_end}')
+        f.write(generate_kind_config(port_start, gpus=gpus))
+        f.flush()
+
+        path_to_package = os.path.dirname(__file__)
+        up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
+
+        # Get directory of script and run it from there
+        cwd = os.path.dirname(os.path.abspath(up_script_path))
+        run_command = f'{up_script_path} {name} {f.name}'
+        if gpus:
+            run_command += ' --gpus'
+        run_command = shlex.split(run_command)
 
         # Setup logging paths
         run_timestamp = sky_logging.get_run_timestamp()
         log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
                                 'local_up.log')
-
-        if cleanup:
-            msg_str = 'Cleaning up remote cluster...'
-        else:
-            msg_str = 'Deploying remote cluster...'
-
-        # Create environment with PYTHONUNBUFFERED=1 to ensure unbuffered output
-        env = os.environ.copy()
-        env['PYTHONUNBUFFERED'] = '1'
+        logger.info(message_str)
 
         with rich_utils.safe_status(
-                ux_utils.spinner_message(msg_str,
+                ux_utils.spinner_message(message_str,
                                          log_path=log_path,
                                          is_local=True)):
             returncode, _, stderr = log_lib.run_with_log(
-                cmd=deploy_command,
+                cmd=run_command,
                 log_path=log_path,
                 require_outputs=True,
                 stream_logs=False,
-                line_processor=log_utils.SkyRemoteUpLineProcessor(
+                line_processor=log_utils.SkyLocalUpLineProcessor(
                     log_path=log_path, is_local=True),
-                cwd=cwd,
-                env=env)
-        if returncode == 0:
-            success = True
-        else:
-            with ux_utils.print_exception_no_traceback():
-                log_hint = ux_utils.log_path_hint(log_path, is_local=True)
-                raise RuntimeError('Failed to deploy remote cluster. '
-                                   f'Full log: {log_hint}'
-                                   f'\nError: {stderr}')
-
-        if success:
-            if cleanup:
-                logger.info(
-                    ux_utils.finishing_message(
-                        '🎉 Remote cluster cleaned up successfully.',
-                        log_path=log_path,
-                        is_local=True))
-            else:
-                logger.info(
-                    ux_utils.finishing_message(
-                        '🎉 Remote cluster deployed successfully.',
-                        log_path=log_path,
-                        is_local=True))
-
-
-def deploy_local_cluster(gpus: bool):
-    cluster_created = False
-
-    # Check if GPUs are available on the host
-    local_gpus_available = backend_utils.check_local_gpus()
-    gpus = gpus and local_gpus_available
-
-    # Check if ~/.kube/config exists:
-    if os.path.exists(os.path.expanduser('~/.kube/config')):
-        curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        skypilot_context = 'kind-skypilot'
-        if curr_context is not None and curr_context != skypilot_context:
-            logger.info(
-                f'Current context in kube config: {curr_context}'
-                '\nWill automatically switch to kind-skypilot after the local '
-                'cluster is created.')
-    message_str = 'Creating local cluster{}...'
-    message_str = message_str.format((' with GPU support (this may take up '
-                                      'to 15 minutes)') if gpus else '')
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
-
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-    run_command = up_script_path + ' --gpus' if gpus else up_script_path
-    run_command = shlex.split(run_command)
-
-    # Setup logging paths
-    run_timestamp = sky_logging.get_run_timestamp()
-    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                            'local_up.log')
-    logger.info(message_str)
-
-    with rich_utils.safe_status(
-            ux_utils.spinner_message(message_str,
-                                     log_path=log_path,
-                                     is_local=True)):
-        returncode, _, stderr = log_lib.run_with_log(
-            cmd=run_command,
-            log_path=log_path,
-            require_outputs=True,
-            stream_logs=False,
-            line_processor=log_utils.SkyLocalUpLineProcessor(log_path=log_path,
-                                                             is_local=True),
-            cwd=cwd)
+                cwd=cwd)
 
     # Kind always writes to stderr even if it succeeds.
     # If the failure happens after the cluster is created, we need
@@ -309,11 +275,11 @@ def deploy_local_cluster(gpus: bool):
     elif returncode == 100:
         logger.info(
             ux_utils.finishing_message(
-                'Local cluster already exists.\n',
+                f'Local cluster {name} already exists.\n',
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=
-                'If you want to delete it instead, run: sky local down'))
+                'If you want to delete it instead, run: `sky local down --name {name}`'))  # pylint: disable=line-too-long
     else:
         with ux_utils.print_exception_no_traceback():
             log_hint = ux_utils.log_path_hint(log_path, is_local=True)
@@ -339,7 +305,7 @@ def deploy_local_cluster(gpus: bool):
         if gpus:
             # Get GPU model by querying the node labels
             label_name_escaped = 'skypilot.co/accelerator'.replace('.', '\\.')
-            gpu_type_cmd = f'kubectl get node skypilot-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
+            gpu_type_cmd = f'kubectl get node {name}-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
             try:
                 # Run the command and capture the output
                 gpu_count_output = subprocess.check_output(gpu_type_cmd,
@@ -375,8 +341,10 @@ def deploy_local_cluster(gpus: bool):
                         'This may cause issues with running tasks.')
         logger.info(
             ux_utils.finishing_message(
-                message=(f'Local Kubernetes cluster created successfully with '
-                         f'{num_cpus} CPUs{gpu_message}.'),
+                message=(
+                    f'Local Kubernetes cluster {name} created successfully '
+                    f'with {num_cpus} CPUs{gpu_message} on host port range '
+                    f'{port_start}-{port_end}.'),
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=(
@@ -384,3 +352,54 @@ def deploy_local_cluster(gpus: bool):
                     'Hint: To change the number of CPUs, change your docker '
                     'runtime settings. See https://kind.sigs.k8s.io/docs/user/quick-start/#settings-for-docker-desktop for more info.'  # pylint: disable=line-too-long
                     f'{gpu_hint}')))
+
+
+def teardown_local_cluster(name: Optional[str] = None):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    cluster_removed = False
+
+    path_to_package = os.path.dirname(__file__)
+    down_script_path = os.path.join(path_to_package, 'delete_cluster.sh')
+
+    cwd = os.path.dirname(os.path.abspath(down_script_path))
+    run_command = f'{down_script_path} {name}'
+    run_command = shlex.split(run_command)
+
+    # Setup logging paths
+    run_timestamp = sky_logging.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_down.log')
+
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(f'Removing local cluster {name}',
+                                     log_path=log_path,
+                                     is_local=True)):
+
+        returncode, stdout, stderr = log_lib.run_with_log(cmd=run_command,
+                                                          log_path=log_path,
+                                                          require_outputs=True,
+                                                          stream_logs=False,
+                                                          cwd=cwd)
+        stderr = stderr.replace('No kind clusters found.\n', '')
+
+        if returncode == 0:
+            cluster_removed = True
+        elif returncode == 100:
+            logger.info(
+                ux_utils.error_message(f'Local cluster {name} does not exist.'))
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(f'Failed to down local cluster {name}. '
+                                   f'Stdout: {stdout}'
+                                   f'\nError: {stderr}')
+    if cluster_removed:
+        # Run sky check
+        with rich_utils.safe_status(
+                ux_utils.spinner_message('Running sky check...')):
+            sky_check.check_capability(sky_cloud.CloudCapability.COMPUTE,
+                                       clouds=['kubernetes'],
+                                       quiet=True)
+        logger.info(
+            ux_utils.finishing_message(f'Local cluster {name} removed.',
+                                       log_path=log_path,
+                                       is_local=True))

sky/utils/kubernetes/rsync_helper.sh

@@ -48,8 +48,16 @@ fi
 
 if [ -z "$context" ] || [ "$context_lower" = "none" ]; then
     # If context is none, it means we are using incluster auth. In this case,
-    # use need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
-    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --kubeconfig=/dev/null -- "$@"
+    # we need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --kubeconfig=/dev/null --"
 else
-    kubectl exec -i "$resource_type/$resource_name" -n "$namespace" --context="$context" -- "$@"
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --context=\"$context\" --"
 fi
+
+# Execute command on remote pod, waiting for rsync to be available first.
+# The waiting happens on the remote pod, not locally, which is more efficient
+# and reliable than polling from the local machine.
+# We wrap the command in a bash script that waits for rsync, then execs the original command.
+# Timeout after MAX_WAIT_TIME_SECONDS seconds.
+MAX_WAIT_TIME_SECONDS=300
+eval "${kubectl_cmd_base% --} -i -- bash -c 'count=0; max_count=$MAX_WAIT_TIME_SECONDS*2; until which rsync >/dev/null 2>&1; do if [ \$count -ge \$max_count ]; then echo \"Error when trying to rsync files to kubernetes cluster. Package installation may have failed.\" >&2; exit 1; fi; sleep 0.5; count=\$((count+1)); done; exec \"\$@\"' -- \"\$@\""

sky/utils/kubernetes_enums.py

@@ -2,26 +2,13 @@
 import enum
 
 
+# TODO(kevin): Remove this enum in v0.13.0.
 class KubernetesNetworkingMode(enum.Enum):
-    """Enum for the different types of networking modes for accessing
-    jump pods.
+    """Enum for the different types of networking modes for accessing pods.
     """
     NODEPORT = 'nodeport'
     PORTFORWARD = 'portforward'
 
-    @classmethod
-    def from_str(cls, mode: str) -> 'KubernetesNetworkingMode':
-        """Returns the enum value for the given string."""
-        if mode.lower() == cls.NODEPORT.value:
-            return cls.NODEPORT
-        elif mode.lower() == cls.PORTFORWARD.value:
-            return cls.PORTFORWARD
-        else:
-            raise ValueError(f'Unsupported kubernetes networking mode: '
-                             f'{mode}. The mode must be either '
-                             f'\'{cls.PORTFORWARD.value}\' or '
-                             f'\'{cls.NODEPORT.value}\'. ')
-
 
 class KubernetesServiceType(enum.Enum):
     """Enum for the different types of services."""
@@ -44,3 +31,8 @@ class KubernetesAutoscalerType(enum.Enum):
     KARPENTER = 'karpenter'
     COREWEAVE = 'coreweave'
     GENERIC = 'generic'
+
+    def emits_autoscale_event(self) -> bool:
+        """Returns whether specific autoscaler emits the event reason
+        TriggeredScaleUp."""
+        return self not in {self.KARPENTER}

sky/utils/lock_events.py

@@ -20,17 +20,17 @@ class DistributedLockEvent:
             f'[DistributedLock.hold]:{lock_id}')
 
     def acquire(self):
-        was_locked = self._lock.is_locked
+        was_locked = self._lock.is_locked  # type: ignore[truthy-function]
         with timeline.Event(f'[DistributedLock.acquire]:{self._lock_id}'):
             self._lock.acquire()
-        if not was_locked and self._lock.is_locked:
+        if not was_locked and self._lock.is_locked:  # type: ignore[truthy-function]  # pylint: disable=line-too-long
             # start holding the lock after initial acquiring
             self._hold_lock_event.begin()
 
     def release(self):
-        was_locked = self._lock.is_locked
+        was_locked = self._lock.is_locked  # type: ignore[truthy-function]
         self._lock.release()
-        if was_locked and not self._lock.is_locked:
+        if was_locked and not self._lock.is_locked:  # type: ignore[truthy-function]  # pylint: disable=line-too-long
             # stop holding the lock after initial releasing
             self._hold_lock_event.end()