skypilot-nightly 1.0.0.dev20240927__py3-none-any.whl → 1.0.0.dev20240928__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'e6b8d2c086544ab5cfdb877ad414eafddaa49cb4'
+_SKYPILOT_COMMIT_SHA = 'dacf27348ae1446c3c93d0ee2fc57702c5366eac'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240927'
+__version__ = '1.0.0.dev20240928'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/cli.py

@@ -5072,15 +5072,7 @@ def local():
     pass
 
 
-@click.option('--gpus/--no-gpus',
-              default=True,
-              is_flag=True,
-              help='Launch cluster without GPU support even '
-              'if GPUs are detected on the host.')
-@local.command('up', cls=_DocumentedCodeCommand)
-@usage_lib.entrypoint
-def local_up(gpus: bool):
-    """Creates a local cluster."""
+def _deploy_local_cluster(gpus: bool):
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -5206,6 +5198,124 @@ def local_up(gpus: bool):
             f'{gpu_hint}')
 
 
+def _deploy_remote_cluster(ip_file: str, ssh_user: str, ssh_key_path: str,
+                           cleanup: bool):
+    success = False
+    path_to_package = os.path.dirname(os.path.dirname(__file__))
+    up_script_path = os.path.join(path_to_package, 'sky/utils/kubernetes',
+                                  'deploy_remote_cluster.sh')
+    # Get directory of script and run it from there
+    cwd = os.path.dirname(os.path.abspath(up_script_path))
+
+    deploy_command = f'{up_script_path} {ip_file} {ssh_user} {ssh_key_path}'
+    if cleanup:
+        deploy_command += ' --cleanup'
+
+    # Convert the command to a format suitable for subprocess
+    deploy_command = shlex.split(deploy_command)
+
+    # Setup logging paths
+    run_timestamp = backend_utils.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_up.log')
+    tail_cmd = 'tail -n100 -f ' + log_path
+
+    # Check if ~/.kube/config exists:
+    if os.path.exists(os.path.expanduser('~/.kube/config')):
+        click.echo('Found existing kube config. '
+                   'It will be backed up to ~/.kube/config.bak.')
+    style = colorama.Style
+    click.echo('To view detailed progress: '
+               f'{style.BRIGHT}{tail_cmd}{style.RESET_ALL}')
+    if cleanup:
+        msg_str = 'Cleaning up remote cluster...'
+    else:
+        msg_str = 'Deploying remote cluster...'
+    with rich_utils.safe_status(f'[bold cyan]{msg_str}'):
+        returncode, _, stderr = log_lib.run_with_log(
+            cmd=deploy_command,
+            log_path=log_path,
+            require_outputs=True,
+            stream_logs=False,
+            line_processor=log_utils.SkyRemoteUpLineProcessor(),
+            cwd=cwd)
+    if returncode == 0:
+        success = True
+    else:
+        with ux_utils.print_exception_no_traceback():
+            raise RuntimeError(
+                'Failed to deploy remote cluster. '
+                f'Full log: {log_path}'
+                f'\nError: {style.BRIGHT}{stderr}{style.RESET_ALL}')
+
+    if success:
+        if cleanup:
+            click.echo(f'{colorama.Fore.GREEN}'
+                       '🎉 Remote cluster cleaned up successfully.'
+                       f'{style.RESET_ALL}')
+        else:
+            click.echo('Cluster deployment done. You can now run tasks on '
+                       'this cluster.\nE.g., run a task with: '
+                       'sky launch --cloud kubernetes -- echo hello world.'
+                       f'\n{colorama.Fore.GREEN}🎉 Remote cluster deployed '
+                       f'successfully. {style.RESET_ALL}')
+
+
+@click.option('--gpus/--no-gpus',
+              default=True,
+              is_flag=True,
+              help='Launch cluster without GPU support even '
+              'if GPUs are detected on the host.')
+@click.option(
+    '--ips',
+    type=str,
+    required=False,
+    help='Path to the file containing IP addresses of remote machines.')
+@click.option('--ssh-user',
+              type=str,
+              required=False,
+              help='SSH username for accessing remote machines.')
+@click.option('--ssh-key-path',
+              type=str,
+              required=False,
+              help='Path to the SSH private key.')
+@click.option('--cleanup',
+              is_flag=True,
+              help='Clean up the remote cluster instead of deploying it.')
+@local.command('up', cls=_DocumentedCodeCommand)
+@usage_lib.entrypoint
+def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
+             cleanup: bool):
+    """Creates a local or remote cluster."""
+
+    def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
+        # If any of --ips, --ssh-user, or --ssh-key-path is specified,
+        # all must be specified
+        if bool(ips) or bool(ssh_user) or bool(ssh_key_path):
+            if not (ips and ssh_user and ssh_key_path):
+                raise click.BadParameter(
+                    'All --ips, --ssh-user, and --ssh-key-path '
+                    'must be specified together.')
+
+        # --cleanup can only be used if --ips, --ssh-user and --ssh-key-path
+        # are all provided
+        if cleanup and not (ips and ssh_user and ssh_key_path):
+            raise click.BadParameter('--cleanup can only be used with '
+                                     '--ips, --ssh-user and --ssh-key-path.')
+
+    _validate_args(ips, ssh_user, ssh_key_path, cleanup)
+
+    # If remote deployment arguments are specified, run remote up script
+    if ips and ssh_user and ssh_key_path:
+        # Convert ips and ssh_key_path to absolute paths
+        ips = os.path.abspath(ips)
+        ssh_key_path = os.path.abspath(ssh_key_path)
+        _deploy_remote_cluster(ips, ssh_user, ssh_key_path, cleanup)
+    else:
+        # Run local deployment (kind) if no remote args are specified
+        _deploy_local_cluster(gpus)
+
+
 @local.command('down', cls=_DocumentedCodeCommand)
 @usage_lib.entrypoint
 def local_down():

sky/utils/kubernetes/deploy_remote_cluster.sh

@@ -0,0 +1,243 @@
+#!/bin/bash
+# Refer to https://skypilot.readthedocs.io/en/latest/reservations/existing-machines.html for details on how to use this script.
+set -e
+
+# Colors for nicer UX
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No color
+
+# Variables
+IPS_FILE=$1
+USER=$2
+SSH_KEY=$3
+K3S_TOKEN=mytoken  # Any string can be used as the token
+CLEANUP=false
+INSTALL_GPU=false
+
+if [[ "$4" == "--cleanup" ]]; then
+    CLEANUP=true
+fi
+
+# Basic argument checks
+if [ -z "$IPS_FILE" ] || [ -z "$USER" ] || [ -z "$SSH_KEY" ]; then
+    >&2 echo -e "${RED}Error: Missing required arguments.${NC}"
+    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [--cleanup]"
+    exit 1
+fi
+
+# Check if SSH key exists
+if [ ! -f "$SSH_KEY" ]; then
+    >&2 echo -e "${RED}Error: SSH key not found: $SSH_KEY${NC}"
+    exit 1
+fi
+
+# Check if IPs file exists
+if [ ! -f "$IPS_FILE" ]; then
+    >&2 echo -e "${RED}Error: IPs file not found: $IPS_FILE${NC}"
+    exit 1
+fi
+
+# Get head node and worker nodes from the IPs file
+HEAD_NODE=$(head -n 1 "$IPS_FILE")
+WORKER_NODES=$(tail -n +2 "$IPS_FILE")
+
+# Check if the IPs file is empty or not formatted correctly
+if [ -z "$HEAD_NODE" ]; then
+    >&2 echo -e "${RED}Error: IPs file is empty or not formatted correctly.${NC}"
+    exit 1
+fi
+
+# Function to show a progress message
+progress_message() {
+    echo -e "${YELLOW}➜ $1${NC}"
+}
+
+# Step to display success
+success_message() {
+    echo -e "${GREEN}✔ $1${NC}"
+}
+
+# Function to run a command on a remote machine via SSH
+run_remote() {
+    local NODE_IP=$1
+    local CMD=$2
+    # echo -e "${YELLOW}Running command on $NODE_IP...${NC}"
+    ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" "$USER@$NODE_IP" "$CMD"
+}
+
+# Function to uninstall k3s and clean up the state on a remote machine
+cleanup_server_node() {
+    local NODE_IP=$1
+    echo -e "${YELLOW}Cleaning up head node $NODE_IP...${NC}"
+    run_remote "$NODE_IP" "
+        echo 'Uninstalling k3s...' &&
+        /usr/local/bin/k3s-uninstall.sh || true &&
+        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+    "
+    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
+}
+
+# Function to uninstall k3s and clean up the state on a remote machine
+cleanup_agent_node() {
+    local NODE_IP=$1
+    echo -e "${YELLOW}Cleaning up node $NODE_IP...${NC}"
+    run_remote "$NODE_IP" "
+        echo 'Uninstalling k3s...' &&
+        /usr/local/bin/k3s-agent-uninstall.sh || true &&
+        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+    "
+    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
+}
+
+check_gpu() {
+    local NODE_IP=$1
+    run_remote "$NODE_IP" "
+        if command -v nvidia-smi &> /dev/null; then
+            nvidia-smi --list-gpus | grep 'GPU 0'
+        fi
+    "
+}
+
+# Pre-flight checks
+run_remote "$HEAD_NODE" "echo 'SSH connection successful'"
+# TODO: Add more pre-flight checks here, including checking if port 6443 is accessible
+
+# If --cleanup flag is set, uninstall k3s and exit
+if [ "$CLEANUP" == "true" ]; then
+    echo -e "${YELLOW}Starting cleanup...${NC}"
+
+    # Clean up head node
+    cleanup_server_node "$HEAD_NODE"
+
+    # Clean up worker nodes
+    for NODE in $WORKER_NODES; do
+        cleanup_agent_node "$NODE"
+    done
+
+    echo -e "${GREEN}Cleanup completed successfully.${NC}"
+    exit 0
+fi
+
+# Step 1: Install k3s on the head node
+progress_message "Deploying Kubernetes on head node ($HEAD_NODE)..."
+run_remote "$HEAD_NODE" "
+    curl -sfL https://get.k3s.io | K3S_TOKEN=$K3S_TOKEN sh - &&
+    mkdir -p ~/.kube &&
+    sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config &&
+    sudo chown \$(id -u):\$(id -g) ~/.kube/config &&
+    for i in {1..3}; do
+        if kubectl wait --for=condition=ready node --all --timeout=2m --kubeconfig ~/.kube/config; then
+            break
+        else
+            echo 'Waiting for nodes to be ready...'
+            sleep 5
+        fi
+    done
+    if [ $i -eq 3 ]; then
+        echo 'Failed to wait for nodes to be ready after 3 attempts'
+        exit 1
+    fi"
+success_message "K3s deployed on head node."
+
+# Check if head node has a GPU
+if check_gpu "$HEAD_NODE"; then
+    echo -e "${YELLOW}GPU detected on head node ($HEAD_NODE).${NC}"
+    INSTALL_GPU=true
+fi
+
+# Fetch the head node's internal IP (this will be passed to worker nodes)
+MASTER_ADDR=$(run_remote "$HEAD_NODE" "hostname -I | awk '{print \$1}'")
+
+echo -e "${GREEN}Master node internal IP: $MASTER_ADDR${NC}"
+
+# Step 2: Install k3s on worker nodes and join them to the master node
+for NODE in $WORKER_NODES; do
+    progress_message "Deploying Kubernetes on worker node ($NODE)..."
+    run_remote "$NODE" "
+        curl -sfL https://get.k3s.io | K3S_URL=https://$MASTER_ADDR:6443 K3S_TOKEN=$K3S_TOKEN sh -"
+    success_message "Kubernetes deployed on worker node ($NODE)."
+
+    # Check if worker node has a GPU
+    if check_gpu "$NODE"; then
+        echo -e "${YELLOW}GPU detected on worker node ($NODE).${NC}"
+        INSTALL_GPU=true
+    fi
+done
+# Step 3: Configure local kubectl to connect to the cluster
+progress_message "Configuring local kubectl to connect to the cluster..."
+scp -o StrictHostKeyChecking=no -i "$SSH_KEY" "$USER@$HEAD_NODE":~/.kube/config ~/.kube/config
+
+# Back up the original kubeconfig file if it exists
+KUBECONFIG_FILE="$HOME/.kube/config"
+if [[ -f "$KUBECONFIG_FILE" ]]; then
+    echo "Backing up existing kubeconfig to $KUBECONFIG_FILE.bak"
+    cp "$KUBECONFIG_FILE" "$KUBECONFIG_FILE.bak"
+fi
+
+# Update kubeconfig for the local machine to use the master node's IP
+# Temporary file to hold the modified kubeconfig
+TEMP_FILE=$(mktemp)
+
+# Remove the certificate-authority-data, and replace the server with the master address
+awk '
+  BEGIN { in_cluster = 0 }
+  /^clusters:/ { in_cluster = 1 }
+  /^users:/ { in_cluster = 0 }
+  in_cluster && /^ *certificate-authority-data:/ { next }
+  in_cluster && /^ *server:/ {
+    print "    server: https://'${HEAD_NODE}:6443'"
+    print "    insecure-skip-tls-verify: true"
+    next
+  }
+  { print }
+' "$KUBECONFIG_FILE" > "$TEMP_FILE"
+
+# Replace the original kubeconfig with the modified one
+mv "$TEMP_FILE" "$KUBECONFIG_FILE"
+
+success_message "kubectl configured to connect to the cluster."
+
+echo "Cluster deployment completed. You can now run 'kubectl get nodes' to verify the setup."
+
+# Install GPU operator if a GPU was detected on any node
+if [ "$INSTALL_GPU" == "true" ]; then
+    echo -e "${YELLOW}GPU detected in the cluster. Installing Nvidia GPU Operator...${NC}"
+    run_remote "$HEAD_NODE" "
+        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 &&
+        chmod 700 get_helm.sh &&
+        ./get_helm.sh &&
+        helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update &&
+        kubectl create namespace gpu-operator --kubeconfig ~/.kube/config || true &&
+        sudo ln -s /sbin/ldconfig /sbin/ldconfig.real || true &&
+        helm install gpu-operator -n gpu-operator --create-namespace nvidia/gpu-operator \
+        --set 'toolkit.env[0].name=CONTAINERD_CONFIG' \
+        --set 'toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml' \
+        --set 'toolkit.env[1].name=CONTAINERD_SOCKET' \
+        --set 'toolkit.env[1].value=/run/k3s/containerd/containerd.sock' \
+        --set 'toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS' \
+        --set 'toolkit.env[2].value=nvidia' &&
+        echo 'Waiting for GPU operator installation...' &&
+        while ! kubectl describe nodes --kubeconfig ~/.kube/config | grep -q 'nvidia.com/gpu:'; do
+            echo 'Waiting for GPU operator...'
+            sleep 5
+        done
+        echo 'GPU operator installed successfully.'"
+    success_message "GPU Operator installed."
+else
+    echo -e "${YELLOW}No GPUs detected. Skipping GPU Operator installation.${NC}"
+fi
+
+# Configure SkyPilot
+progress_message "Configuring SkyPilot..."
+sky check kubernetes
+success_message "SkyPilot configured successfully."
+
+# Display final success message
+echo -e "${GREEN}==== 🎉 Kubernetes cluster deployment completed successfully 🎉 ====${NC}"
+echo "You can now interact with your Kubernetes cluster through SkyPilot: "
+echo "  • List available GPUs: sky show-gpus --cloud kubernetes"
+echo "  • Launch a GPU development pod: sky launch -c devbox --cloud kubernetes --gpus A100:1"
+echo "  • Connect to pod with SSH: ssh devbox"
+echo "  • Connect to pod with VSCode: code --remote ssh-remote+devbox '/'"

sky/utils/log_utils.py

@@ -1,6 +1,7 @@
 """Logging utils."""
 import enum
-from typing import List, Optional
+import types
+from typing import List, Optional, Type
 
 import colorama
 import pendulum
@@ -15,13 +16,15 @@ logger = sky_logging.init_logger(__name__)
 class LineProcessor(object):
     """A processor for log lines."""
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         pass
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         pass
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         pass
 
@@ -34,12 +37,12 @@ class RayUpLineProcessor(LineProcessor):
         RUNTIME_SETUP = 1
         PULLING_DOCKER_IMAGES = 2
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         self.state = self.ProvisionStatus.LAUNCH
         self.status_display = rich_utils.safe_status('[bold cyan]Launching')
         self.status_display.start()
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         if ('Success.' in log_line and
                 self.state == self.ProvisionStatus.LAUNCH):
             logger.info(f'{colorama.Fore.GREEN}Head node is up.'
@@ -60,7 +63,9 @@ class RayUpLineProcessor(LineProcessor):
                 '[bold cyan]Launching - Preparing SkyPilot runtime')
             self.state = self.ProvisionStatus.RUNTIME_SETUP
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         self.status_display.stop()
 
@@ -68,13 +73,13 @@ class RayUpLineProcessor(LineProcessor):
 class SkyLocalUpLineProcessor(LineProcessor):
     """A processor for `sky local up` log lines."""
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         status = rich_utils.safe_status('[bold cyan]Creating local cluster - '
                                         'initializing Kubernetes')
         self.status_display = status
         self.status_display.start()
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         if 'Kind cluster created.' in log_line:
             logger.info(f'{colorama.Fore.GREEN}Kubernetes is running.'
                         f'{colorama.Style.RESET_ALL}')
@@ -124,7 +129,80 @@ class SkyLocalUpLineProcessor(LineProcessor):
                 f'{colorama.Fore.GREEN}Nginx Ingress Controller installed.'
                 f'{colorama.Style.RESET_ALL}')
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
+        del except_type, except_value, traceback  # unused
+        self.status_display.stop()
+
+
+class SkyRemoteUpLineProcessor(LineProcessor):
+    """A processor for deploy_remote_cluster.sh log lines."""
+
+    def __enter__(self) -> None:
+        status = rich_utils.safe_status('[bold cyan]Creating remote cluster')
+        self.status_display = status
+        self.status_display.start()
+
+    def process_line(self, log_line: str) -> None:
+        # Pre-flight checks
+        if 'SSH connection successful' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}SSH connection established.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Kubernetes installation steps
+        if 'Deploying Kubernetes on head node' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'deploying Kubernetes on head node')
+        if 'K3s deployed on head node.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ K3s successfully deployed on head node.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Worker nodes
+        if 'Deploying Kubernetes on worker node' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'deploying Kubernetes on worker nodes')
+        if 'Kubernetes deployed on worker node' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ K3s successfully deployed on worker node.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Cluster configuration
+        if 'Configuring local kubectl to connect to the cluster...' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'configuring local kubectl')
+        if 'kubectl configured to connect to the cluster.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ kubectl configured for the remote cluster.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # GPU operator installation
+        if 'Installing Nvidia GPU Operator...' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'installing Nvidia GPU Operator')
+        if 'GPU Operator installed.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ Nvidia GPU Operator installed successfully.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Cleanup steps
+        if 'Cleaning up head node' in log_line:
+            self.status_display.update('[bold cyan]Cleaning up head node')
+        if 'Cleaning up node' in log_line:
+            self.status_display.update('[bold cyan]Cleaning up worker node')
+        if 'cleaned up successfully' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
+
+        # Final status
+        if 'Cluster deployment completed.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}✔ Remote k3s is running.'
+                        f'{colorama.Style.RESET_ALL}')
+
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         self.status_display.stop()
 

{skypilot_nightly-1.0.0.dev20240927.dist-info → skypilot_nightly-1.0.0.dev20240928.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240927
+Version: 1.0.0.dev20240928
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240927.dist-info → skypilot_nightly-1.0.0.dev20240928.dist-info}/RECORD

@@ -1,8 +1,8 @@
-sky/__init__.py,sha256=S-XaZNVM-9OM5oGtcUfWmQC9CLW7HvN9ckCN-KCbPio,5854
+sky/__init__.py,sha256=8BEk3x0IPkFli8tjp7axkkM5mwQ1GuCABWwTMppkPcc,5854
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=o8ZhUf4VSN8WtjWcUUGYg-HVskaqaoMK4ZobHC-HVYU,20697
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
-sky/cli.py,sha256=DMnZ-vLBuFazKmiMavidSYkQvv_YvXnZALJkHaLveDM,201714
+sky/cli.py,sha256=9h4yO8p962960qUjvQ-xSusrtdh8TXNNQ1sfV0OqgZc,206262
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
 sky/core.py,sha256=YF_6kwj8Ja171Oycb8L25SZ7V_ylZYovFS_jpnjwGo0,34408
 sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
@@ -252,7 +252,7 @@ sky/utils/dag_utils.py,sha256=gjGZiJj4_GYsraXX67e6ElvbmOByJcyjSfvVgYZiXvs,5588
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
 sky/utils/env_options.py,sha256=1VXyd3bhiUgGfCpmmTqM9PagRo1ILBH4-pzIxmIeE6E,861
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
-sky/utils/log_utils.py,sha256=W7FYK7xzvbq4V-8R-ihLtz939ryvtABug6O-4DFrjho,8139
+sky/utils/log_utils.py,sha256=yVu3etgKhiVYX8UG-JFPWZujxWBT4kwxZ5oAPIdjtGs,12054
 sky/utils/resources_utils.py,sha256=snByBxgx3Hnjfch2uysdAA3D-OAwrnuzTDHug36s5H4,6515
 sky/utils/rich_utils.py,sha256=5ZVhzlFx-nhqMXwv00eO9xC4rz7ibDlfD2lmGhZrJEY,1581
 sky/utils/schemas.py,sha256=QT0Fxri2o0SiWkky1DlZhA1dzQRQoB5OdVaej0wJvhc,28787
@@ -265,6 +265,7 @@ sky/utils/cli_utils/status_utils.py,sha256=9odkfXiXLMD14XJsqve6sGvHpe7ThHXpC6ic9
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=rv5Lz6AR00yBJMRyScfMSQiGKptMhtHWRsvyG20-u9c,7764
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
+sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=vGj0mD0tejHDRy8ulwKOvOF2mfLyT5J8fp7GVqEe_EY,8478
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=AcYhuuG5jXWGHUmyRuH-oKy5qcn92gXhu6bXOt6eD6g,9274
 sky/utils/kubernetes/gpu_labeler.py,sha256=MEUv0U4ACDcNwtFVltlv017XJMjxx1Bndf6fL0i6eqg,6960
@@ -272,9 +273,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=Ma-N9a271fTfdgP5-8XIQL7KPf8IPUo-uY004PCdUFo,747
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240927.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240927.dist-info/METADATA,sha256=GXz5qTbQuxyKpPHIVPT5vsvlpo3bRyataK8Vtj6rovw,18948
-skypilot_nightly-1.0.0.dev20240927.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-skypilot_nightly-1.0.0.dev20240927.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240927.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240927.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240928.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240928.dist-info/METADATA,sha256=AT9cnsY7Uj7BK0COu8mOXiCtfyCFrjtk7OBQvqx-_Nk,18948
+skypilot_nightly-1.0.0.dev20240928.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+skypilot_nightly-1.0.0.dev20240928.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240928.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240928.dist-info/RECORD,,