skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/utils/db_utils.py

@@ -1,100 +0,0 @@
-"""Utils for sky databases."""
-import contextlib
-import sqlite3
-import threading
-from typing import Any, Callable, Optional
-
-# This parameter (passed to sqlite3.connect) controls how long we will wait to
-# obtains a database lock (not necessarily during connection, but whenever it is
-# needed). It is not a connection timeout.
-# Even in WAL mode, only a single writer is allowed at a time. Other writers
-# will block until the write lock can be obtained. This behavior is described in
-# the SQLite documentation for WAL: https://www.sqlite.org/wal.html
-# Python's default timeout is 5s. In normal usage, lock contention is very low,
-# and this is more than sufficient. However, in some highly concurrent cases,
-# such as a jobs controller suddenly recovering thousands of jobs at once, we
-# can see a small number of processes that take much longer to obtain the lock.
-# In contrived highly contentious cases, around 0.1% of transactions will take
-# >30s to take the lock. We have not seen cases that take >60s. For cases up to
-# 1000x parallelism, this is thus thought to be a conservative setting.
-# For more info, see the PR description for #4552.
-_DB_TIMEOUT_S = 60
-
-
-@contextlib.contextmanager
-def safe_cursor(db_path: str):
-    """A newly created, auto-committing, auto-closing cursor."""
-    conn = sqlite3.connect(db_path, timeout=_DB_TIMEOUT_S)
-    cursor = conn.cursor()
-    try:
-        yield cursor
-    finally:
-        cursor.close()
-        conn.commit()
-        conn.close()
-
-
-def add_column_to_table(
-    cursor: 'sqlite3.Cursor',
-    conn: 'sqlite3.Connection',
-    table_name: str,
-    column_name: str,
-    column_type: str,
-    copy_from: Optional[str] = None,
-    value_to_replace_existing_entries: Optional[Any] = None,
-):
-    """Add a column to a table."""
-    for row in cursor.execute(f'PRAGMA table_info({table_name})'):
-        if row[1] == column_name:
-            break
-    else:
-        try:
-            add_column_cmd = (f'ALTER TABLE {table_name} '
-                              f'ADD COLUMN {column_name} {column_type}')
-            cursor.execute(add_column_cmd)
-            if copy_from is not None:
-                cursor.execute(f'UPDATE {table_name} '
-                               f'SET {column_name} = {copy_from}')
-            if value_to_replace_existing_entries is not None:
-                cursor.execute(
-                    f'UPDATE {table_name} '
-                    f'SET {column_name} = (?) '
-                    f'WHERE {column_name} IS NULL',
-                    (value_to_replace_existing_entries,))
-        except sqlite3.OperationalError as e:
-            if 'duplicate column name' in str(e):
-                # We may be trying to add the same column twice, when
-                # running multiple threads. This is fine.
-                pass
-            else:
-                raise
-    conn.commit()
-
-
-def rename_column(
-    cursor: 'sqlite3.Cursor',
-    conn: 'sqlite3.Connection',
-    table_name: str,
-    old_name: str,
-    new_name: str,
-):
-    """Rename a column in a table."""
-    # NOTE: This only works for sqlite3 >= 3.25.0. Be careful to use this.
-
-    for row in cursor.execute(f'PRAGMA table_info({table_name})'):
-        if row[1] == old_name:
-            cursor.execute(f'ALTER TABLE {table_name} '
-                           f'RENAME COLUMN {old_name} to {new_name}')
-            break
-    conn.commit()
-
-
-class SQLiteConn(threading.local):
-    """Thread-local connection to the sqlite3 database."""
-
-    def __init__(self, db_path: str, create_table: Callable):
-        super().__init__()
-        self.db_path = db_path
-        self.conn = sqlite3.connect(db_path, timeout=_DB_TIMEOUT_S)
-        self.cursor = self.conn.cursor()
-        create_table(self.cursor, self.conn)

sky/utils/kubernetes/deploy_remote_cluster.sh

@@ -1,308 +0,0 @@
-#!/bin/bash
-# Refer to https://docs.skypilot.co/en/latest/reservations/existing-machines.html for details on how to use this script.
-set -e
-
-# Colors for nicer UX
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No color
-
-# Variables
-CLEANUP=false
-INSTALL_GPU=false
-POSITIONAL_ARGS=()
-PASSWORD=""
-
-# Process all arguments
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --cleanup)
-            CLEANUP=true
-            shift
-            ;;
-        --password)
-            PASSWORD=$2
-            shift
-            shift
-            ;;
-        *)
-            POSITIONAL_ARGS+=("$1")
-            shift
-            ;;
-    esac
-done
-
-# Restore positional arguments in correct order
-set -- "${POSITIONAL_ARGS[@]}"
-
-# Assign positional arguments to variables
-IPS_FILE=$1
-USER=$2
-SSH_KEY=$3
-CONTEXT_NAME=${4:-default}
-K3S_TOKEN=mytoken  # Any string can be used as the token
-# Create temporary askpass script for sudo
-ASKPASS_BLOCK="# Create temporary askpass script
-ASKPASS_SCRIPT=\$(mktemp)
-trap 'rm -f \$ASKPASS_SCRIPT' EXIT INT TERM ERR QUIT
-cat > \$ASKPASS_SCRIPT << EOF
-#!/bin/bash
-echo $PASSWORD
-EOF
-chmod 700 \$ASKPASS_SCRIPT
-# Use askpass
-export SUDO_ASKPASS=\$ASKPASS_SCRIPT
-"
-
-# Basic argument checks
-if [ -z "$IPS_FILE" ] || [ -z "$USER" ] || [ -z "$SSH_KEY" ]; then
-    >&2 echo -e "${RED}Error: Missing required arguments.${NC}"
-    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [context-name] [--cleanup] [--password password]"
-    exit 1
-fi
-
-# Check if SSH key exists
-if [ ! -f "$SSH_KEY" ]; then
-    >&2 echo -e "${RED}Error: SSH key not found: $SSH_KEY${NC}"
-    exit 1
-fi
-
-# Check if IPs file exists
-if [ ! -f "$IPS_FILE" ]; then
-    >&2 echo -e "${RED}Error: IPs file not found: $IPS_FILE${NC}"
-    exit 1
-fi
-
-# Get head node and worker nodes from the IPs file
-HEAD_NODE=$(head -n 1 "$IPS_FILE")
-WORKER_NODES=$(tail -n +2 "$IPS_FILE")
-
-# Check if the IPs file is empty or not formatted correctly
-if [ -z "$HEAD_NODE" ]; then
-    >&2 echo -e "${RED}Error: IPs file is empty or not formatted correctly.${NC}"
-    exit 1
-fi
-
-# Function to show a progress message
-progress_message() {
-    echo -e "${YELLOW}➜ $1${NC}"
-}
-
-# Step to display success
-success_message() {
-    echo -e "${GREEN}✔ $1${NC}"
-}
-
-# Function to run a command on a remote machine via SSH
-run_remote() {
-    local NODE_IP=$1
-    local CMD=$2
-    # echo -e "${YELLOW}Running command on $NODE_IP...${NC}"
-    ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "$SSH_KEY" "$USER@$NODE_IP" "$CMD"
-}
-
-# Function to uninstall k3s and clean up the state on a remote machine
-cleanup_server_node() {
-    local NODE_IP=$1
-    echo -e "${YELLOW}Cleaning up head node $NODE_IP...${NC}"
-    run_remote "$NODE_IP" "
-        $ASKPASS_BLOCK
-        echo 'Uninstalling k3s...' &&
-        sudo -A /usr/local/bin/k3s-uninstall.sh || true &&
-        sudo -A rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
-    "
-    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
-}
-
-# Function to uninstall k3s and clean up the state on a remote machine
-cleanup_agent_node() {
-    local NODE_IP=$1
-    echo -e "${YELLOW}Cleaning up node $NODE_IP...${NC}"
-    run_remote "$NODE_IP" "
-        $ASKPASS_BLOCK
-        echo 'Uninstalling k3s...' &&
-        sudo -A /usr/local/bin/k3s-agent-uninstall.sh || true &&
-        sudo -A rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
-    "
-    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
-}
-
-check_gpu() {
-    local NODE_IP=$1
-    if run_remote "$NODE_IP" "command -v nvidia-smi &> /dev/null && nvidia-smi --query-gpu=gpu_name --format=csv,noheader &> /dev/null"; then
-        return 0  # GPU detected
-    else
-        return 1  # No GPU detected
-    fi
-}
-
-# Pre-flight checks
-run_remote "$HEAD_NODE" "echo 'SSH connection successful'"
-# TODO: Add more pre-flight checks here, including checking if port 6443 is accessible
-
-# If --cleanup flag is set, uninstall k3s and exit
-if [ "$CLEANUP" == "true" ]; then
-    echo -e "${YELLOW}Starting cleanup...${NC}"
-
-    # Clean up head node
-    cleanup_server_node "$HEAD_NODE"
-
-    # Clean up worker nodes
-    for NODE in $WORKER_NODES; do
-        cleanup_agent_node "$NODE"
-    done
-
-    # Remove the context from local kubeconfig if it exists
-    if [ -f "$HOME/.kube/config" ]; then
-        progress_message "Removing context '$CONTEXT_NAME' from local kubeconfig..."
-        kubectl config delete-context "$CONTEXT_NAME" 2>/dev/null || true
-        kubectl config delete-cluster "$CONTEXT_NAME" 2>/dev/null || true
-        kubectl config delete-user "$CONTEXT_NAME" 2>/dev/null || true
-        # Update the current context to the first available context
-        kubectl config use-context $(kubectl config view -o jsonpath='{.contexts[0].name}') 2>/dev/null || true
-        success_message "Context '$CONTEXT_NAME' removed from local kubeconfig."
-    fi
-
-    echo -e "${GREEN}Cleanup completed successfully.${NC}"
-    exit 0
-fi
-
-# Step 1: Install k3s on the head node
-progress_message "Deploying Kubernetes on head node ($HEAD_NODE)..."
-run_remote "$HEAD_NODE" "
-    $ASKPASS_BLOCK
-    curl -sfL https://get.k3s.io | K3S_TOKEN=$K3S_TOKEN sudo -E -A sh - &&
-    mkdir -p ~/.kube &&
-    sudo -A cp /etc/rancher/k3s/k3s.yaml ~/.kube/config &&
-    sudo -A chown \$(id -u):\$(id -g) ~/.kube/config &&
-    for i in {1..3}; do
-        if kubectl wait --for=condition=ready node --all --timeout=2m --kubeconfig ~/.kube/config; then
-            break
-        else
-            echo 'Waiting for nodes to be ready...'
-            sleep 5
-        fi
-    done
-    if [ \$i -eq 3 ]; then
-        echo 'Failed to wait for nodes to be ready after 3 attempts'
-        exit 1
-    fi"
-success_message "K3s deployed on head node."
-
-# Check if head node has a GPU
-if check_gpu "$HEAD_NODE"; then
-    echo -e "${YELLOW}GPU detected on head node ($HEAD_NODE).${NC}"
-    INSTALL_GPU=true
-fi
-
-# Fetch the head node's internal IP (this will be passed to worker nodes)
-MASTER_ADDR=$(run_remote "$HEAD_NODE" "hostname -I | awk '{print \$1}'")
-
-echo -e "${GREEN}Master node internal IP: $MASTER_ADDR${NC}"
-
-# Step 2: Install k3s on worker nodes and join them to the master node
-for NODE in $WORKER_NODES; do
-    progress_message "Deploying Kubernetes on worker node ($NODE)..."
-    run_remote "$NODE" "
-        $ASKPASS_BLOCK
-        curl -sfL https://get.k3s.io | K3S_URL=https://$MASTER_ADDR:6443 K3S_TOKEN=$K3S_TOKEN sudo -E -A sh -"
-    success_message "Kubernetes deployed on worker node ($NODE)."
-
-    # Check if worker node has a GPU
-    if check_gpu "$NODE"; then
-        echo -e "${YELLOW}GPU detected on worker node ($NODE).${NC}"
-        INSTALL_GPU=true
-    fi
-done
-# Step 3: Configure local kubectl to connect to the cluster
-progress_message "Configuring local kubectl to connect to the cluster..."
-
-# Create temporary directory for kubeconfig operations
-TEMP_DIR=$(mktemp -d)
-TEMP_KUBECONFIG="$TEMP_DIR/kubeconfig"
-
-# Get the kubeconfig from remote server
-scp -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "$SSH_KEY" "$USER@$HEAD_NODE":~/.kube/config "$TEMP_KUBECONFIG"
-
-# Create .kube directory if it doesn't exist
-mkdir -p "$HOME/.kube"
-
-# Create empty kubeconfig if it doesn't exist
-KUBECONFIG_FILE="$HOME/.kube/config"
-if [[ ! -f "$KUBECONFIG_FILE" ]]; then
-    touch "$KUBECONFIG_FILE"
-fi
-
-# Modify the temporary kubeconfig to update server address and context name
-awk -v context="$CONTEXT_NAME" '
-  /^clusters:/ { in_cluster = 1 }
-  /^users:/ { in_cluster = 0 }
-  in_cluster && /^ *certificate-authority-data:/ { next }
-  in_cluster && /^ *server:/ {
-    print "    server: https://'${HEAD_NODE}:6443'"
-    print "    insecure-skip-tls-verify: true"
-    next
-  }
-  /name: default/ { sub("name: default", "name: " context) }
-  /cluster: default/ { sub("cluster: default", "cluster: " context) }
-  /user: default/ { sub("user: default", "user: " context) }
-  /current-context: default/ { sub("current-context: default", "current-context: " context) }
-  { print }
-' "$TEMP_KUBECONFIG" > "$TEMP_DIR/modified_config"
-
-# Merge the configurations using kubectl
-KUBECONFIG="$KUBECONFIG_FILE:$TEMP_DIR/modified_config" kubectl config view --flatten > "$TEMP_DIR/merged_config"
-mv "$TEMP_DIR/merged_config" "$KUBECONFIG_FILE"
-
-# Set the new context as the current context
-kubectl config use-context "$CONTEXT_NAME"
-
-# Clean up temporary files
-rm -rf "$TEMP_DIR"
-
-success_message "kubectl configured with new context '$CONTEXT_NAME'."
-
-echo "Cluster deployment completed. You can now run 'kubectl get nodes' to verify the setup."
-
-# Install GPU operator if a GPU was detected on any node
-if [ "$INSTALL_GPU" == "true" ]; then
-    echo -e "${YELLOW}GPU detected in the cluster. Installing Nvidia GPU Operator...${NC}"
-    run_remote "$HEAD_NODE" "
-        $ASKPASS_BLOCK
-        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 &&
-        chmod 700 get_helm.sh &&
-        ./get_helm.sh &&
-        helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update &&
-        kubectl create namespace gpu-operator --kubeconfig ~/.kube/config || true &&
-        sudo -A ln -s /sbin/ldconfig /sbin/ldconfig.real || true &&
-        helm install gpu-operator -n gpu-operator --create-namespace nvidia/gpu-operator \
-        --set 'toolkit.env[0].name=CONTAINERD_CONFIG' \
-        --set 'toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml' \
-        --set 'toolkit.env[1].name=CONTAINERD_SOCKET' \
-        --set 'toolkit.env[1].value=/run/k3s/containerd/containerd.sock' \
-        --set 'toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS' \
-        --set 'toolkit.env[2].value=nvidia' &&
-        echo 'Waiting for GPU operator installation...' &&
-        while ! kubectl describe nodes --kubeconfig ~/.kube/config | grep -q 'nvidia.com/gpu:'; do
-            echo 'Waiting for GPU operator...'
-            sleep 5
-        done
-        echo 'GPU operator installed successfully.'"
-    success_message "GPU Operator installed."
-else
-    echo -e "${YELLOW}No GPUs detected. Skipping GPU Operator installation.${NC}"
-fi
-
-# Configure SkyPilot
-progress_message "Configuring SkyPilot..."
-sky check kubernetes
-success_message "SkyPilot configured successfully."
-
-# Display final success message
-echo -e "${GREEN}==== 🎉 Kubernetes cluster deployment completed successfully 🎉 ====${NC}"
-echo "You can now interact with your Kubernetes cluster through SkyPilot: "
-echo "  • List available GPUs: sky show-gpus --cloud kubernetes"
-echo "  • Launch a GPU development pod: sky launch -c devbox --cloud kubernetes --gpus A100:1"
-echo "  • Connect to pod with SSH: ssh devbox"
-echo "  • Connect to pod with VSCode: code --remote ssh-remote+devbox '/'"

sky/utils/kubernetes/ssh_jump_lifecycle_manager.py

@@ -1,191 +0,0 @@
-"""Manages lifecycle of ssh jump pod.
-
-This script runs inside ssh jump pod as the main process (PID 1).
-
-It terminates itself (by removing ssh jump service and pod via a call to
-kubeapi) if it does not see ray pods in the duration of 10 minutes. If the
-user re-launches a task before the duration is over, then ssh jump pod is being
-reused and will terminate itself when it sees that no ray clusters exist in
-that duration.
-
-To allow multiple users to the share the same SSH jump pod,
-this script also reloads SSH keys from the mounted secret volume on an
-interval and updates `~/.ssh/authorized_keys`.
-"""
-import datetime
-import os
-import subprocess
-import sys
-import threading
-import time
-
-from kubernetes import client
-from kubernetes import config
-
-# Load kube config
-config.load_incluster_config()
-
-v1 = client.CoreV1Api()
-
-current_name = os.getenv('MY_POD_NAME')
-current_namespace = os.getenv('MY_POD_NAMESPACE')
-
-# The amount of time in seconds where no Ray pods exist in which after that time
-# ssh jump pod terminates itself
-alert_threshold = int(os.getenv('ALERT_THRESHOLD', '600'))
-# The amount of time in seconds to wait between Ray pods existence checks
-retry_interval = int(os.getenv('RETRY_INTERVAL', '60'))
-# The amount of time in seconds to wait between SSH key reloads
-reload_interval = int(os.getenv('RELOAD_INTERVAL', '5'))
-
-# Ray pods are labeled with this value i.e., ssh jump name which is unique per
-# user (based on user hash)
-label_selector = f'skypilot-ssh-jump={current_name}'
-
-
-def poll(interval, leading=True):
-    """Decorator factory for polling function. To stop polling, return True.
-
-    Args:
-        interval (int): The amount of time to wait between function calls.
-        leading (bool): Whether to wait before (rather than after) calls.
-    """
-
-    def decorator(func):
-
-        def wrapper(*args, **kwargs):
-            while True:
-                if leading:
-                    time.sleep(interval)
-                done = func(*args, **kwargs)
-                if done:
-                    return
-                if not leading:
-                    time.sleep(interval)
-
-        return wrapper
-
-    return decorator
-
-
-# Flag to terminate the reload keys thread when the lifecycle thread
-# terminates.
-terminated = False
-
-
-@poll(interval=reload_interval, leading=False)
-def reload_keys():
-    """Reloads SSH keys from mounted secret volume."""
-
-    if terminated:
-        sys.stdout.write('[SSH Key Reloader] Terminated.\n')
-        return True
-
-    # Reload SSH keys from mounted secret volume if changed.
-    tmpfile = '/tmp/sky-ssh-keys'
-    try:
-        subprocess.check_output(
-            f'cat /etc/secret-volume/ssh-publickey* > {tmpfile}', shell=True)
-        try:
-            subprocess.check_output(f'diff {tmpfile} ~/.ssh/authorized_keys',
-                                    shell=True)
-            sys.stdout.write(
-                '[SSH Key Reloader] No keys changed, continuing.\n')
-        except subprocess.CalledProcessError as e:
-            if e.returncode == 1:
-                sys.stdout.write(
-                    '[SSH Key Reloader] Changes detected, reloading.\n')
-                subprocess.check_output(f'mv {tmpfile} ~/.ssh/authorized_keys',
-                                        shell=True)
-            else:
-                raise
-    except Exception as e:
-        sys.stdout.write(
-            f'[SSH Key Reloader][ERROR] Failed to reload SSH keys: {e}\n')
-        raise
-
-
-alert_delta = datetime.timedelta(seconds=alert_threshold)
-retry_interval_delta = datetime.timedelta(seconds=retry_interval)
-# Accumulated time of where no SkyPilot cluster exists. Compared
-# against alert_threshold.
-nocluster_delta = datetime.timedelta()
-
-
-@poll(interval=retry_interval)
-def manage_lifecycle():
-    """Manages lifecycle of ssh jump pod."""
-
-    global terminated, nocluster_delta
-
-    try:
-        ret = v1.list_namespaced_pod(current_namespace,
-                                     label_selector=label_selector)
-    except Exception as e:
-        sys.stdout.write('[Lifecycle] [ERROR] listing pods failed with '
-                         f'error: {e}\n')
-        raise
-
-    if not ret.items:
-        sys.stdout.write(
-            f'[Lifecycle] Did not find pods with label '
-            f'"{label_selector}" in namespace {current_namespace}\n')
-        nocluster_delta = nocluster_delta + retry_interval_delta
-        sys.stdout.write(
-            f'[Lifecycle] Time since no pods found: {nocluster_delta}, alert '
-            f'threshold: {alert_delta}\n')
-    else:
-        sys.stdout.write(
-            f'[Lifecycle] Found pods with label "{label_selector}" in '
-            f'namespace {current_namespace}\n')
-        # reset ..
-        nocluster_delta = datetime.timedelta()
-        sys.stdout.write(
-            f'[Lifecycle] nocluster_delta is reset: {nocluster_delta}\n')
-
-    if nocluster_delta >= alert_delta:
-        sys.stdout.write(
-            f'[Lifecycle] nocluster_delta: {nocluster_delta} crossed alert '
-            f'threshold: {alert_delta}. Time to terminate myself and my '
-            'service.\n')
-        try:
-            # ssh jump resources created under same name
-            v1.delete_namespaced_service(current_name, current_namespace)
-            v1.delete_namespaced_pod(current_name, current_namespace)
-        except Exception as e:
-            sys.stdout.write('[Lifecycle][ERROR] Deletion failed. Exiting '
-                             f'poll() with error: {e}\n')
-            raise
-
-        terminated = True
-        return True
-
-
-def main():
-    sys.stdout.write('SkyPilot SSH Jump Pod Lifecycle Manager\n')
-    sys.stdout.write(f'current_name: {current_name}\n')
-    sys.stdout.write(f'current_namespace: {current_namespace}\n')
-    sys.stdout.write(f'alert_threshold time: {alert_threshold}\n')
-    sys.stdout.write(f'retry_interval time: {retry_interval}\n')
-    sys.stdout.write(f'reload_interval time: {reload_interval}\n')
-    sys.stdout.write(f'label_selector: {label_selector}\n')
-
-    if not current_name or not current_namespace:
-        # Raise Exception with message to terminate pod
-        raise Exception('Missing environment variables MY_POD_NAME or '
-                        'MY_POD_NAMESPACE')
-
-    threads = [
-        threading.Thread(target=manage_lifecycle),
-        threading.Thread(target=reload_keys)
-    ]
-    sys.stdout.write(f'Polling with {len(threads)} threads.\n')
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join()
-    sys.stdout.write('Done.\n')
-
-
-if __name__ == '__main__':
-    main()