skypilot-nightly 1.0.0.dev20251203__py3-none-any.whl → 1.0.0.dev20260112__py3-none-any.whl

sky/utils/instance_links.py

@@ -0,0 +1,139 @@
+"""Utility functions for generating instance links for cloud providers."""
+from typing import Dict
+
+from sky import sky_logging
+from sky.provision import common
+from sky.provision import constants as provision_constants
+
+logger = sky_logging.init_logger(__name__)
+
+# URL templates for each cloud provider
+# Placeholders:
+#   {region} - Cloud region
+#   {project_id} - GCP project ID
+#   {subscription_id} - Azure subscription ID
+#   {resource_group} - Azure resource group
+#   {tag_key} - Tag key used to identify cluster instances
+#   {cluster_name} - Name of the cluster
+
+AWS_INSTANCES_URL = ('https://{region}.console.aws.amazon.com/ec2/v2/home'
+                     '?region={region}#Instances:tag:{tag_key}={cluster_name}')
+
+# Azure doesn't support direct tag filter URLs, so we link to the resource group
+AZURE_RESOURCE_GROUP_URL = (
+    'https://portal.azure.com/#@/resource/subscriptions'
+    '/{subscription_id}/resourceGroups/{resource_group}/overview')
+
+# GCP Console base URL
+GCP_INSTANCES_BASE_URL = 'https://console.cloud.google.com/compute/instances'
+
+
+def _build_gcp_instances_url(project_id: str, tag_key: str,
+                             cluster_name: str) -> str:
+    """Build GCP instances URL with label filter.
+
+    GCP Console uses a pageState parameter with a specially encoded filter.
+    The filter JSON structure is:
+        [{"k":"","t":10,"v":"\"label_key:label_value\"","s":true}]
+
+    Where:
+        - k: filter key (empty for label filters)
+        - t: filter type (10 = label filter)
+        - v: filter value with escaped quotes around "label_key:label_value"
+        - s: unknown, always true
+
+    GCP uses a mix of:
+        - Standard URL encoding for outer structure (%22 for ")
+        - Underscore notation inside the filter (_22 for ", _3A for :, etc.)
+        - Double URL-encoding for brackets (%255B = %5B = [)
+    """
+    # Build the filter value: \"tag_key:cluster_name\"
+    # Using underscore notation: _5C_22 = \", _3A = :
+    filter_value = f'_5C_22{tag_key}_3A{cluster_name}_5C_22'
+
+    # Build the filter object using underscore notation for internal quotes and
+    # colons.
+    # {"k":"","t":10,"v":"<filter_value>","s":true}
+    # _22 = ", _3A = :, _2C = ,
+    filter_obj = (
+        f'_22k_22_3A_22_22_2C'  # "k":"",
+        f'_22t_22_3A10_2C'  # "t":10,
+        f'_22v_22_3A_22{filter_value}_22_2C'  # "v":"<value>",
+        f'_22s_22_3Atrue')  # "s":true
+
+    # Wrap in array brackets (double URL-encoded: %255B = %5B = [, %257D = %7D)
+    filter_array = f'%255B%257B{filter_obj}%257D%255D'
+
+    # Build pageState: ("instances":("p":0,"f":"<filter>"))
+    # %22 = " (standard URL encoding)
+    page_state = f'(%22instances%22:(%22p%22:0,%22f%22:%22{filter_array}%22))'
+
+    return (
+        f'{GCP_INSTANCES_BASE_URL}?project={project_id}&pageState={page_state}')
+
+
+def generate_instance_links(
+    cluster_info: common.ClusterInfo,
+    cluster_name: str,
+) -> Dict[str, str]:
+    """Generate instance links for a cluster based on the cloud provider.
+
+    Creates links to filtered views in cloud consoles that show all instances
+    belonging to the cluster (useful for multi-node jobs).
+
+    Args:
+        cluster_info: ClusterInfo object containing instance information.
+        cluster_name: Cluster name for tag-based filtering.
+
+    Returns:
+        Dictionary mapping link labels to URLs. Empty dict if links cannot be
+        generated (e.g., for Kubernetes or unsupported clouds).
+    """
+    links: Dict[str, str] = {}
+    provider_name = cluster_info.provider_name.lower()
+    provider_config = cluster_info.provider_config or {}
+
+    # Skip Kubernetes and other non-cloud providers
+    if provider_name in ('kubernetes', 'local'):
+        return links
+
+    # Tag used by SkyPilot to identify cluster instances
+    tag_key = provision_constants.TAG_RAY_CLUSTER_NAME
+
+    if provider_name == 'aws':
+        region = provider_config.get('region')
+        if not region:
+            logger.debug('AWS region not found in provider config, '
+                         'skipping instance links')
+            return links
+        links['AWS Instances'] = AWS_INSTANCES_URL.format(
+            region=region,
+            tag_key=tag_key,
+            cluster_name=cluster_name,
+        )
+
+    elif provider_name == 'gcp':
+        project_id = provider_config.get('project_id')
+        if not project_id:
+            logger.debug('GCP project_id not found in provider config, '
+                         'skipping instance links')
+            return links
+        links['GCP Instances'] = _build_gcp_instances_url(
+            project_id=project_id,
+            tag_key=tag_key,
+            cluster_name=cluster_name,
+        )
+
+    elif provider_name == 'azure':
+        subscription_id = provider_config.get('subscription_id')
+        resource_group = provider_config.get('resource_group')
+        if not subscription_id or not resource_group:
+            logger.debug('Azure subscription_id or resource_group not found '
+                         'in provider config, skipping instance links')
+            return links
+        links['Azure Resource Group'] = AZURE_RESOURCE_GROUP_URL.format(
+            subscription_id=subscription_id,
+            resource_group=resource_group,
+        )
+
+    return links

sky/utils/interactive_utils.py

@@ -0,0 +1,49 @@
+"""Utilities for server-side interactive SSH functionality."""
+import array
+import socket
+
+
+def get_pty_socket_path(session_id: str) -> str:
+    """Get the Unix socket path for PTY file descriptor passing."""
+    return f'/tmp/sky_pty_{session_id}.sock'
+
+
+def send_fd(sock: socket.socket, fd: int) -> None:
+    """Send file descriptor via Unix socket using SCM_RIGHTS.
+
+    SCM_RIGHTS allows us to send or receive a set of open
+    file descriptors from another process.
+
+    See:
+    https://man7.org/linux/man-pages/man7/unix.7.html
+    https://man7.org/linux/man-pages/man3/cmsg.3.html
+
+    Args:
+        sock: Connected Unix socket.
+        fd: File descriptor to send.
+    """
+    sock.sendmsg(
+        [b'x'],  # Dummy data
+        [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [fd]))])
+
+
+def recv_fd(sock: socket.socket) -> int:
+    """Receive file descriptor via Unix socket using SCM_RIGHTS.
+
+    Args:
+        sock: Connected Unix socket.
+
+    Returns:
+        Received file descriptor.
+
+    Raises:
+        RuntimeError: If no file descriptor was received.
+    """
+    # NOTE: recvmsg() has no async equivalent
+    _, ancdata, _, _ = sock.recvmsg(
+        1, socket.CMSG_SPACE(array.array('i', [0]).itemsize))
+    if not ancdata:
+        raise RuntimeError('No file descriptor received - '
+                           'sender may have closed connection')
+    _, _, cmsg_data = ancdata[0]
+    return array.array('i', cmsg_data)[0]

sky/utils/kubernetes/generate_kubeconfig.sh

@@ -12,20 +12,20 @@
 #   * Specify SKYPILOT_NAMESPACE env var to override the default namespace where the service account is created.
 #   * Specify SKYPILOT_SA_NAME env var to override the default service account name.
 #   * Specify SKIP_SA_CREATION=1 to skip creating the service account and use an existing one
-#   * Specify SUPER_USER=1 to create a service account with cluster-admin permissions
+#   * Specify SUPER_USER=0 to create a service account with minimal permissions
 #
 # Usage:
-#   # Create "sky-sa" service account with minimal permissions in "default" namespace and generate kubeconfig
+#   # Create "sky-sa" service account in "default" namespace and generate kubeconfig
 #   $ ./generate_kubeconfig.sh
 #
-#   # Create "my-sa" service account with minimal permissions in "my-namespace" namespace and generate kubeconfig
+#   # Create "my-sa" service account in "my-namespace" namespace and generate kubeconfig
 #   $ SKYPILOT_SA_NAME=my-sa SKYPILOT_NAMESPACE=my-namespace ./generate_kubeconfig.sh
 #
 #   # Use an existing service account "my-sa" in "my-namespace" namespace and generate kubeconfig
 #   $ SKIP_SA_CREATION=1 SKYPILOT_SA_NAME=my-sa SKYPILOT_NAMESPACE=my-namespace ./generate_kubeconfig.sh
 #
-#   # Create "sky-sa" service account with cluster-admin permissions in "default" namespace
-#   $ SUPER_USER=1 ./generate_kubeconfig.sh
+#   # Create "sky-sa" service account with minimal permissions in "default" namespace (manual setup may be required)
+#   $ SUPER_USER=0 ./generate_kubeconfig.sh
 
 set -eu -o pipefail
 
@@ -33,11 +33,18 @@ set -eu -o pipefail
 # use default.
 SKYPILOT_SA=${SKYPILOT_SA_NAME:-sky-sa}
 NAMESPACE=${SKYPILOT_NAMESPACE:-default}
-SUPER_USER=${SUPER_USER:-0}
+SUPER_USER=${SUPER_USER:-1}
 
-echo "Service account: ${SKYPILOT_SA}"
-echo "Namespace: ${NAMESPACE}"
-echo "Super user permissions: ${SUPER_USER}"
+echo "=========================================="
+echo "SkyPilot Kubeconfig Generation"
+echo "=========================================="
+echo "Service Account: ${SKYPILOT_SA}"
+echo "Namespace:       ${NAMESPACE}"
+if [ "${SUPER_USER}" != "1" ]; then
+  echo "Permissions:     Minimal (manual setup may be required)"
+  SUPER_USER=0
+fi
+echo ""
 
 # Set OS specific values.
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
@@ -53,7 +60,7 @@ fi
 
 # If the user has set SKIP_SA_CREATION=1, skip creating the service account.
 if [ -z ${SKIP_SA_CREATION+x} ]; then
-  echo "Creating the Kubernetes Service Account with ${SUPER_USER:+super user}${SUPER_USER:-minimal} RBAC permissions."
+  echo "[1/3] Creating Kubernetes Service Account and RBAC permissions..."
   if [ "${SUPER_USER}" = "1" ]; then
     # Create service account with cluster-admin permissions
     kubectl apply -f - <<EOF
@@ -219,7 +226,8 @@ roleRef:
 EOF
   fi
 # Apply optional ingress-related roles, but don't make the script fail if it fails
-kubectl apply -f - <<EOF || echo "Failed to apply optional ingress-related roles. Nginx ingress is likely not installed. This is not critical and the script will continue."
+echo "      → Applying optional ingress permissions (skipped if ingress-nginx not installed)..."
+kubectl apply -f - 2>/dev/null <<EOF || true
 # Optional: Role for accessing ingress resources
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -253,8 +261,13 @@ roleRef:
   name: ${SKYPILOT_SA}-role-ingress-nginx  # Use the same name as the role at line 119
   apiGroup: rbac.authorization.k8s.io
 EOF
+else
+  echo "[1/3] Skipping service account creation (using existing account)..."
 fi
 
+echo ""
+echo "[2/3] Creating service account token..."
+
 # Checks if secret entry was defined for Service account. If defined it means that Kubernetes server has a
 # version bellow 1.24, otherwise one must manually create the secret and bind it to the Service account to have a non expiring token.
 # After Kubernetes v1.24 Service accounts no longer generate automatic tokens/secrets.
@@ -293,7 +306,9 @@ CURRENT_CONTEXT=$(kubectl config current-context)
 CURRENT_CLUSTER=$(kubectl config view -o jsonpath="{.contexts[?(@.name == \"${CURRENT_CONTEXT}\"})].context.cluster}")
 CURRENT_CLUSTER_ADDR=$(kubectl config view -o jsonpath="{.clusters[?(@.name == \"${CURRENT_CLUSTER}\"})].cluster.server}")
 
-echo "Writing kubeconfig."
+echo ""
+echo "[3/3] Generating kubeconfig file..."
+
 cat > kubeconfig <<EOF
 apiVersion: v1
 clusters:
@@ -316,24 +331,18 @@ users:
     token: ${SA_TOKEN}
 EOF
 
-echo "---
-Done!
-
-Kubeconfig using service account '${SKYPILOT_SA}' in namespace '${NAMESPACE}' written at $(pwd)/kubeconfig
-
-Copy the generated kubeconfig file to your ~/.kube/ directory to use it with
-kubectl and skypilot:
-
-# Backup your existing kubeconfig file
-mv ~/.kube/config ~/.kube/config.bak
-cp kubeconfig ~/.kube/config
-
-# Verify that you can access the cluster
-kubectl get pods
-
-Also add this to your ~/.sky/config.yaml to use the new service account:
-
-# ~/.sky/config.yaml
-kubernetes:
-  remote_identity: ${SKYPILOT_SA}
-"
+echo ""
+echo "=========================================="
+echo "✓ SUCCESS!"
+echo "=========================================="
+echo ""
+echo "Kubeconfig file created successfully!"
+echo ""
+echo "  Service Account: ${SKYPILOT_SA}"
+echo "  Namespace:       ${NAMESPACE}"
+echo "  Location:        $(pwd)/kubeconfig"
+echo ""
+echo "Next steps:"
+echo "  Refer to this page for setting up the credential for remote API server:"
+echo "  https://docs.skypilot.co/en/latest/reference/api-server/api-server-admin-deploy.html#optional-configure-cloud-accounts"
+echo ""

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -1,13 +1,11 @@
-"""Utility functions for deploying Kubernetes clusters."""
+"""Utility functions for deploying local Kubernetes kind clusters."""
 import os
 import random
 import shlex
 import subprocess
 import tempfile
 import textwrap
-from typing import List, Optional, Tuple
-
-import colorama
+from typing import Optional, Tuple
 
 from sky import check as sky_check
 from sky import sky_logging
@@ -20,7 +18,6 @@ from sky.utils import log_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
-from sky.utils.kubernetes import deploy_ssh_node_pools
 
 logger = sky_logging.init_logger(__name__)
 
@@ -32,95 +29,6 @@ LOCAL_CLUSTER_INTERNAL_PORT_START = 30000
 LOCAL_CLUSTER_INTERNAL_PORT_END = 30099
 
 
-def check_ssh_cluster_dependencies(
-        raise_error: bool = True) -> Optional[List[str]]:
-    """Checks if the dependencies for ssh cluster are installed.
-
-    Args:
-        raise_error: set to true when the dependency needs to be present.
-            set to false for `sky check`, where reason strings are compiled
-            at the end.
-
-    Returns: the reasons list if there are missing dependencies.
-    """
-    # error message
-    jq_message = ('`jq` is required to setup ssh cluster.')
-
-    # save
-    reasons = []
-    required_binaries = []
-
-    # Ensure jq is installed
-    try:
-        subprocess.run(['jq', '--version'],
-                       stdout=subprocess.DEVNULL,
-                       stderr=subprocess.DEVNULL,
-                       check=True)
-    except (FileNotFoundError, subprocess.CalledProcessError):
-        required_binaries.append('jq')
-        reasons.append(jq_message)
-
-    if required_binaries:
-        reasons.extend([
-            'On Debian/Ubuntu, install the missing dependenc(ies) with:',
-            f'  $ sudo apt install {" ".join(required_binaries)}',
-            'On MacOS, install with: ',
-            f'  $ brew install {" ".join(required_binaries)}',
-        ])
-        if raise_error:
-            with ux_utils.print_exception_no_traceback():
-                raise RuntimeError('\n'.join(reasons))
-        return reasons
-    return None
-
-
-def deploy_ssh_cluster(cleanup: bool = False,
-                       infra: Optional[str] = None,
-                       kubeconfig_path: Optional[str] = None):
-    """Deploy a Kubernetes cluster on SSH targets.
-
-    This function reads ~/.sky/ssh_node_pools.yaml and uses it to deploy a
-    Kubernetes cluster on the specified machines.
-
-    Args:
-        cleanup: Whether to clean up the cluster instead of deploying.
-        infra: Name of the cluster in ssh_node_pools.yaml to use.
-            If None, the first cluster in the file will be used.
-        kubeconfig_path: Path to save the Kubernetes configuration file.
-            If None, the default ~/.kube/config will be used.
-    """
-    check_ssh_cluster_dependencies()
-
-    action = 'Cleanup' if cleanup else 'Deployment'
-    msg_str = f'Initializing SSH Node Pools {action}...'
-
-    with rich_utils.safe_status(ux_utils.spinner_message(msg_str)):
-        try:
-            deploy_ssh_node_pools.deploy_clusters(
-                infra=infra, cleanup=cleanup, kubeconfig_path=kubeconfig_path)
-        except Exception as e:  # pylint: disable=broad-except
-            logger.error(str(e))
-            with ux_utils.print_exception_no_traceback():
-                raise RuntimeError(
-                    'Failed to deploy SkyPilot on some Node Pools.') from e
-
-    logger.info('')
-    if cleanup:
-        logger.info(
-            ux_utils.finishing_message(
-                '🎉 SSH Node Pools cleaned up successfully.'))
-    else:
-        logger.info(
-            ux_utils.finishing_message(
-                '🎉 SSH Node Pools set up successfully. ',
-                follow_up_message=(
-                    f'Run `{colorama.Style.BRIGHT}'
-                    f'sky check ssh'
-                    f'{colorama.Style.RESET_ALL}` to verify access, '
-                    f'`{colorama.Style.BRIGHT}sky launch --infra ssh'
-                    f'{colorama.Style.RESET_ALL}` to launch a cluster.')))
-
-
 def generate_kind_config(port_start: int,
                          num_nodes: int = 1,
                          gpus: bool = False) -> str:

sky/utils/kubernetes/rsync_helper.sh

@@ -60,4 +60,8 @@ fi
 # We wrap the command in a bash script that waits for rsync, then execs the original command.
 # Timeout after MAX_WAIT_TIME_SECONDS seconds.
 MAX_WAIT_TIME_SECONDS=300
-eval "${kubectl_cmd_base% --} -i -- bash -c 'count=0; max_count=$MAX_WAIT_TIME_SECONDS*2; until which rsync >/dev/null 2>&1; do if [ \$count -ge \$max_count ]; then echo \"Error when trying to rsync files to kubernetes cluster. Package installation may have failed.\" >&2; exit 1; fi; sleep 0.5; count=\$((count+1)); done; exec \"\$@\"' -- \"\$@\""
+MAX_WAIT_COUNT=$((MAX_WAIT_TIME_SECONDS * 2))
+# Use --norc --noprofile to prevent bash from sourcing startup files that might
+# output to stdout and corrupt the rsync protocol. All debug output must go to
+# stderr (>&2) to keep stdout clean for rsync communication.
+eval "${kubectl_cmd_base% --} -i -- bash --norc --noprofile -c 'count=0; until which rsync >/dev/null 2>&1; do if [ \$count -ge $MAX_WAIT_COUNT ]; then echo \"Error when trying to rsync files to kubernetes cluster. Package installation may have failed.\" >&2; exit 1; fi; sleep 0.5; count=\$((count+1)); done; exec \"\$@\"' -- \"\$@\""