skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/workspaces/server.py

@@ -0,0 +1,101 @@
+"""REST API for workspace management."""
+
+import fastapi
+
+from sky.server.requests import executor
+from sky.server.requests import payloads
+from sky.server.requests import request_names
+from sky.server.requests import requests as api_requests
+from sky.workspaces import core
+
+router = fastapi.APIRouter()
+
+
+@router.get('')
+# pylint: disable=redefined-builtin
+async def get(request: fastapi.Request) -> None:
+    """Gets workspace config on the server."""
+    # Have to manually inject user info into the request body because the
+    # request body is not available in the GET endpoint.
+    auth_user = request.state.auth_user
+    auth_user_env_vars_kwargs = {
+        'env_vars': auth_user.to_env_vars()
+    } if auth_user else {}
+    request_body = payloads.RequestBody(**auth_user_env_vars_kwargs)
+
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_GET,
+        request_body=request_body,
+        func=core.get_workspaces,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )
+
+
+@router.post('/update')
+async def update(request: fastapi.Request,
+                 update_workspace_body: payloads.UpdateWorkspaceBody) -> None:
+    """Updates a specific workspace configuration."""
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_UPDATE,
+        request_body=update_workspace_body,
+        func=core.update_workspace,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )
+
+
+@router.post('/create')
+async def create(request: fastapi.Request,
+                 create_workspace_body: payloads.CreateWorkspaceBody) -> None:
+    """Creates a new workspace configuration."""
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_CREATE,
+        request_body=create_workspace_body,
+        func=core.create_workspace,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )
+
+
+@router.post('/delete')
+async def delete(request: fastapi.Request,
+                 delete_workspace_body: payloads.DeleteWorkspaceBody) -> None:
+    """Deletes a workspace configuration."""
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_DELETE,
+        request_body=delete_workspace_body,
+        func=core.delete_workspace,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )
+
+
+@router.get('/config')
+async def get_config(request: fastapi.Request) -> None:
+    """Gets the entire SkyPilot configuration."""
+    auth_user = request.state.auth_user
+    auth_user_env_vars_kwargs = {
+        'env_vars': auth_user.to_env_vars()
+    } if auth_user else {}
+    get_config_body = payloads.GetConfigBody(**auth_user_env_vars_kwargs)
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_GET_CONFIG,
+        request_body=get_config_body,
+        func=core.get_config,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )
+
+
+@router.post('/config')
+async def update_config(request: fastapi.Request,
+                        update_config_body: payloads.UpdateConfigBody) -> None:
+    """Updates the entire SkyPilot configuration."""
+    await executor.schedule_request_async(
+        request_id=request.state.request_id,
+        request_name=request_names.RequestName.WORKSPACES_UPDATE_CONFIG,
+        request_body=update_config_body,
+        func=core.update_config,
+        schedule_type=api_requests.ScheduleType.SHORT,
+    )

sky/workspaces/utils.py

@@ -0,0 +1,56 @@
+"""Utils for workspaces."""
+import collections
+from typing import Any, Dict, List
+
+from sky import global_user_state
+from sky import sky_logging
+
+logger = sky_logging.init_logger(__name__)
+
+
+def get_workspace_users(workspace_config: Dict[str, Any]) -> List[str]:
+    """Get the users that should have access to a workspace.
+
+    workspace_config is a dict with the following keys:
+    - private: bool
+    - allowed_users: list of user names or IDs
+
+    This function will automatically resolve the user names to IDs.
+
+    Args:
+        workspace_config: The configuration of the workspace.
+
+    Returns:
+        List of user IDs that should have access to the workspace.
+        For private workspaces, returns specific user IDs.
+        For public workspaces, returns ['*'] to indicate all users.
+    """
+    if workspace_config.get('private', False):
+        user_ids = []
+        workspace_user_name_or_ids = workspace_config.get('allowed_users', [])
+        all_users = global_user_state.get_all_users()
+        all_user_ids = {user.id for user in all_users}
+        all_user_map = collections.defaultdict(list)
+        for user in all_users:
+            all_user_map[user.name].append(user.id)
+
+        # Resolve user names to IDs
+        for user_name_or_id in workspace_user_name_or_ids:
+            if user_name_or_id in all_user_ids:
+                user_ids.append(user_name_or_id)
+            elif user_name_or_id in all_user_map:
+                if len(all_user_map[user_name_or_id]) > 1:
+                    user_ids_str = ', '.join(all_user_map[user_name_or_id])
+                    raise ValueError(
+                        f'User {user_name_or_id!r} has multiple IDs: '
+                        f'{user_ids_str}. Please specify the user '
+                        f'ID instead.')
+                user_ids.append(all_user_map[user_name_or_id][0])
+            else:
+                logger.warning(
+                    f'User {user_name_or_id!r} not found in all users')
+                continue
+        return user_ids
+    else:
+        # Public workspace - return '*' to indicate all users should have access
+        return ['*']

sky_templates/README.md

@@ -0,0 +1,3 @@
+# SkyPilot Templates
+
+This package contains templates for users to use in their SkyPilot clusters, jobs, and services.

sky_templates/__init__.py

@@ -0,0 +1,3 @@
+"""SkyPilot templates."""
+
+__version__ = '1.0.0-dev0'

sky_templates/ray/start_cluster

@@ -0,0 +1,183 @@
+#!/bin/bash
+# Starts a Ray cluster on a SkyPilot cluster.
+#
+# This script starts a Ray cluster using default Ray ports (6379, 8265),
+# which are different from SkyPilot's system Ray ports (6380, 8266).
+# This allows users to run their own Ray applications independently of
+# SkyPilot's internal Ray cluster.
+#
+# Environment Variables:
+#   RAY_HEAD_PORT=6379                     - Ray head node port
+#   RAY_DASHBOARD_PORT=8265                - Ray dashboard port
+#   RAY_DASHBOARD_HOST=127.0.0.1           - Dashboard host (set to 0.0.0.0 to expose externally)
+#   RAY_DASHBOARD_AGENT_LISTEN_PORT=       - (Optional) Dashboard agent listen port
+#   RAY_HEAD_IP_ADDRESS=                   - (Optional) Node IP address
+#   RAY_CMD=ray                            - (Optional) Command to invoke Ray (e.g., "uv run ray")
+#
+# Usage:
+#   ~/sky_templates/ray/start_cluster
+#
+#   # With custom configurations
+#   export RAY_DASHBOARD_HOST=0.0.0.0
+#   export RAY_DASHBOARD_PORT=8280
+#   ~/sky_templates/ray/start_cluster
+#
+#   # With uv
+#   export RAY_CMD="uv run ray"
+#   ~/sky_templates/ray/start_cluster
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+RAY_HEAD_PORT=${RAY_HEAD_PORT:-6379}
+RAY_DASHBOARD_PORT=${RAY_DASHBOARD_PORT:-8265}
+RAY_DASHBOARD_HOST=${RAY_DASHBOARD_HOST:-127.0.0.1}
+RAY_DASHBOARD_AGENT_LISTEN_PORT=${RAY_DASHBOARD_AGENT_LISTEN_PORT:-}
+RAY_HEAD_IP_ADDRESS=${RAY_HEAD_IP_ADDRESS:-}
+
+RAY_CMD=${RAY_CMD:-ray}
+# Tokenize the command string into an array so multi-word commands
+# (e.g., "uv run ray") are handled safely when expanded later.
+eval "RAY_CMD_ARR=( ${RAY_CMD} )"
+
+# Convenience wrapper to invoke the configured Ray command with arbitrary args.
+run_ray() {
+    "${RAY_CMD_ARR[@]}" "$@"
+}
+
+echo -e "${GREEN}Starting Ray cluster...${NC}"
+
+# Ensure ray[default] is installed (we need [default] to do `ray list nodes`)
+# Pin to existing version if Ray is already installed to avoid upgrading existing version.
+RAY_VERSION=$(run_ray --version 2>/dev/null | cut -d' ' -f3 || echo "")
+if [ -n "${RAY_VERSION}" ]; then
+    # Pin to existing version.
+    VERSION_SPEC="==${RAY_VERSION}"
+else
+    echo -e "${YELLOW}Installing ray[default]...${NC}"
+    VERSION_SPEC=""
+fi
+
+# Pin click<8.3.0 to avoid incompatibility with Ray on Python 3.10
+# click 8.3.0 and 8.3.1 breaks Ray CLI due to deepcopy issues with sentinel values
+# See: https://github.com/ray-project/ray/issues/56747
+# TODO(kevin): Remove this once the issue is fixed in a future click release
+RAY_INSTALL_SPEC="ray[default]${VERSION_SPEC} click<8.3.0"
+uv pip install ${RAY_INSTALL_SPEC} || uv pip install --system ${RAY_INSTALL_SPEC}
+
+# Verify Ray is working
+if ! run_ray --version > /dev/null; then
+    echo -e "${RED}Error: Failed to install Ray.${NC}"
+    exit 1
+fi
+echo -e "${GREEN}Ray $(run_ray --version | cut -d' ' -f3) is installed.${NC}"
+
+RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+if [ "${SKYPILOT_NODE_RANK}" -ne 0 ]; then
+    HEAD_IP=$(echo "${SKYPILOT_NODE_IPS}" | head -n1)
+    RAY_ADDRESS="${HEAD_IP}:${RAY_HEAD_PORT}"
+fi
+
+# Check if user-space Ray is already running
+if run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${YELLOW}Ray cluster is already running.${NC}"
+    run_ray status --address="${RAY_ADDRESS}"
+    exit 0
+fi
+
+TIMEOUT=300
+
+if [ "${SKYPILOT_NODE_RANK}" -eq 0 ]; then
+    echo -e "${GREEN}Starting Ray head node...${NC}"
+
+    RAY_START_CMD="start --head \
+        --port=${RAY_HEAD_PORT} \
+        --dashboard-port=${RAY_DASHBOARD_PORT} \
+        --dashboard-host=${RAY_DASHBOARD_HOST} \
+        --disable-usage-stats \
+        --include-dashboard=True"
+
+    # Add --num-gpus only if > 0
+    if [ "${SKYPILOT_NUM_GPUS_PER_NODE}" -gt 0 ]; then
+        RAY_START_CMD="${RAY_START_CMD} --num-gpus=${SKYPILOT_NUM_GPUS_PER_NODE}"
+    fi
+
+    # Add optional dashboard agent listen port if specified
+    if [ -n "${RAY_DASHBOARD_AGENT_LISTEN_PORT}" ]; then
+        RAY_START_CMD="${RAY_START_CMD} --dashboard-agent-listen-port=${RAY_DASHBOARD_AGENT_LISTEN_PORT}"
+    fi
+
+    # Add optional node IP address if specified
+    if [ -n "${RAY_HEAD_IP_ADDRESS}" ]; then
+        RAY_START_CMD="${RAY_START_CMD} --node-ip-address=${RAY_HEAD_IP_ADDRESS}"
+    fi
+
+    run_ray ${RAY_START_CMD}
+
+    start_time=$(date +%s)
+    while ! run_ray health-check --address="${RAY_ADDRESS}" &>/dev/null; do
+        if [ "$(( $(date +%s) - start_time ))" -ge "$TIMEOUT" ]; then
+            echo -e "${RED}Timed out waiting for head node. Exiting.${NC}" >&2
+            exit 1
+        fi
+        echo "Head node not healthy yet. Retrying in 1s..."
+        sleep 1
+    done
+
+    echo -e "${GREEN}Head node started successfully.${NC}"
+
+    # Wait for all worker nodes to join
+    if [ "${SKYPILOT_NUM_NODES}" -gt 1 ]; then
+        echo "Waiting for all ${SKYPILOT_NUM_NODES} nodes to join..."
+        start_time=$(date +%s)
+        while true; do
+            if [ "$(( $(date +%s) - start_time ))" -ge "${TIMEOUT}" ]; then
+                echo -e "${RED}Error: Timeout waiting for nodes.${NC}" >&2
+                exit 1
+            fi
+            ready_nodes=$(run_ray list nodes --format=json | python3 -c "import sys, json; print(len(json.load(sys.stdin)))")
+            if [ "${ready_nodes}" -ge "${SKYPILOT_NUM_NODES}" ]; then
+                break
+            fi
+            echo "Waiting... (${ready_nodes} / ${SKYPILOT_NUM_NODES} nodes ready)"
+            sleep 5
+        done
+        echo -e "${GREEN}All ${SKYPILOT_NUM_NODES} nodes have joined.${NC}"
+    fi
+
+    # Add sleep to after `ray start` to give ray enough time to daemonize
+    sleep 5
+else
+    echo -e "${GREEN}Starting Ray worker node...${NC}"
+
+    echo "Waiting for head node at ${RAY_ADDRESS}..."
+    start_time=$(date +%s)
+    while ! run_ray health-check --address="${RAY_ADDRESS}" &>/dev/null; do
+        if [ "$(( $(date +%s) - start_time ))" -ge "$TIMEOUT" ]; then
+            echo -e "${RED}Timed out waiting for head node. Exiting.${NC}" >&2
+            exit 1
+        fi
+        echo "Head node not healthy yet. Retrying in 1s..."
+        sleep 1
+    done
+
+    echo -e "${GREEN}Head node is healthy. Starting worker node...${NC}"
+    WORKER_CMD="start --address=${RAY_ADDRESS} --disable-usage-stats"
+
+    # Add --num-gpus only if > 0
+    if [ "${SKYPILOT_NUM_GPUS_PER_NODE}" -gt 0 ]; then
+        WORKER_CMD="${WORKER_CMD} --num-gpus=${SKYPILOT_NUM_GPUS_PER_NODE}"
+    fi
+
+    run_ray ${WORKER_CMD}
+
+    echo -e "${GREEN}Worker node started successfully.${NC}"
+
+    # Add sleep to after `ray start` to give ray enough time to daemonize
+    sleep 5
+fi

sky_templates/ray/stop_cluster

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Stops a user Ray cluster on a SkyPilot cluster.
+#
+# This script stops a Ray cluster running on custom ports (default 6379),
+# which is separate from SkyPilot's internal Ray cluster (port 6380).
+#
+# IMPORTANT: This script uses pkill to stop Ray processes, NOT 'ray stop',
+# as 'ray stop' can interfere with SkyPilot's internal operations.
+#
+# Environment Variables:
+#   RAY_HEAD_PORT=6379                     - Ray head node port to stop
+#   RAY_CMD=ray                            - (Optional) Command to invoke Ray (e.g., "uv run ray")
+#
+# Usage:
+#   # Stop default Ray cluster (port 6379)
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+#
+#   # Stop Ray cluster on custom port
+#   export RAY_HEAD_PORT=6385
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+#
+#   # With uv
+#   export RAY_CMD="uv run ray"
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+RAY_HEAD_PORT=${RAY_HEAD_PORT:-6379}
+RAY_CMD=${RAY_CMD:-ray}
+# Tokenize the command string into an array so multi-word commands (e.g., "uv run ray")
+# are handled safely when expanded later.
+eval "RAY_CMD_ARR=( ${RAY_CMD} )"
+
+run_ray() {
+    "${RAY_CMD_ARR[@]}" "$@"
+}
+
+echo -e "${GREEN}Stopping Ray cluster on port ${RAY_HEAD_PORT}...${NC}"
+
+RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+if [ "$SKYPILOT_NODE_RANK" -ne 0 ]; then
+    HEAD_IP=$(echo "$SKYPILOT_NODE_IPS" | head -n1)
+    RAY_ADDRESS="${HEAD_IP}:${RAY_HEAD_PORT}"
+fi
+
+# Check if Ray is running
+if ! run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${YELLOW}No Ray cluster found running on port ${RAY_HEAD_PORT}.${NC}"
+    exit 0
+fi
+
+# Use pkill to stop Ray processes instead of 'ray stop'
+# This prevents interfering with SkyPilot's internal Ray cluster (port 6380)
+echo -e "${YELLOW}Killing Ray processes on port ${RAY_HEAD_PORT}...${NC}"
+
+pkill -f "ray.*[=:]${RAY_HEAD_PORT}" || true
+
+echo -e "${GREEN}Ray processes killed.${NC}"
+# Wait a moment for processes to terminate
+sleep 5
+
+# Verify Ray is stopped
+if run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${RED}Warning: Ray cluster may still be running. Try manually:${NC}"
+    echo -e "${RED}  pkill -9 -f 'ray.*[=:]${RAY_HEAD_PORT}'${NC}"
+    exit 1
+else
+    echo -e "${GREEN}Ray cluster successfully stopped.${NC}"
+fi