skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/volumes/volume.py

@@ -13,6 +13,10 @@ VOLUME_TYPE_TO_CLOUD = {
     volume_lib.VolumeType.PVC: clouds.Kubernetes(),
     volume_lib.VolumeType.RUNPOD_NETWORK_VOLUME: clouds.RunPod(),
 }
+CLOUD_TO_VOLUME_TYPE = {
+    clouds.Kubernetes(): [volume_lib.VolumeType.PVC],
+    clouds.RunPod(): [volume_lib.VolumeType.RUNPOD_NETWORK_VOLUME],
+}
 
 
 class Volume:
@@ -25,7 +29,7 @@ class Volume:
             infra: Optional[str] = None,
             size: Optional[str] = None,
             labels: Optional[Dict[str, str]] = None,
-            resource_name: Optional[str] = None,
+            use_existing: Optional[bool] = None,
             config: Optional[Dict[str, Any]] = None):
         """Initialize a Volume instance.
 
@@ -35,6 +39,7 @@ class Volume:
             infra: Infrastructure specification
             size: Volume size
             labels: Volume labels
+            use_existing: Whether to use an existing volume
             config: Additional configuration
         """
         self.name = name
@@ -42,7 +47,7 @@ class Volume:
         self.infra = infra
         self.size = size
         self.labels = labels or {}
-        self.resource_name = resource_name
+        self.use_existing = use_existing
         self.config = config or {}
 
         self.cloud: Optional[str] = None
@@ -70,17 +75,16 @@ class Volume:
                              infra=config.get('infra'),
                              size=config.get('size'),
                              labels=config.get('labels'),
-                             resource_name=config.get('resource_name'),
+                             use_existing=config.get('use_existing'),
                              config=config.get('config', {}))
         if vt == volume_lib.VolumeType.RUNPOD_NETWORK_VOLUME:
-            return RunpodNetworkVolume(
-                name=config.get('name'),
-                type=vol_type_val,
-                infra=config.get('infra'),
-                size=config.get('size'),
-                labels=config.get('labels'),
-                resource_name=config.get('resource_name'),
-                config=config.get('config', {}))
+            return RunpodNetworkVolume(name=config.get('name'),
+                                       type=vol_type_val,
+                                       infra=config.get('infra'),
+                                       size=config.get('size'),
+                                       labels=config.get('labels'),
+                                       use_existing=config.get('use_existing'),
+                                       config=config.get('config', {}))
 
         raise ValueError(f'Invalid volume type: {vol_type_val}')
 
@@ -92,7 +96,7 @@ class Volume:
             'infra': self.infra,
             'size': self.size,
             'labels': self.labels,
-            'resource_name': self.resource_name,
+            'use_existing': self.use_existing,
             'config': self.config,
             'cloud': self.cloud,
             'region': self.region,
@@ -100,7 +104,7 @@ class Volume:
         }
 
     def _normalize_config(self) -> None:
-        """Adjust and validate the config."""
+        """Normalize and validate the config."""
         # Validate schema
         common_utils.validate_schema(self.to_yaml_config(),
                                      schemas.get_volume_schema(),
@@ -115,8 +119,17 @@ class Volume:
         self.region = infra_info.region
         self.zone = infra_info.zone
 
-        # Validate the volume config
-        self._validate_config()
+        # Set cloud from volume type if not specified
+        cloud_obj_from_type = VOLUME_TYPE_TO_CLOUD.get(
+            volume_lib.VolumeType(self.type))
+        if self.cloud:
+            cloud_obj = registry.CLOUD_REGISTRY.from_str(self.cloud)
+            assert cloud_obj is not None
+            if not cloud_obj.is_same_cloud(cloud_obj_from_type):
+                raise ValueError(
+                    f'Invalid cloud {self.cloud} for volume type {self.type}')
+        else:
+            self.cloud = str(cloud_obj_from_type)
 
     def _adjust_config(self) -> None:
         """Adjust the volume config (e.g., parse size)."""
@@ -132,41 +145,41 @@ class Volume:
         except ValueError as e:
             raise ValueError(f'Invalid size {self.size}: {e}') from e
 
-    def _validate_config(self) -> None:
-        """Validate the volume config."""
-        cloud_obj_from_type = VOLUME_TYPE_TO_CLOUD.get(
-            volume_lib.VolumeType(self.type))
-        if self.cloud:
-            cloud_obj = registry.CLOUD_REGISTRY.from_str(self.cloud)
-            assert cloud_obj is not None
-            if not cloud_obj.is_same_cloud(cloud_obj_from_type):
-                raise ValueError(
-                    f'Invalid cloud {self.cloud} for volume type {self.type}')
-        else:
-            self.cloud = str(cloud_obj_from_type)
-            cloud_obj = cloud_obj_from_type
-            assert cloud_obj is not None
+    def validate(self, skip_cloud_compatibility: bool = False) -> None:
+        """Validates the volume."""
+        self.validate_name()
+        self.validate_size()
+        if not skip_cloud_compatibility:
+            self.validate_cloud_compatibility()
+        # Extra, type-specific validations
+        self._validate_config_extra()
 
-        self.region, self.zone = cloud_obj.validate_region_zone(
-            self.region, self.zone)
+    def validate_name(self) -> None:
+        """Validates if the volume name is set."""
+        assert self.name is not None, 'Volume name must be set'
+
+    def validate_size(self) -> None:
+        """Validates that size is specified for new volumes."""
+        if not self.use_existing and not self.size:
+            raise ValueError('Size is required for new volumes. '
+                             'Please specify the size in the YAML file or '
+                             'use the --size flag.')
+
+    def validate_cloud_compatibility(self) -> None:
+        """Validates region, zone, name, labels with the cloud."""
+        cloud_obj = registry.CLOUD_REGISTRY.from_str(self.cloud)
+        assert cloud_obj is not None
 
         valid, err_msg = cloud_obj.is_volume_name_valid(self.name)
         if not valid:
             raise ValueError(f'Invalid volume name: {err_msg}')
 
-        if not self.resource_name and not self.size:
-            raise ValueError('Size is required for new volumes. '
-                             'Please specify the size in the YAML file or '
-                             'use the --size flag.')
         if self.labels:
             for key, value in self.labels.items():
                 valid, err_msg = cloud_obj.is_label_valid(key, value)
                 if not valid:
                     raise ValueError(f'{err_msg}')
 
-        # Extra, type-specific validations
-        self._validate_config_extra()
-
     # Hook methods for subclasses
     def _validate_config_extra(self) -> None:
         """Additional type-specific validation.
@@ -185,7 +198,7 @@ class RunpodNetworkVolume(Volume):
     """RunPod Network Volume."""
 
     def _validate_config_extra(self) -> None:
-        if self.size is not None:
+        if not self.use_existing and self.size is not None:
             try:
                 size_int = int(self.size)
                 if size_int < volume_lib.MIN_RUNPOD_NETWORK_VOLUME_SIZE_GB:
@@ -196,8 +209,7 @@ class RunpodNetworkVolume(Volume):
                 raise ValueError(f'Invalid volume size {self.size!r}: '
                                  f'{e}') from e
         if not self.zone:
-            raise ValueError(
-                'RunPod DataCenterId is required to create a network '
-                'volume. Set the zone in the infra field.')
+            raise ValueError('RunPod DataCenterId is required for network '
+                             'volumes. Set the zone in the infra field.')
 
         return

sky/workspaces/core.py

@@ -14,6 +14,7 @@ from sky.backends import backend_utils
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.users import permission
+from sky.users import rbac
 from sky.utils import annotations
 from sky.utils import common_utils
 from sky.utils import config_utils
@@ -147,11 +148,15 @@ def _compare_workspace_configs(
     private_new = new_config.get('private', False)
     private_changed = private_old != private_new
 
+    admin_user_ids = permission.permission_service.get_users_for_role(
+        rbac.RoleName.ADMIN.value)
     # Get allowed users (resolve to user IDs for comparison)
     allowed_users_old = workspaces_utils.get_workspace_users(
         current_config) if private_old else []
+    allowed_users_old += admin_user_ids
     allowed_users_new = workspaces_utils.get_workspace_users(
         new_config) if private_new else []
+    allowed_users_new += admin_user_ids
 
     # Convert to sets for easier comparison
     old_users_set = set(allowed_users_old)
@@ -188,6 +193,24 @@ def _compare_workspace_configs(
         added_users=added_users)
 
 
+def _validate_workspace_config_changes_with_lock(
+        workspace_name: str, current_config: Dict[str, Any],
+        new_config: Dict[str, Any]) -> None:
+    lock_id = backend_utils.workspace_lock_id(workspace_name)
+    lock_timeout = backend_utils.WORKSPACE_LOCK_TIMEOUT_SECONDS
+    try:
+        with locks.get_lock(lock_id, lock_timeout):
+            # Validate the configuration changes based on active resources
+            _validate_workspace_config_changes(workspace_name, current_config,
+                                               new_config)
+    except locks.LockTimeout as e:
+        raise RuntimeError(
+            f'Failed to validate workspace {workspace_name!r} due to '
+            'a timeout when trying to access database. Please '
+            f'try again or manually remove the lock at {lock_id}. '
+            f'{common_utils.format_exception(e)}') from None
+
+
 def _validate_workspace_config_changes(workspace_name: str,
                                        current_config: Dict[str, Any],
                                        new_config: Dict[str, Any]) -> None:
@@ -232,7 +255,7 @@ def _validate_workspace_config_changes(workspace_name: str,
                     f' private. Checking that all active resources belong'
                     f' to allowed users.')
 
-                error_summary, missed_users_names = (
+                error_summary, missed_users_names, _ = (
                     resource_checker.check_users_workspaces_active_resources(
                         config_comparison.allowed_users_new, [workspace_name]))
                 if error_summary:
@@ -259,11 +282,35 @@ def _validate_workspace_config_changes(workspace_name: str,
                     f'Checking that removed users'
                     f' {config_comparison.removed_users} do not have'
                     f' active resources in workspace {workspace_name!r}.')
-                user_operations = []
-                for user_id in config_comparison.removed_users:
-                    user_operations.append((user_id, 'remove'))
-                resource_checker.check_no_active_resources_for_users(
-                    user_operations)
+                error_summary, missed_users_names, missed_user_dict = (
+                    resource_checker.check_users_workspaces_active_resources(
+                        config_comparison.allowed_users_new, [workspace_name]))
+                if error_summary:
+                    error_user_ids = []
+                    for user_id in config_comparison.removed_users:
+                        if user_id in missed_user_dict:
+                            error_user_ids.append(user_id)
+                    error_user_names = []
+                    if error_user_ids:
+                        error_user_names = [
+                            missed_user_dict[user_id]
+                            for user_id in error_user_ids
+                        ]
+
+                    error_msg = 'Cannot '
+                    error_users_list = ', '.join(error_user_names)
+                    if len(error_user_names) == 1:
+                        error_msg += f'remove user {error_users_list!r} ' \
+                        f'from workspace {workspace_name!r} because the ' \
+                        f'user has {error_summary}'
+                    else:
+                        error_msg += f'remove users {error_users_list!r}' \
+                        f' from workspace {workspace_name!r} because the' \
+                        f' users have {error_summary}'
+                    error_msg += ', but not in the allowed_users list.' \
+                    ' Please either add the users to allowed_users or' \
+                    ' ask them to terminate their resources.'
+                    raise ValueError(error_msg)
     else:
         # Other configuration changes - check that workspace has no active
         # resources
@@ -310,20 +357,8 @@ def update_workspace(workspace_name: str, config: Dict[str,
                                                     default_value={})
     current_config = current_workspaces.get(workspace_name, {})
 
-    if current_config:
-        lock_id = backend_utils.workspace_lock_id(workspace_name)
-        lock_timeout = backend_utils.WORKSPACE_LOCK_TIMEOUT_SECONDS
-        try:
-            with locks.get_lock(lock_id, lock_timeout):
-                # Validate the configuration changes based on active resources
-                _validate_workspace_config_changes(workspace_name,
-                                                   current_config, config)
-        except locks.LockTimeout as e:
-            raise RuntimeError(
-                f'Failed to validate workspace {workspace_name!r} due to '
-                'a timeout when trying to access database. Please '
-                f'try again or manually remove the lock at {lock_id}. '
-                f'{common_utils.format_exception(e)}') from None
+    _validate_workspace_config_changes_with_lock(workspace_name, current_config,
+                                                 config)
 
     def update_workspace_fn(workspaces: Dict[str, Any]) -> None:
         """Function to update workspace inside the lock."""
@@ -510,7 +545,8 @@ def update_config(config: Dict[str, Any]) -> Dict[str, Any]:
         # If workspace configuration is changing, validate and mark for checking
         if current_workspace_config != new_workspace_config:
             _validate_workspace_config(workspace_name, new_workspace_config)
-            workspaces_to_check.append((workspace_name, 'update'))
+            _validate_workspace_config_changes_with_lock(
+                workspace_name, current_workspace_config, new_workspace_config)
             users = workspaces_utils.get_workspace_users(new_workspace_config)
             workspaces_to_check_policy['update'][workspace_name] = users
 

sky/workspaces/server.py

@@ -4,6 +4,7 @@ import fastapi
 
 from sky.server.requests import executor
 from sky.server.requests import payloads
+from sky.server.requests import request_names
 from sky.server.requests import requests as api_requests
 from sky.workspaces import core
 
@@ -22,9 +23,9 @@ async def get(request: fastapi.Request) -> None:
     } if auth_user else {}
     request_body = payloads.RequestBody(**auth_user_env_vars_kwargs)
 
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.get',
+        request_name=request_names.RequestName.WORKSPACES_GET,
         request_body=request_body,
         func=core.get_workspaces,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -35,9 +36,9 @@ async def get(request: fastapi.Request) -> None:
 async def update(request: fastapi.Request,
                  update_workspace_body: payloads.UpdateWorkspaceBody) -> None:
     """Updates a specific workspace configuration."""
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.update',
+        request_name=request_names.RequestName.WORKSPACES_UPDATE,
         request_body=update_workspace_body,
         func=core.update_workspace,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -48,9 +49,9 @@ async def update(request: fastapi.Request,
 async def create(request: fastapi.Request,
                  create_workspace_body: payloads.CreateWorkspaceBody) -> None:
     """Creates a new workspace configuration."""
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.create',
+        request_name=request_names.RequestName.WORKSPACES_CREATE,
         request_body=create_workspace_body,
         func=core.create_workspace,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -61,9 +62,9 @@ async def create(request: fastapi.Request,
 async def delete(request: fastapi.Request,
                  delete_workspace_body: payloads.DeleteWorkspaceBody) -> None:
     """Deletes a workspace configuration."""
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.delete',
+        request_name=request_names.RequestName.WORKSPACES_DELETE,
         request_body=delete_workspace_body,
         func=core.delete_workspace,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -78,9 +79,9 @@ async def get_config(request: fastapi.Request) -> None:
         'env_vars': auth_user.to_env_vars()
     } if auth_user else {}
     get_config_body = payloads.GetConfigBody(**auth_user_env_vars_kwargs)
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.get_config',
+        request_name=request_names.RequestName.WORKSPACES_GET_CONFIG,
         request_body=get_config_body,
         func=core.get_config,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -91,9 +92,9 @@ async def get_config(request: fastapi.Request) -> None:
 async def update_config(request: fastapi.Request,
                         update_config_body: payloads.UpdateConfigBody) -> None:
     """Updates the entire SkyPilot configuration."""
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='workspaces.update_config',
+        request_name=request_names.RequestName.WORKSPACES_UPDATE_CONFIG,
         request_body=update_config_body,
         func=core.update_config,
         schedule_type=api_requests.ScheduleType.SHORT,

sky_templates/README.md

@@ -0,0 +1,3 @@
+# SkyPilot Templates
+
+This package contains templates for users to use in their SkyPilot clusters, jobs, and services.

sky_templates/__init__.py

@@ -0,0 +1,3 @@
+"""SkyPilot templates."""
+
+__version__ = '1.0.0-dev0'

sky_templates/ray/start_cluster

@@ -0,0 +1,183 @@
+#!/bin/bash
+# Starts a Ray cluster on a SkyPilot cluster.
+#
+# This script starts a Ray cluster using default Ray ports (6379, 8265),
+# which are different from SkyPilot's system Ray ports (6380, 8266).
+# This allows users to run their own Ray applications independently of
+# SkyPilot's internal Ray cluster.
+#
+# Environment Variables:
+#   RAY_HEAD_PORT=6379                     - Ray head node port
+#   RAY_DASHBOARD_PORT=8265                - Ray dashboard port
+#   RAY_DASHBOARD_HOST=127.0.0.1           - Dashboard host (set to 0.0.0.0 to expose externally)
+#   RAY_DASHBOARD_AGENT_LISTEN_PORT=       - (Optional) Dashboard agent listen port
+#   RAY_HEAD_IP_ADDRESS=                   - (Optional) Node IP address
+#   RAY_CMD=ray                            - (Optional) Command to invoke Ray (e.g., "uv run ray")
+#
+# Usage:
+#   ~/sky_templates/ray/start_cluster
+#
+#   # With custom configurations
+#   export RAY_DASHBOARD_HOST=0.0.0.0
+#   export RAY_DASHBOARD_PORT=8280
+#   ~/sky_templates/ray/start_cluster
+#
+#   # With uv
+#   export RAY_CMD="uv run ray"
+#   ~/sky_templates/ray/start_cluster
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+RAY_HEAD_PORT=${RAY_HEAD_PORT:-6379}
+RAY_DASHBOARD_PORT=${RAY_DASHBOARD_PORT:-8265}
+RAY_DASHBOARD_HOST=${RAY_DASHBOARD_HOST:-127.0.0.1}
+RAY_DASHBOARD_AGENT_LISTEN_PORT=${RAY_DASHBOARD_AGENT_LISTEN_PORT:-}
+RAY_HEAD_IP_ADDRESS=${RAY_HEAD_IP_ADDRESS:-}
+
+RAY_CMD=${RAY_CMD:-ray}
+# Tokenize the command string into an array so multi-word commands
+# (e.g., "uv run ray") are handled safely when expanded later.
+eval "RAY_CMD_ARR=( ${RAY_CMD} )"
+
+# Convenience wrapper to invoke the configured Ray command with arbitrary args.
+run_ray() {
+    "${RAY_CMD_ARR[@]}" "$@"
+}
+
+echo -e "${GREEN}Starting Ray cluster...${NC}"
+
+# Ensure ray[default] is installed (we need [default] to do `ray list nodes`)
+# Pin to existing version if Ray is already installed to avoid upgrading existing version.
+RAY_VERSION=$(run_ray --version 2>/dev/null | cut -d' ' -f3 || echo "")
+if [ -n "${RAY_VERSION}" ]; then
+    # Pin to existing version.
+    VERSION_SPEC="==${RAY_VERSION}"
+else
+    echo -e "${YELLOW}Installing ray[default]...${NC}"
+    VERSION_SPEC=""
+fi
+
+# Pin click<8.3.0 to avoid incompatibility with Ray on Python 3.10
+# click 8.3.0 and 8.3.1 breaks Ray CLI due to deepcopy issues with sentinel values
+# See: https://github.com/ray-project/ray/issues/56747
+# TODO(kevin): Remove this once the issue is fixed in a future click release
+RAY_INSTALL_SPEC="ray[default]${VERSION_SPEC} click<8.3.0"
+uv pip install ${RAY_INSTALL_SPEC} || uv pip install --system ${RAY_INSTALL_SPEC}
+
+# Verify Ray is working
+if ! run_ray --version > /dev/null; then
+    echo -e "${RED}Error: Failed to install Ray.${NC}"
+    exit 1
+fi
+echo -e "${GREEN}Ray $(run_ray --version | cut -d' ' -f3) is installed.${NC}"
+
+RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+if [ "${SKYPILOT_NODE_RANK}" -ne 0 ]; then
+    HEAD_IP=$(echo "${SKYPILOT_NODE_IPS}" | head -n1)
+    RAY_ADDRESS="${HEAD_IP}:${RAY_HEAD_PORT}"
+fi
+
+# Check if user-space Ray is already running
+if run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${YELLOW}Ray cluster is already running.${NC}"
+    run_ray status --address="${RAY_ADDRESS}"
+    exit 0
+fi
+
+TIMEOUT=300
+
+if [ "${SKYPILOT_NODE_RANK}" -eq 0 ]; then
+    echo -e "${GREEN}Starting Ray head node...${NC}"
+
+    RAY_START_CMD="start --head \
+        --port=${RAY_HEAD_PORT} \
+        --dashboard-port=${RAY_DASHBOARD_PORT} \
+        --dashboard-host=${RAY_DASHBOARD_HOST} \
+        --disable-usage-stats \
+        --include-dashboard=True"
+
+    # Add --num-gpus only if > 0
+    if [ "${SKYPILOT_NUM_GPUS_PER_NODE}" -gt 0 ]; then
+        RAY_START_CMD="${RAY_START_CMD} --num-gpus=${SKYPILOT_NUM_GPUS_PER_NODE}"
+    fi
+
+    # Add optional dashboard agent listen port if specified
+    if [ -n "${RAY_DASHBOARD_AGENT_LISTEN_PORT}" ]; then
+        RAY_START_CMD="${RAY_START_CMD} --dashboard-agent-listen-port=${RAY_DASHBOARD_AGENT_LISTEN_PORT}"
+    fi
+
+    # Add optional node IP address if specified
+    if [ -n "${RAY_HEAD_IP_ADDRESS}" ]; then
+        RAY_START_CMD="${RAY_START_CMD} --node-ip-address=${RAY_HEAD_IP_ADDRESS}"
+    fi
+
+    run_ray ${RAY_START_CMD}
+
+    start_time=$(date +%s)
+    while ! run_ray health-check --address="${RAY_ADDRESS}" &>/dev/null; do
+        if [ "$(( $(date +%s) - start_time ))" -ge "$TIMEOUT" ]; then
+            echo -e "${RED}Timed out waiting for head node. Exiting.${NC}" >&2
+            exit 1
+        fi
+        echo "Head node not healthy yet. Retrying in 1s..."
+        sleep 1
+    done
+
+    echo -e "${GREEN}Head node started successfully.${NC}"
+
+    # Wait for all worker nodes to join
+    if [ "${SKYPILOT_NUM_NODES}" -gt 1 ]; then
+        echo "Waiting for all ${SKYPILOT_NUM_NODES} nodes to join..."
+        start_time=$(date +%s)
+        while true; do
+            if [ "$(( $(date +%s) - start_time ))" -ge "${TIMEOUT}" ]; then
+                echo -e "${RED}Error: Timeout waiting for nodes.${NC}" >&2
+                exit 1
+            fi
+            ready_nodes=$(run_ray list nodes --format=json | python3 -c "import sys, json; print(len(json.load(sys.stdin)))")
+            if [ "${ready_nodes}" -ge "${SKYPILOT_NUM_NODES}" ]; then
+                break
+            fi
+            echo "Waiting... (${ready_nodes} / ${SKYPILOT_NUM_NODES} nodes ready)"
+            sleep 5
+        done
+        echo -e "${GREEN}All ${SKYPILOT_NUM_NODES} nodes have joined.${NC}"
+    fi
+
+    # Add sleep to after `ray start` to give ray enough time to daemonize
+    sleep 5
+else
+    echo -e "${GREEN}Starting Ray worker node...${NC}"
+
+    echo "Waiting for head node at ${RAY_ADDRESS}..."
+    start_time=$(date +%s)
+    while ! run_ray health-check --address="${RAY_ADDRESS}" &>/dev/null; do
+        if [ "$(( $(date +%s) - start_time ))" -ge "$TIMEOUT" ]; then
+            echo -e "${RED}Timed out waiting for head node. Exiting.${NC}" >&2
+            exit 1
+        fi
+        echo "Head node not healthy yet. Retrying in 1s..."
+        sleep 1
+    done
+
+    echo -e "${GREEN}Head node is healthy. Starting worker node...${NC}"
+    WORKER_CMD="start --address=${RAY_ADDRESS} --disable-usage-stats"
+
+    # Add --num-gpus only if > 0
+    if [ "${SKYPILOT_NUM_GPUS_PER_NODE}" -gt 0 ]; then
+        WORKER_CMD="${WORKER_CMD} --num-gpus=${SKYPILOT_NUM_GPUS_PER_NODE}"
+    fi
+
+    run_ray ${WORKER_CMD}
+
+    echo -e "${GREEN}Worker node started successfully.${NC}"
+
+    # Add sleep to after `ray start` to give ray enough time to daemonize
+    sleep 5
+fi

sky_templates/ray/stop_cluster

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Stops a user Ray cluster on a SkyPilot cluster.
+#
+# This script stops a Ray cluster running on custom ports (default 6379),
+# which is separate from SkyPilot's internal Ray cluster (port 6380).
+#
+# IMPORTANT: This script uses pkill to stop Ray processes, NOT 'ray stop',
+# as 'ray stop' can interfere with SkyPilot's internal operations.
+#
+# Environment Variables:
+#   RAY_HEAD_PORT=6379                     - Ray head node port to stop
+#   RAY_CMD=ray                            - (Optional) Command to invoke Ray (e.g., "uv run ray")
+#
+# Usage:
+#   # Stop default Ray cluster (port 6379)
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+#
+#   # Stop Ray cluster on custom port
+#   export RAY_HEAD_PORT=6385
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+#
+#   # With uv
+#   export RAY_CMD="uv run ray"
+#   ~/sky_templates/ray/stop_ray_cluster.sh
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+RAY_HEAD_PORT=${RAY_HEAD_PORT:-6379}
+RAY_CMD=${RAY_CMD:-ray}
+# Tokenize the command string into an array so multi-word commands (e.g., "uv run ray")
+# are handled safely when expanded later.
+eval "RAY_CMD_ARR=( ${RAY_CMD} )"
+
+run_ray() {
+    "${RAY_CMD_ARR[@]}" "$@"
+}
+
+echo -e "${GREEN}Stopping Ray cluster on port ${RAY_HEAD_PORT}...${NC}"
+
+RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+if [ "$SKYPILOT_NODE_RANK" -ne 0 ]; then
+    HEAD_IP=$(echo "$SKYPILOT_NODE_IPS" | head -n1)
+    RAY_ADDRESS="${HEAD_IP}:${RAY_HEAD_PORT}"
+fi
+
+# Check if Ray is running
+if ! run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${YELLOW}No Ray cluster found running on port ${RAY_HEAD_PORT}.${NC}"
+    exit 0
+fi
+
+# Use pkill to stop Ray processes instead of 'ray stop'
+# This prevents interfering with SkyPilot's internal Ray cluster (port 6380)
+echo -e "${YELLOW}Killing Ray processes on port ${RAY_HEAD_PORT}...${NC}"
+
+pkill -f "ray.*[=:]${RAY_HEAD_PORT}" || true
+
+echo -e "${GREEN}Ray processes killed.${NC}"
+# Wait a moment for processes to terminate
+sleep 5
+
+# Verify Ray is stopped
+if run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+    echo -e "${RED}Warning: Ray cluster may still be running. Try manually:${NC}"
+    echo -e "${RED}  pkill -9 -f 'ray.*[=:]${RAY_HEAD_PORT}'${NC}"
+    exit 1
+else
+    echo -e "${GREEN}Ray cluster successfully stopped.${NC}"
+fi