skypilot-nightly 1.0.0.dev20250910__py3-none-any.whl → 1.0.0.dev20250913__py3-none-any.whl

sky/skylet/services.py

@@ -1,11 +1,19 @@
 """gRPC service implementations for skylet."""
 
+import os
+
 import grpc
 
 from sky import sky_logging
+from sky.jobs import state as managed_job_state
 from sky.schemas.generated import autostopv1_pb2
 from sky.schemas.generated import autostopv1_pb2_grpc
+from sky.schemas.generated import jobsv1_pb2
+from sky.schemas.generated import jobsv1_pb2_grpc
+from sky.serve import serve_state
 from sky.skylet import autostop_lib
+from sky.skylet import constants
+from sky.skylet import job_lib
 
 logger = sky_logging.init_logger(__name__)
 
@@ -42,3 +50,198 @@ class AutostopServiceImpl(autostopv1_pb2_grpc.AutostopServiceServicer):
                 is_autostopping=is_autostopping)
         except Exception as e:  # pylint: disable=broad-except
             context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+
+class JobsServiceImpl(jobsv1_pb2_grpc.JobsServiceServicer):
+    """Implementation of the JobsService gRPC service."""
+
+    def AddJob(  # type: ignore[return]
+            self, request: jobsv1_pb2.AddJobRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.AddJobResponse:
+        try:
+            job_name = request.job_name if request.HasField('job_name') else '-'
+            job_id, log_dir = job_lib.add_job(job_name, request.username,
+                                              request.run_timestamp,
+                                              request.resources_str,
+                                              request.metadata)
+            return jobsv1_pb2.AddJobResponse(job_id=job_id, log_dir=log_dir)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def QueueJob(  # type: ignore[return]
+            self, request: jobsv1_pb2.QueueJobRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.QueueJobResponse:
+        try:
+            job_id = request.job_id
+            # Create log directory and file
+            remote_log_dir = os.path.expanduser(request.remote_log_dir)
+            os.makedirs(remote_log_dir, exist_ok=True)
+            remote_log_path = os.path.join(remote_log_dir, 'run.log')
+            open(remote_log_path, 'a').close()  # pylint: disable=unspecified-encoding
+
+            script_path = os.path.expanduser(request.script_path)
+            os.makedirs(os.path.dirname(script_path), exist_ok=True)
+
+            # If `codegen` is not provided, assume script is already
+            # uploaded to `script_path` via rsync.
+            if request.HasField('codegen'):
+                with open(script_path, 'w', encoding='utf-8') as f:
+                    f.write(request.codegen)
+                os.chmod(script_path, 0o755)
+
+            cd = f'cd {constants.SKY_REMOTE_WORKDIR}'
+            job_submit_cmd = (
+                # JOB_CMD_IDENTIFIER is used for identifying the process
+                # retrieved with pid is the same driver process.
+                f'{job_lib.JOB_CMD_IDENTIFIER.format(job_id)} && '
+                f'{cd} && {constants.SKY_PYTHON_CMD} -u {script_path}'
+                # Do not use &>, which is not POSIX and may not work.
+                # Note that the order of ">filename 2>&1" matters.
+                f' > {remote_log_path} 2>&1')
+            job_lib.scheduler.queue(job_id, job_submit_cmd)
+
+            if request.HasField('managed_job'):
+                managed_job = request.managed_job
+                pool = managed_job.pool if managed_job.HasField(
+                    'pool') else None
+                pool_hash = None
+                if pool is not None:
+                    pool_hash = serve_state.get_service_hash(pool)
+                # Add the managed job to job queue database.
+                managed_job_state.set_job_info(job_id, managed_job.name,
+                                               managed_job.workspace,
+                                               managed_job.entrypoint, pool,
+                                               pool_hash)
+                # Set the managed job to PENDING state to make sure that
+                # this managed job appears in the `sky jobs queue`, even
+                # if it needs to wait to be submitted.
+                # We cannot set the managed job to PENDING state in the
+                # job template (jobs-controller.yaml.j2), as it may need
+                # to wait for the run commands to be scheduled on the job
+                # controller in high-load cases.
+                for task in managed_job.tasks:
+                    managed_job_state.set_pending(job_id, task.task_id,
+                                                  task.name, task.resources_str,
+                                                  task.metadata_json)
+            return jobsv1_pb2.QueueJobResponse()
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def UpdateStatus(  # type: ignore[return]
+            self, request: jobsv1_pb2.UpdateStatusRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.UpdateStatusResponse:
+        try:
+            job_lib.update_status()
+            return jobsv1_pb2.UpdateStatusResponse()
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetJobQueue(  # type: ignore[return]
+            self, request: jobsv1_pb2.GetJobQueueRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.GetJobQueueResponse:
+        try:
+            user_hash = request.user_hash if request.HasField(
+                'user_hash') else None
+            all_jobs = request.all_jobs
+            jobs_info = job_lib.get_jobs_info(user_hash=user_hash,
+                                              all_jobs=all_jobs)
+            return jobsv1_pb2.GetJobQueueResponse(jobs=jobs_info)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def CancelJobs(  # type: ignore[return]
+            self, request: jobsv1_pb2.CancelJobsRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.CancelJobsResponse:
+        try:
+            job_ids = list(request.job_ids) if request.job_ids else []
+            user_hash = request.user_hash if request.HasField(
+                'user_hash') else None
+            cancelled_job_ids = job_lib.cancel_jobs(job_ids, request.cancel_all,
+                                                    user_hash)
+            return jobsv1_pb2.CancelJobsResponse(
+                cancelled_job_ids=cancelled_job_ids)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def FailAllInProgressJobs(  # type: ignore[return]
+        self, _: jobsv1_pb2.FailAllInProgressJobsRequest,
+        context: grpc.ServicerContext
+    ) -> jobsv1_pb2.FailAllInProgressJobsResponse:
+        try:
+            job_lib.fail_all_jobs_in_progress()
+            return jobsv1_pb2.FailAllInProgressJobsResponse()
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def TailLogs(
+            self,
+            request: jobsv1_pb2.TailLogsRequest,  # type: ignore[return]
+            context: grpc.ServicerContext):
+        # TODO(kevin): implement this
+        raise NotImplementedError('TailLogs is not implemented')
+
+    def GetJobStatus(  # type: ignore[return]
+            self, request: jobsv1_pb2.GetJobStatusRequest,
+            context: grpc.ServicerContext) -> jobsv1_pb2.GetJobStatusResponse:
+        try:
+            if request.job_ids:
+                job_ids = list(request.job_ids)
+            else:
+                latest_job_id = job_lib.get_latest_job_id()
+                job_ids = [latest_job_id] if latest_job_id is not None else []
+            job_statuses = job_lib.get_statuses(job_ids)
+            for job_id, status in job_statuses.items():
+                job_statuses[job_id] = job_lib.JobStatus(status).to_protobuf(
+                ) if status is not None else jobsv1_pb2.JOB_STATUS_UNSPECIFIED
+            return jobsv1_pb2.GetJobStatusResponse(job_statuses=job_statuses)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetJobSubmittedTimestamp(  # type: ignore[return]
+        self, request: jobsv1_pb2.GetJobSubmittedTimestampRequest,
+        context: grpc.ServicerContext
+    ) -> jobsv1_pb2.GetJobSubmittedTimestampResponse:
+        try:
+            job_id = request.job_id if request.HasField(
+                'job_id') else job_lib.get_latest_job_id()
+            timestamp = job_lib.get_job_submitted_or_ended_timestamp(
+                job_id, False)
+            if timestamp is None:
+                context.abort(grpc.StatusCode.NOT_FOUND,
+                              f'Job {job_id} not found')
+            return jobsv1_pb2.GetJobSubmittedTimestampResponse(
+                timestamp=timestamp)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetJobEndedTimestamp(  # type: ignore[return]
+        self, request: jobsv1_pb2.GetJobEndedTimestampRequest,
+        context: grpc.ServicerContext
+    ) -> jobsv1_pb2.GetJobEndedTimestampResponse:
+        try:
+            job_id = request.job_id if request.HasField(
+                'job_id') else job_lib.get_latest_job_id()
+            timestamp = job_lib.get_job_submitted_or_ended_timestamp(
+                job_id, True)
+            if timestamp is None:
+                context.abort(grpc.StatusCode.NOT_FOUND,
+                              f'Job {job_id} not found or not ended')
+            return jobsv1_pb2.GetJobEndedTimestampResponse(timestamp=timestamp)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetLogDirsForJobs(  # type: ignore[return]
+            self, request: jobsv1_pb2.GetLogDirsForJobsRequest,
+            context: grpc.ServicerContext
+    ) -> jobsv1_pb2.GetLogDirsForJobsResponse:
+        try:
+            if request.job_ids:
+                job_ids = list(request.job_ids)
+            else:
+                latest_job_id = job_lib.get_latest_job_id()
+                job_ids = [latest_job_id] if latest_job_id is not None else []
+            job_log_dirs = job_lib.get_job_log_dirs(job_ids)
+            return jobsv1_pb2.GetLogDirsForJobsResponse(
+                job_log_dirs=job_log_dirs)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))

sky/skylet/skylet.py

@@ -9,6 +9,7 @@ import grpc
 import sky
 from sky import sky_logging
 from sky.schemas.generated import autostopv1_pb2_grpc
+from sky.schemas.generated import jobsv1_pb2_grpc
 from sky.skylet import constants
 from sky.skylet import events
 from sky.skylet import services
@@ -50,6 +51,9 @@ def start_grpc_server(port: int = constants.SKYLET_GRPC_PORT) -> grpc.Server:
     autostopv1_pb2_grpc.add_AutostopServiceServicer_to_server(
         services.AutostopServiceImpl(), server)
 
+    jobsv1_pb2_grpc.add_JobsServiceServicer_to_server(
+        services.JobsServiceImpl(), server)
+
     listen_addr = f'127.0.0.1:{port}'
     server.add_insecure_port(listen_addr)
 

sky/task.py

@@ -20,6 +20,7 @@ from sky.provision import docker_utils
 from sky.serve import service_spec
 from sky.skylet import constants
 from sky.utils import common_utils
+from sky.utils import git
 from sky.utils import registry
 from sky.utils import schemas
 from sky.utils import ux_utils
@@ -1596,6 +1597,67 @@ class Task:
                 d[k] = v
         return d
 
+    def update_workdir(self, workdir: Optional[str], git_url: Optional[str],
+                       git_ref: Optional[str]) -> 'Task':
+        """Updates the task workdir.
+
+        Args:
+            workdir: The workdir to update.
+            git_url: The git url to update.
+            git_ref: The git ref to update.
+        """
+        if self.workdir is None or isinstance(self.workdir, str):
+            if workdir is not None:
+                self.workdir = workdir
+                return self
+            if git_url is not None:
+                self.workdir = {}
+                self.workdir['url'] = git_url
+                if git_ref is not None:
+                    self.workdir['ref'] = git_ref
+                return self
+            return self
+        if git_url is not None:
+            self.workdir['url'] = git_url
+        if git_ref is not None:
+            self.workdir['ref'] = git_ref
+        return self
+
+    def update_envs_and_secrets_from_workdir(self) -> 'Task':
+        """Updates the task envs and secrets from the workdir."""
+        if self.workdir is None:
+            return self
+        if not isinstance(self.workdir, dict):
+            return self
+        url = self.workdir['url']
+        ref = self.workdir.get('ref', '')
+        token = os.environ.get(git.GIT_TOKEN_ENV_VAR)
+        ssh_key_path = os.environ.get(git.GIT_SSH_KEY_PATH_ENV_VAR)
+        try:
+            git_repo = git.GitRepo(url, ref, token, ssh_key_path)
+            clone_info = git_repo.get_repo_clone_info()
+            if clone_info is None:
+                return self
+            self.envs[git.GIT_URL_ENV_VAR] = clone_info.url
+            if ref:
+                ref_type = git_repo.get_ref_type()
+                if ref_type == git.GitRefType.COMMIT:
+                    self.envs[git.GIT_COMMIT_HASH_ENV_VAR] = ref
+                elif ref_type == git.GitRefType.BRANCH:
+                    self.envs[git.GIT_BRANCH_ENV_VAR] = ref
+                elif ref_type == git.GitRefType.TAG:
+                    self.envs[git.GIT_TAG_ENV_VAR] = ref
+            if clone_info.token is None and clone_info.ssh_key is None:
+                return self
+            if clone_info.token is not None:
+                self.secrets[git.GIT_TOKEN_ENV_VAR] = clone_info.token
+            if clone_info.ssh_key is not None:
+                self.secrets[git.GIT_SSH_KEY_ENV_VAR] = clone_info.ssh_key
+        except exceptions.GitError as e:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'{str(e)}') from None
+        return self
+
     def to_yaml_config(self,
                        use_user_specified_yaml: bool = False) -> Dict[str, Any]:
         """Returns a yaml-style dict representation of the task.

sky/templates/kubernetes-ray.yml.j2

@@ -654,8 +654,125 @@ available_node_types:
                 # after v0.11.0 release.
                 touch /tmp/apt_ssh_setup_started
 
-                DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update > /tmp/apt-update.log 2>&1 || \
-                echo "Warning: apt-get update failed. Continuing anyway..." >> /tmp/apt-update.log
+                # Helper: run apt-get update with retries
+                apt_update_with_retries() {
+                  # do not fail the whole shell; we handle return codes
+                  set +e
+                  local log=/tmp/apt-update.log
+                  local tries=3
+                  local delay=1
+                  local i
+                  for i in $(seq 1 $tries); do
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update >> "$log" 2>&1 && { set -e; return 0; }
+                    echo "apt-get update attempt $i/$tries failed; retrying in ${delay}s" >> "$log"
+                    sleep $delay
+                    delay=$((delay * 2))
+                  done
+                  set -e
+                  return 1
+                }
+                apt_install_with_retries() {
+                  local packages="$@"
+                  [ -z "$packages" ] && return 0
+                  set +e
+                  local log=/tmp/apt-update.log
+                  local tries=3
+                  local delay=1
+                  local i
+                  for i in $(seq 1 $tries); do
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $packages && { set -e; return 0; }
+                    echo "apt-get install failed for: $packages (attempt $i/$tries). Running -f install and retrying..." >> "$log"
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get -f install -y >> "$log" 2>&1 || true
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get clean >> "$log" 2>&1 || true
+                    sleep $delay
+                    delay=$((delay * 2))
+                  done
+                  set -e
+                  return 1
+                }
+                apt_update_install_with_retries() {
+                  apt_update_with_retries
+                  apt_install_with_retries "$@"
+                }
+                backup_dir=/etc/apt/sources.list.backup_skypilot
+                backup_source() {
+                  $(prefix_cmd) mkdir -p "$backup_dir"
+                  if [ -f /etc/apt/sources.list ] && [ ! -f "$backup_dir/sources.list" ]; then
+                    $(prefix_cmd) cp -a /etc/apt/sources.list "$backup_dir/sources.list" || true
+                  fi
+                }
+                restore_source() {
+                  if [ -f "$backup_dir/sources.list" ]; then
+                    $(prefix_cmd) cp -a "$backup_dir/sources.list" /etc/apt/sources.list || true
+                  fi
+                }
+                update_apt_sources() {
+                  local host=$1
+                  local apt_file=$2
+                  $(prefix_cmd) sed -i -E "s|https?://[a-zA-Z0-9.-]+\.ubuntu\.com/ubuntu|http://$host/ubuntu|g" $apt_file
+                }
+                # Helper: install packages across mirrors with retries
+                apt_install_with_mirrors() {
+                  local required=$1; shift
+                  local packages="$@"
+                  [ -z "$packages" ] && return 0
+                  set +e
+                  # Install packages with default sources first
+                  local log=/tmp/apt-update.log
+                  echo "$(date +%Y-%m-%d\ %H:%M:%S) Installing packages: $packages" >> "$log"
+                  restore_source
+                  apt_update_install_with_retries $packages >> "$log" 2>&1 && { set -e; return 0; }
+                  echo "Install failed with default sources: $packages" >> "$log"
+                  # Detect distro (ubuntu/debian)
+                  local APT_OS="unknown"
+                  if [ -f /etc/os-release ]; then
+                    . /etc/os-release
+                    case "$ID" in
+                      debian) APT_OS="debian" ;;
+                      ubuntu) APT_OS="ubuntu" ;;
+                      *)
+                        if [ -n "$ID_LIKE" ]; then
+                          case " $ID $ID_LIKE " in
+                            *ubuntu*) APT_OS="ubuntu" ;;
+                            *debian*) APT_OS="debian" ;;
+                          esac
+                        fi
+                        ;;
+                    esac
+                  fi
+                  # Build mirror candidates
+                  # deb.debian.org is a CDN endpoint, if one backend goes down,
+                  # the CDN automatically fails over to another mirror,
+                  # so we only retry for ubuntu here.
+                  if [ "$APT_OS" = "ubuntu" ]; then
+                    # Backup current sources once
+                    backup_source
+                    # Selected from https://launchpad.net/ubuntu/+archivemirrors
+                    # and results from apt-select
+                    local MIRROR_CANDIDATES="mirrors.wikimedia.org mirror.umd.edu"
+                    for host in $MIRROR_CANDIDATES; do
+                      echo "Trying APT mirror ($APT_OS): $host" >> "$log"
+                      if [ -f /etc/apt/sources.list ]; then
+                        update_apt_sources $host /etc/apt/sources.list
+                      else
+                        echo "Error: /etc/apt/sources.list not found" >> "$log"
+                        break
+                      fi
+                      apt_update_install_with_retries $packages >> "$log" 2>&1 && { set -e; return 0; }
+                      echo "Install failed with mirror ($APT_OS): $host" >> "$log"
+                      # Restore to default sources
+                      restore_source
+                    done
+                  fi
+                  set -e
+                  if [ "$required" = "1" ]; then
+                    echo "Error: required package install failed across all mirrors: $packages" >> "$log"
+                    return 1
+                  else
+                    echo "Optional package install failed across all mirrors: $packages; skipping." >> "$log"
+                    return 0
+                  fi
+                }
                 # Install both fuse2 and fuse3 for compatibility for all possible fuse adapters in advance,
                 # so that both fusemount and fusermount3 can be masked before enabling SSH access.
                 PACKAGES="rsync curl wget netcat gcc patch pciutils fuse fuse3 openssh-server";
@@ -682,7 +799,7 @@ available_node_types:
                 done;
                 if [ ! -z "$INSTALL_FIRST" ]; then
                   echo "Installing core packages: $INSTALL_FIRST";
-                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $INSTALL_FIRST;
+                  apt_install_with_mirrors 1 $INSTALL_FIRST || { echo "Error: core package installation failed." >> /tmp/apt-update.log; exit 1; }
                 fi;
                 # SSH and other packages are not necessary, so we disable set -e
                 set +e

sky/templates/seeweb-ray.yml.j2

@@ -0,0 +1,108 @@
+cluster_name: {{ cluster_name_on_cloud }}
+
+max_workers: {{ num_nodes - 1 }}
+upscaling_speed: {{ num_nodes - 1 }}
+idle_timeout_minutes: 5
+
+provider:
+  type: external
+  module: sky.provision.seeweb
+  region: "{{ region }}"
+
+auth:
+  ssh_user: ecuser
+  ssh_private_key: {{ ssh_private_key }}
+
+available_node_types:
+  ray_head_default:
+    resources: {}
+    node_config:
+      plan: {{ instance_type }}
+      image: {{ image_id }}
+      location: {{ region }}
+      {% if seeweb_gpu_config is not none %}
+      gpu: {{ seeweb_gpu_config.gpu }}
+      gpu_label: "{{ seeweb_gpu_config.gpu_label }}"
+      {% endif %}
+      disk: {{ disk_size }}
+
+head_node_type: ray_head_default
+
+file_mounts: {
+  "~/.seeweb_cloud/seeweb_keys": "~/.seeweb_cloud/seeweb_keys",
+  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
+{%- for remote_path, local_path in credentials.items() %}
+  "{{remote_path}}": "{{local_path}}",
+{%- endfor %}
+  "~/.ssh/sky-cluster-key": "{{ssh_private_key}}",
+}
+
+rsync_exclude: []
+
+setup_commands:
+  - |
+    touch ~/.bashrc;
+    echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts || true;
+    echo "127.0.0.1 localhost" | sudo tee -a /etc/hosts || true;
+    sudo systemctl stop unattended-upgrades || true;
+    sudo systemctl disable unattended-upgrades || true;
+    sudo apt update && sudo apt install -y patch || sudo yum install -y patch || true;
+    {{ conda_installation_commands }}
+    {{ ray_skypilot_installation_commands }}
+
+head_start_ray_commands:
+  - |
+    retry_ray() {
+      local n=0; local max=30
+      until [ $n -ge $max ]; do
+        export SKYPILOT_NUM_GPUS=0
+        command -v nvidia-smi >/dev/null 2>&1 && \
+          SKYPILOT_NUM_GPUS=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l)
+
+        ray stop || true
+        RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 \
+        ray start --disable-usage-stats --head \
+          --port={{ ray_port }} --dashboard-port={{ ray_dashboard_port }} \
+          --object-manager-port=8076 \
+          --autoscaling-config=~/ray_bootstrap_config.yaml \
+          --num-gpus=$SKYPILOT_NUM_GPUS --temp-dir {{ ray_temp_dir }} && break
+
+        echo "[head] Ray failed to start ($((++n))/$max), retrying in 5s..."
+        sleep 5
+      done
+      [ $n -eq $max ] && { echo "Ray head failed"; exit 1; }
+    }
+    retry_ray
+
+worker_start_ray_commands:
+  - |
+    retry_ray() {
+      local n=0; local max=30
+      until [ $n -ge $max ]; do
+        SKYPILOT_NUM_GPUS=0
+        command -v nvidia-smi >/dev/null 2>&1 && \
+          SKYPILOT_NUM_GPUS=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l)
+
+        ray stop || true
+        RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 \
+        ray start --disable-usage-stats \
+          --address=$RAY_HEAD_IP:{{ ray_port }} \
+          --object-manager-port=8076 \
+          --num-gpus=$SKYPILOT_NUM_GPUS --temp-dir {{ ray_temp_dir }} && break
+
+        echo "[worker] Ray failed to start ($((++n))/$max), retrying in 5s..."
+        sleep 5
+      done
+      [ $n -eq $max ] && { echo "Ray worker failed"; exit 1; }
+    }
+    retry_ray
+
+head_node: {}
+worker_nodes: {}
+
+head_setup_commands: []
+worker_setup_commands: []
+
+cluster_synced_files: []
+file_mounts_sync_continuously: False

sky/utils/accelerator_registry.py

@@ -107,10 +107,12 @@ def canonicalize_accelerator_name(accelerator: str,
     if not names and cloud_str in ['Kubernetes', None]:
         with rich_utils.safe_status(
                 ux_utils.spinner_message('Listing accelerators on Kubernetes')):
+            # Only search for Kubernetes to reduce the lookup cost.
+            # For other clouds, the catalog has been searched in previous steps.
             searched = catalog.list_accelerators(
                 name_filter=accelerator,
                 case_sensitive=False,
-                clouds=cloud_str,
+                clouds='Kubernetes',
             )
         names = list(searched.keys())
         if accelerator in names: