skypilot-nightly 1.0.0.dev20250426__py3-none-any.whl → 1.0.0.dev20250428__py3-none-any.whl

sky/serve/serve_state.py

@@ -479,6 +479,14 @@ def total_number_provisioning_replicas() -> int:
     return provisioning_count
 
 
+def get_replicas_at_status(
+    service_name: str,
+    status: ReplicaStatus,
+) -> List['replica_managers.ReplicaInfo']:
+    replicas = get_replica_infos(service_name)
+    return [replica for replica in replicas if replica.status == status]
+
+
 # === Version functions ===
 def add_version(service_name: str) -> int:
     """Adds a version to the database."""
@@ -549,3 +557,36 @@ def delete_all_versions(service_name: str) -> None:
             """\
             DELETE FROM version_specs
             WHERE service_name=(?)""", (service_name,))
+
+
+def get_latest_version(service_name: str) -> Optional[int]:
+    with db_utils.safe_cursor(_DB_PATH) as cursor:
+        rows = cursor.execute(
+            """\
+            SELECT MAX(version) FROM version_specs
+            WHERE service_name=(?)""", (service_name,)).fetchall()
+    if not rows or rows[0][0] is None:
+        return None
+    return rows[0][0]
+
+
+def get_service_controller_port(service_name: str) -> int:
+    """Gets the controller port of a service."""
+    with db_utils.safe_cursor(_DB_PATH) as cursor:
+        cursor.execute('SELECT controller_port FROM services WHERE name = ?',
+                       (service_name,))
+        row = cursor.fetchone()
+        if row is None:
+            raise ValueError(f'Service {service_name} does not exist.')
+        return row[0]
+
+
+def get_service_load_balancer_port(service_name: str) -> int:
+    """Gets the load balancer port of a service."""
+    with db_utils.safe_cursor(_DB_PATH) as cursor:
+        cursor.execute('SELECT load_balancer_port FROM services WHERE name = ?',
+                       (service_name,))
+        row = cursor.fetchone()
+        if row is None:
+            raise ValueError(f'Service {service_name} does not exist.')
+        return row[0]

sky/serve/service.py

@@ -25,6 +25,7 @@ from sky.serve import load_balancer
 from sky.serve import replica_managers
 from sky.serve import serve_state
 from sky.serve import serve_utils
+from sky.skylet import constants as skylet_constants
 from sky.utils import common_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
@@ -136,8 +137,25 @@ def _cleanup(service_name: str) -> bool:
     return failed
 
 
+def _cleanup_task_run_script(job_id: int) -> None:
+    """Clean up task run script.
+    Please see `kubernetes-ray.yml.j2` for more details.
+    """
+    task_run_dir = pathlib.Path(
+        skylet_constants.PERSISTENT_RUN_SCRIPT_DIR).expanduser()
+    if task_run_dir.exists():
+        this_task_run_script = task_run_dir / f'sky_job_{job_id}'
+        if this_task_run_script.exists():
+            this_task_run_script.unlink()
+            logger.info(f'Task run script {this_task_run_script} removed')
+        else:
+            logger.warning(f'Task run script {this_task_run_script} not found')
+
+
 def _start(service_name: str, tmp_task_yaml: str, job_id: int):
-    """Starts the service."""
+    """Starts the service.
+    This including the controller and load balancer.
+    """
     # Generate ssh key pair to avoid race condition when multiple sky.launch
     # are executed at the same time.
     authentication.get_or_generate_keys()
@@ -147,62 +165,79 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
     # Already checked before submit to controller.
     assert task.service is not None, task
     service_spec = task.service
-    if (len(serve_state.get_services()) >=
-            serve_utils.get_num_service_threshold()):
-        cleanup_storage(tmp_task_yaml)
-        with ux_utils.print_exception_no_traceback():
-            raise RuntimeError('Max number of services reached.')
-    success = serve_state.add_service(
-        service_name,
-        controller_job_id=job_id,
-        policy=service_spec.autoscaling_policy_str(),
-        requested_resources_str=backend_utils.get_task_resources_str(task),
-        load_balancing_policy=service_spec.load_balancing_policy,
-        status=serve_state.ServiceStatus.CONTROLLER_INIT,
-        tls_encrypted=service_spec.tls_credential is not None)
-    # Directly throw an error here. See sky/serve/api.py::up
-    # for more details.
-    if not success:
-        cleanup_storage(tmp_task_yaml)
-        with ux_utils.print_exception_no_traceback():
-            raise ValueError(f'Service {service_name} already exists.')
-
-    # Add initial version information to the service state.
-    serve_state.add_or_update_version(service_name, constants.INITIAL_VERSION,
-                                      service_spec)
-
-    # Create the service working directory.
+
+    def is_recovery_mode(service_name: str) -> bool:
+        """Check if service exists in database to determine recovery mode.
+        """
+        service = serve_state.get_service_from_name(service_name)
+        return service is not None
+
+    is_recovery = is_recovery_mode(service_name)
+    logger.info(f'It is a {"first" if not is_recovery else "recovery"} run')
+
+    if is_recovery:
+        version = serve_state.get_latest_version(service_name)
+        if version is None:
+            raise ValueError(f'No version found for service {service_name}')
+    else:
+        version = constants.INITIAL_VERSION
+        # Add initial version information to the service state.
+        serve_state.add_or_update_version(service_name, version, service_spec)
+
     service_dir = os.path.expanduser(
         serve_utils.generate_remote_service_dir_name(service_name))
-    os.makedirs(service_dir, exist_ok=True)
-
-    # Copy the tmp task yaml file to the final task yaml file.
-    # This is for the service name conflict case. The _execute will
-    # sync file mounts first and then realized a name conflict. We
-    # don't want the new file mounts to overwrite the old one, so we
-    # sync to a tmp file first and then copy it to the final name
-    # if there is no name conflict.
-    task_yaml = serve_utils.generate_task_yaml_file_name(
-        service_name, constants.INITIAL_VERSION)
-    shutil.copy(tmp_task_yaml, task_yaml)
-
-    # Generate load balancer log file name.
-    load_balancer_log_file = os.path.expanduser(
-        serve_utils.generate_remote_load_balancer_log_file_name(service_name))
+    task_yaml = serve_utils.generate_task_yaml_file_name(service_name, version)
+
+    if not is_recovery:
+        if (len(serve_state.get_services()) >=
+                serve_utils.get_num_service_threshold()):
+            cleanup_storage(tmp_task_yaml)
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError('Max number of services reached.')
+        success = serve_state.add_service(
+            service_name,
+            controller_job_id=job_id,
+            policy=service_spec.autoscaling_policy_str(),
+            requested_resources_str=backend_utils.get_task_resources_str(task),
+            load_balancing_policy=service_spec.load_balancing_policy,
+            status=serve_state.ServiceStatus.CONTROLLER_INIT,
+            tls_encrypted=service_spec.tls_credential is not None)
+        # Directly throw an error here. See sky/serve/api.py::up
+        # for more details.
+        if not success:
+            cleanup_storage(tmp_task_yaml)
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'Service {service_name} already exists.')
+
+        # Create the service working directory.
+        os.makedirs(service_dir, exist_ok=True)
+
+        # Copy the tmp task yaml file to the final task yaml file.
+        # This is for the service name conflict case. The _execute will
+        # sync file mounts first and then realized a name conflict. We
+        # don't want the new file mounts to overwrite the old one, so we
+        # sync to a tmp file first and then copy it to the final name
+        # if there is no name conflict.
+        shutil.copy(tmp_task_yaml, task_yaml)
 
     controller_process = None
     load_balancer_process = None
     try:
         with filelock.FileLock(
                 os.path.expanduser(constants.PORT_SELECTION_FILE_LOCK_PATH)):
-            controller_port = common_utils.find_free_port(
-                constants.CONTROLLER_PORT_START)
-
-            # We expose the controller to the public network when running
-            # inside a kubernetes cluster to allow external load balancers
-            # (example, for high availability load balancers) to communicate
-            # with the controller.
-            def _get_host():
+            # Start the controller.
+            controller_port = (
+                common_utils.find_free_port(constants.CONTROLLER_PORT_START)
+                if not is_recovery else
+                serve_state.get_service_controller_port(service_name))
+
+            def _get_controller_host():
+                """Get the controller host address.
+                We expose the controller to the public network when running
+                inside a kubernetes cluster to allow external load balancers
+                (example, for high availability load balancers) to communicate
+                with the controller.
+                """
                 if 'KUBERNETES_SERVICE_HOST' in os.environ:
                     return '0.0.0.0'
                 # Not using localhost to avoid using ipv6 address and causing
@@ -211,26 +246,28 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
                 # ('::1', 20001, 0, 0): cannot assign requested address
                 return '127.0.0.1'
 
-            controller_host = _get_host()
-
-            # Start the controller.
+            controller_host = _get_controller_host()
             controller_process = multiprocessing.Process(
                 target=controller.run_controller,
                 args=(service_name, service_spec, task_yaml, controller_host,
                       controller_port))
             controller_process.start()
-            serve_state.set_service_controller_port(service_name,
-                                                    controller_port)
 
-            controller_addr = f'http://{controller_host}:{controller_port}'
+            if not is_recovery:
+                serve_state.set_service_controller_port(service_name,
+                                                        controller_port)
 
-            load_balancer_port = common_utils.find_free_port(
-                constants.LOAD_BALANCER_PORT_START)
-
-            # Extract the load balancing policy from the service spec
-            policy_name = service_spec.load_balancing_policy
+            controller_addr = f'http://{controller_host}:{controller_port}'
 
             # Start the load balancer.
+            load_balancer_port = (
+                common_utils.find_free_port(constants.LOAD_BALANCER_PORT_START)
+                if not is_recovery else
+                serve_state.get_service_load_balancer_port(service_name))
+            load_balancer_log_file = os.path.expanduser(
+                serve_utils.generate_remote_load_balancer_log_file_name(
+                    service_name))
+
             # TODO(tian): Probably we could enable multiple ports specified in
             # service spec and we could start multiple load balancers.
             # After that, we will have a mapping from replica port to endpoint.
@@ -238,11 +275,14 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
                 target=ux_utils.RedirectOutputForProcess(
                     load_balancer.run_load_balancer,
                     load_balancer_log_file).run,
-                args=(controller_addr, load_balancer_port, policy_name,
+                args=(controller_addr, load_balancer_port,
+                      service_spec.load_balancing_policy,
                       service_spec.tls_credential))
             load_balancer_process.start()
-            serve_state.set_service_load_balancer_port(service_name,
-                                                       load_balancer_port)
+
+            if not is_recovery:
+                serve_state.set_service_load_balancer_port(
+                    service_name, load_balancer_port)
 
         while True:
             _handle_signal(service_name)
@@ -262,6 +302,7 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
             force=True)
         for process in process_to_kill:
             process.join()
+
         failed = _cleanup(service_name)
         if failed:
             serve_state.set_service_status_and_active_versions(
@@ -273,8 +314,12 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
             serve_state.delete_all_versions(service_name)
             logger.info(f'Service {service_name} terminated successfully.')
 
+        _cleanup_task_run_script(job_id)
+
 
 if __name__ == '__main__':
+    logger.info('Starting service...')
+
     parser = argparse.ArgumentParser(description='Sky Serve Service')
     parser.add_argument('--service-name',
                         type=str,

sky/server/requests/executor.py

@@ -493,15 +493,15 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
     # Determine the job capacity of the workers based on the system resources.
     cpu_count = common_utils.get_cpu_count()
     mem_size_gb = common_utils.get_mem_size_gb()
-    mem_size_gb = max(0, mem_size_gb - server_constants.MIN_AVAIL_MEM_GB)
+    mem_for_workers = max(0, mem_size_gb - server_constants.MIN_AVAIL_MEM_GB)
     # Runs in low resource mode if the available memory is less than
     # server_constants.MIN_AVAIL_MEM_GB.
     max_parallel_for_long = _max_long_worker_parallism(cpu_count,
-                                                       mem_size_gb,
+                                                       mem_for_workers,
                                                        local=not deploy)
     max_parallel_for_short = _max_short_worker_parallism(
-        mem_size_gb, max_parallel_for_long)
-    if mem_size_gb < server_constants.MIN_AVAIL_MEM_GB:
+        mem_for_workers, max_parallel_for_long)
+    if not deploy and mem_size_gb < server_constants.MIN_AVAIL_MEM_GB:
         # Permanent worker process may have significant memory consumption
         # (~350MB per worker) after running commands like `sky check`, so we
         # don't start any permanent workers in low resource local mode. This

sky/skylet/constants.py

@@ -368,6 +368,13 @@ ROLE_ASSIGNMENT_FAILURE_ERROR_MSG = (
     'Failed to assign Storage Blob Data Owner role to the '
     'storage account {storage_account_name}.')
 
+# Constants for path in K8S pod to store persistent setup and run scripts
+# so that we can run them again after the pod restarts.
+# Path within user home. For HA controller, assumes home directory is
+# persistent through PVC. See kubernetes-ray.yml.j2.
+PERSISTENT_SETUP_SCRIPT_PATH = '~/.sky/.controller_recovery_setup_commands.sh'
+PERSISTENT_RUN_SCRIPT_DIR = '~/.sky/.controller_recovery_task_run'
+
 # The placeholder for the local skypilot config path in file mounts for
 # controllers.
 LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path'

sky/task.py

@@ -306,7 +306,7 @@ class Task:
         self.service_name: Optional[str] = None
 
         # Filled in by the optimizer.  If None, this Task is not planned.
-        self.best_resources = None
+        self.best_resources: Optional[sky.Resources] = None
 
         # For internal use only.
         self.file_mounts_mapping = file_mounts_mapping

sky/templates/kubernetes-ray.yml.j2

@@ -267,11 +267,14 @@ available_node_types:
             {%- for label_key, label_value in labels.items() %}
             {{ label_key }}: {{ label_value|tojson }}
             {%- endfor %}
+            {% if high_availability %}
+            app: {{cluster_name_on_cloud}}
+            {% endif %}
       spec:
         # serviceAccountName: skypilot-service-account
         serviceAccountName: {{k8s_service_account_name}}
         automountServiceAccountToken: {{k8s_automount_sa_token}}
-        restartPolicy: Never
+        restartPolicy: {{ "Always" if high_availability else "Never" }}
 
         # Add node selector if GPU/TPUs are requested:
         {% if (k8s_acc_label_key is not none and k8s_acc_label_value is not none) or (k8s_spot_label_key is not none) %}
@@ -311,6 +314,11 @@ available_node_types:
             path: {{k8s_fusermount_shared_dir}}
             type: DirectoryOrCreate
         {% endif %}
+        {% if high_availability %}
+        - name: {{k8s_high_availability_deployment_volume_mount_name}}
+          persistentVolumeClaim:
+            claimName: {{cluster_name_on_cloud}}-{{k8s_high_availability_deployment_volume_mount_name}}
+        {% endif %}
         containers:
         - name: ray-node
           imagePullPolicy: IfNotPresent
@@ -470,6 +478,7 @@ available_node_types:
                 done
                 {{ conda_installation_commands }}
                 {{ ray_installation_commands }}
+
                 VIRTUAL_ENV=~/skypilot-runtime ~/.local/bin/uv pip install skypilot[kubernetes,remote]
                 touch /tmp/ray_skypilot_installation_complete
                 echo "=== Ray and skypilot installation completed ==="
@@ -573,6 +582,28 @@ available_node_types:
                   sleep 0.1
                 done
               }
+
+              {% if high_availability %}
+              mkdir -p {{k8s_high_availability_deployment_run_script_dir}}
+              if [ -f {{k8s_high_availability_deployment_volume_mount_path}}/k8s_container_ready ]; then
+                # ! Keep this aligned with `CloudVmRayBackend._setup()`
+                # Suppose all `task.setup` are the same for skyserve controller task.
+                # So be careful for compatibility issue once you change it.
+                chmod +x {{k8s_high_availability_deployment_setup_script_path}}
+                /bin/bash --login -c "true && export OMP_NUM_THREADS=1 PYTHONWARNINGS='ignore' && {{k8s_high_availability_deployment_setup_script_path}} > /tmp/controller_recovery_setup_commands.log 2>&1"
+                echo "=== Controller setup commands completed for recovery ==="
+
+                for file in {{k8s_high_availability_deployment_run_script_dir}}/*; do
+                  # ! Keep this aligned with `CloudVmRayBackend._execute()`
+                  chmod +x $file
+                  /bin/bash --login -c "true && export OMP_NUM_THREADS=1 PYTHONWARNINGS='ignore' && $file > /tmp/task_run_$(basename $file).log 2>&1"
+                  echo "=== Controller task run for service (file: $file) completed for recovery ==="
+                done
+              fi
+
+              touch {{k8s_high_availability_deployment_volume_mount_path}}/k8s_container_ready
+              {% endif %}
+
               trap : TERM INT; log_tail || sleep infinity & wait
 
           ports:
@@ -593,6 +624,10 @@ available_node_types:
           # /tmp which cause slowdowns if is not a shared memory volume.
           - mountPath: /dev/shm
             name: dshm
+          {% if high_availability %}
+          - name: {{k8s_high_availability_deployment_volume_mount_name}}
+            mountPath: {{k8s_high_availability_deployment_volume_mount_path}}
+          {% endif %}
           {% if k8s_fuse_device_required %}
           - name: fusermount-shared-dir
             mountPath: {{k8s_fusermount_shared_dir}}
@@ -616,7 +651,92 @@ available_node_types:
               {{k8s_resource_key}}: {{accelerator_count}}
               {% endif %}
             {% endif %}
-            
+      
+      {% if high_availability %}
+      pvc_spec:
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: {{cluster_name_on_cloud}}-{{k8s_high_availability_deployment_volume_mount_name}}
+          namespace: {{k8s_namespace}}
+        spec:
+          accessModes:
+            - ReadWriteOnce # Our controller pod is singleton
+          {% if k8s_high_availability_storage_class_name is not none %}
+          storageClassName: {{k8s_high_availability_storage_class_name}}
+          {% endif %}
+          resources:
+            requests:
+              storage: {{disk_size}}Gi
+
+      deployment_spec:
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: {{cluster_name_on_cloud}}-deployment
+          namespace: {{k8s_namespace}}
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: {{cluster_name_on_cloud}}
+          template:
+            # The only difference between the pod spec and this section is the initContainers
+            metadata:
+              # should be replaced by pod metadata
+            spec:
+              securityContext:
+                fsGroup: 1000
+              # To prevent the home dir provided by the docker image from being overriden by pvc mounting,
+              # we use initContainers to copy it first to /mnt/home, which will later be mounted to home dir.
+              initContainers:
+                - name: init-copy-home
+                  image: {{image_id}}
+                  command: ["/bin/sh", "-c"]
+                  args:
+                    - |
+                      # Define path for the marker file created by the main container upon successful startup.
+                      # This file persists in the PVC across Pod restarts.
+                      MARKER_FILE="/mnt/home/k8s_container_ready"
+                      SOURCE_PATH="{{k8s_high_availability_deployment_volume_mount_path}}"
+                      DEST_PATH="/mnt/home"
+
+                      # We only need to copy the initial home directory contents from the image
+                      # the *first* time a Pod uses a *new* PVC.
+                      # On subsequent Pod starts (e.g., after a crash or update), the PVC
+                      # already contains the necessary data (and potentially user modifications).
+                      # The presence of MARKER_FILE (created by the main container in a previous
+                      # successful run) indicates the PVC is already initialized. Checking for
+                      # it prevents unnecessary and time-consuming rsync operations on every restart.
+                      if [ ! -f "$MARKER_FILE" ]; then
+                        echo "Marker '$MARKER_FILE' not found. PVC likely needs initialization."
+                        echo "Copying initial home directory from image ($SOURCE_PATH/) to PVC ($DEST_PATH)..."
+
+                        # Use rsync with -rl (recursive, links) instead of -a (archive).
+                        # This avoids preserving times (-t) and permissions (-p) implied by -a,
+                        # which caused 'Operation not permitted' errors on the PVC root directory (/mnt/home).
+                        # Owner/group preservation (-o, -g) is also skipped (default for -rl), ensuring
+                        # files are owned by the container's user/group.
+                        rsync -rl "$SOURCE_PATH/" "$DEST_PATH"
+
+                        # Check if rsync failed
+                        if [ $? -ne 0 ]; then
+                           echo "ERROR: rsync failed during home directory initialization." >&2
+                           exit 1 # Exit initContainer with error if copy fails
+                        fi
+                        echo "Home directory initialization copy complete."
+                      else
+                        # If marker exists, skip the copy
+                        echo "Marker '$MARKER_FILE' found. Skipping initial home directory copy."
+                      fi
+                      echo "Current contents of $DEST_PATH:"
+                      ls -la "$DEST_PATH"
+                  volumeMounts:
+                    # Mount the persistent volume claim into the initContainer
+                    - name: {{k8s_high_availability_deployment_volume_mount_name}}
+                      mountPath: /mnt/home # Temporary mount point for initialization
+              # should be replaced by pod spec
+      {% endif %}
 setup_commands:
   # Disable `unattended-upgrades` to prevent apt-get from hanging. It should be called at the beginning before the process started to avoid being blocked. (This is a temporary fix.)
   # Add ~/.ssh/sky-cluster-key to SSH config to allow nodes within a cluster to connect to each other

sky/utils/command_runner.py

@@ -325,6 +325,7 @@ class CommandRunner:
         direction = 'up' if up else 'down'
         error_msg = (f'Failed to rsync {direction}: {source} -> {target}. '
                      'Ensure that the network is stable, then retry.')
+
         subprocess_utils.handle_returncode(returncode,
                                            command,
                                            error_msg,
@@ -718,6 +719,7 @@ class KubernetesCommandRunner(CommandRunner):
     def __init__(
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
+        deployment: Optional[str] = None,
         **kwargs,
     ):
         """Initialize KubernetesCommandRunner.
@@ -733,11 +735,19 @@ class KubernetesCommandRunner(CommandRunner):
         del kwargs
         super().__init__(node)
         (self.namespace, self.context), self.pod_name = node
+        self.deployment = deployment
 
     @property
     def node_id(self) -> str:
         return f'{self.context}-{self.namespace}-{self.pod_name}'
 
+    @property
+    def kube_identifier(self) -> str:
+        if self.deployment is not None:
+            return f'deployment/{self.deployment}'
+        else:
+            return f'pod/{self.pod_name}'
+
     def port_forward_command(self,
                              port_forward: List[Tuple[int, int]],
                              connect_timeout: int = 1) -> List[str]:
@@ -758,11 +768,12 @@ class KubernetesCommandRunner(CommandRunner):
             kubectl_args += ['--context', self.context]
         local_port, remote_port = port_forward[0]
         local_port_str = f'{local_port}' if local_port is not None else ''
+
         kubectl_cmd = [
             'kubectl',
             *kubectl_args,
             'port-forward',
-            f'pod/{self.pod_name}',
+            self.kube_identifier,
             f'{local_port_str}:{remote_port}',
         ]
         return kubectl_cmd
@@ -785,7 +796,8 @@ class KubernetesCommandRunner(CommandRunner):
             source_bashrc: bool = False,
             skip_num_lines: int = 0,
             **kwargs) -> Union[int, Tuple[int, str, str]]:
-        """Uses 'kubectl exec' to run 'cmd' on a pod by its name and namespace.
+        """Uses 'kubectl exec' to run 'cmd' on a pod or deployment by its
+        name and namespace.
 
         Args:
             cmd: The command to run.
@@ -828,7 +840,9 @@ class KubernetesCommandRunner(CommandRunner):
         # case, need to set KUBECONFIG to /dev/null to avoid using kubeconfig.
         if self.context is None:
             kubectl_args += ['--kubeconfig', '/dev/null']
-        kubectl_args += [self.pod_name]
+
+        kubectl_args += [self.kube_identifier]
+
         if ssh_mode == SshMode.LOGIN:
             assert isinstance(cmd, list), 'cmd must be a list for login mode.'
             base_cmd = ['kubectl', 'exec', '-it', *kubectl_args, '--']

sky/utils/command_runner.pyi

@@ -206,6 +206,8 @@ class KubernetesCommandRunner(CommandRunner):
     def __init__(
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
+        deployment: Optional[str] = ...,
+        **kwargs,
     ) -> None:
         ...
 

sky/utils/controller_utils.py

@@ -193,6 +193,30 @@ class Controllers(enum.Enum):
         return None
 
 
+def high_availability_specified(cluster_name: Optional[str],
+                                skip_warning: bool = True) -> bool:
+    """Check if the controller high availability is specified in user config.
+    """
+    controller = Controllers.from_name(cluster_name)
+    if controller is None:
+        return False
+
+    if skypilot_config.loaded():
+        high_availability = skypilot_config.get_nested(
+            (controller.value.controller_type, 'controller',
+             'high_availability'), False)
+        if high_availability:
+            if controller.value.controller_type != 'serve':
+                if not skip_warning:
+                    print(f'{colorama.Fore.RED}High availability controller is'
+                          'only supported for SkyServe controller. It cannot'
+                          f'be enabled for {controller.value.name}.'
+                          f'Skipping this flag.{colorama.Style.RESET_ALL}')
+            else:
+                return True
+    return False
+
+
 # Install cli dependencies. Not using SkyPilot wheels because the wheel
 # can be cleaned up by another process.
 def _get_cloud_dependencies_installation_commands(