skypilot-nightly 1.0.0.dev20250326__py3-none-any.whl → 1.0.0.dev20250328__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '4f8721ab403668198ffd61297874c4c365e642b7'
+_SKYPILOT_COMMIT_SHA = '4dab9ac5b17fd6dc807b98dcbccecf481011835a'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250326'
+__version__ = '1.0.0.dev20250328'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/gcp.py

@@ -68,6 +68,13 @@ def credential_error_exception():
     return exceptions.DefaultCredentialsError
 
 
+@common.load_lazy_modules(_LAZY_MODULES)
+def auth_error_exception():
+    """GoogleAuthError exception."""
+    from google.auth import exceptions
+    return exceptions.GoogleAuthError
+
+
 @common.load_lazy_modules(_LAZY_MODULES)
 def gcp_auth_refresh_error_exception():
     """GCP auth refresh error exception."""

sky/authentication.py

@@ -33,6 +33,7 @@ import colorama
 import filelock
 
 from sky import clouds
+from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
@@ -214,6 +215,9 @@ def setup_gcp_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
             sys.exit(1)
         else:
             raise
+    except gcp.auth_error_exception() as e:
+        raise exceptions.InvalidCloudCredentials(
+            f'{common_utils.format_exception(e)}')
     except socket.timeout:
         logger.error('Socket timed out when trying to get the GCP project. '
                      'Please check your network connection.')

sky/backends/cloud_vm_ray_backend.py

@@ -1089,6 +1089,21 @@ class FailoverCloudErrorHandlerV2:
             FailoverCloudErrorHandlerV2._default_handler(
                 blocked_resources, launchable_resources, region, zones, error)
 
+    @staticmethod
+    def _aws_handler(blocked_resources: Set['resources_lib.Resources'],
+                     launchable_resources: 'resources_lib.Resources',
+                     region: 'clouds.Region',
+                     zones: Optional[List['clouds.Zone']],
+                     error: Exception) -> None:
+        logger.info(f'AWS handler error: {error}')
+        # Block AWS if the credential has expired.
+        if isinstance(error, exceptions.InvalidCloudCredentials):
+            _add_to_blocked_resources(
+                blocked_resources, resources_lib.Resources(cloud=clouds.AWS()))
+        else:
+            FailoverCloudErrorHandlerV2._default_handler(
+                blocked_resources, launchable_resources, region, zones, error)
+
     @staticmethod
     def _default_handler(blocked_resources: Set['resources_lib.Resources'],
                          launchable_resources: 'resources_lib.Resources',
@@ -1419,6 +1434,17 @@ class RetryingVmProvisioner(object):
                 # does not have nodes labeled with GPU types.
                 logger.info(f'{e}')
                 continue
+            except exceptions.InvalidCloudCredentials as e:
+                # Failed due to invalid cloud credentials.
+                logger.warning(f'{common_utils.format_exception(e)}')
+                # We should block the entire cloud for invalid cloud credentials
+                _add_to_blocked_resources(
+                    self._blocked_resources,
+                    to_provision.copy(region=None, zone=None))
+                raise exceptions.ResourcesUnavailableError(
+                    f'Failed to provision on cloud {to_provision.cloud} due to '
+                    f'invalid cloud credentials: '
+                    f'{common_utils.format_exception(e)}')
             except exceptions.InvalidCloudConfigs as e:
                 # Failed due to invalid user configs in ~/.sky/config.yaml.
                 logger.warning(f'{common_utils.format_exception(e)}')

sky/cli.py

@@ -5455,11 +5455,17 @@ def local():
     type=str,
     required=False,
     help='Name to use for the kubeconfig context. Defaults to "default".')
+@click.option('--password',
+              type=str,
+              required=False,
+              help='Password for the ssh-user to execute sudo commands. '
+              'Required only if passwordless sudo is not setup.')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, context_name: Optional[str], async_call: bool):
+             cleanup: bool, context_name: Optional[str],
+             password: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5505,7 +5511,7 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
     request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
-                              context_name)
+                              context_name, password)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 

sky/client/cli.py

@@ -5455,11 +5455,17 @@ def local():
     type=str,
     required=False,
     help='Name to use for the kubeconfig context. Defaults to "default".')
+@click.option('--password',
+              type=str,
+              required=False,
+              help='Password for the ssh-user to execute sudo commands. '
+              'Required only if passwordless sudo is not setup.')
 @local.command('up', cls=_DocumentedCodeCommand)
 @_add_click_options(_COMMON_OPTIONS)
 @usage_lib.entrypoint
 def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
-             cleanup: bool, context_name: Optional[str], async_call: bool):
+             cleanup: bool, context_name: Optional[str],
+             password: Optional[str], async_call: bool):
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
@@ -5505,7 +5511,7 @@ def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
                 f'Failed to read SSH key file {ssh_key_path}: {str(e)}')
 
     request_id = sdk.local_up(gpus, ip_list, ssh_user, ssh_key, cleanup,
-                              context_name)
+                              context_name, password)
     _async_call_or_wait(request_id, async_call, request_name='local up')
 
 

sky/client/sdk.py

@@ -1295,7 +1295,8 @@ def local_up(gpus: bool,
              ssh_user: Optional[str],
              ssh_key: Optional[str],
              cleanup: bool,
-             context_name: Optional[str] = None) -> server_common.RequestId:
+             context_name: Optional[str] = None,
+             password: Optional[str] = None) -> server_common.RequestId:
     """Launches a Kubernetes cluster on local machines.
 
     Returns:
@@ -1314,7 +1315,8 @@ def local_up(gpus: bool,
                                 ssh_user=ssh_user,
                                 ssh_key=ssh_key,
                                 cleanup=cleanup,
-                                context_name=context_name)
+                                context_name=context_name,
+                                password=password)
     response = requests.post(f'{server_common.get_server_url()}/local_up',
                              json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)

sky/clouds/do.py

@@ -1,6 +1,7 @@
 """ Digital Ocean Cloud. """
 
 import json
+import os
 import typing
 from typing import Dict, Iterator, List, Optional, Tuple, Union
 
@@ -279,13 +280,11 @@ class DO(clouds.Cloud):
         return True, None
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
-        try:
-            do_utils.client()
-            return {
-                f'~/.config/doctl/{_CREDENTIAL_FILE}': do_utils.CREDENTIALS_PATH
-            }
-        except do_utils.DigitalOceanError:
+        if not os.path.exists(os.path.expanduser(do_utils.CREDENTIALS_PATH)):
             return {}
+        return {
+            f'~/.config/doctl/{_CREDENTIAL_FILE}': do_utils.CREDENTIALS_PATH
+        }
 
     @classmethod
     def get_current_user_identity(cls) -> Optional[List[str]]:

sky/clouds/service_catalog/data_fetchers/fetch_ibm.py

@@ -0,0 +1,176 @@
+"""A script that generates the Lambda Cloud catalog.
+
+Usage:
+    python fetch_ibm.py [-h] [--api-key API_KEY]
+                             [--api-key-path API_KEY_PATH]
+
+If neither --api-key nor --api-key-path are provided, this script will parse
+`~/.ibm/credentials.yaml` to look for IBM API key (`iam_api_key`).
+"""
+import argparse
+import csv
+from datetime import datetime
+import json
+import os
+from typing import Dict, List, Optional, Tuple
+
+import requests
+import yaml
+
+TOKEN_ENDPOINT = 'https://iam.cloud.ibm.com/identity/token'
+REGIONS_ENDPOINT = f'https://us-south.iaas.cloud.ibm.com/v1/regions?version={datetime.today().strftime("%Y-%m-%d")}&generation=2'  # pylint: disable=line-too-long
+ENDPOINT = 'https://cloud.lambdalabs.com/api/v1/instance-types'
+DEFAULT_IBM_CREDENTIALS_PATH = os.path.expanduser('~/.ibm/credentials.yaml')
+
+
+def _fetch_token(api_key: Optional[str] = None,
+                 api_key_path: Optional[str] = None,
+                 ibm_token: Optional[str] = None) -> str:
+    if ibm_token is None:
+        if api_key is None:
+            if api_key_path is None:
+                api_key_path = DEFAULT_IBM_CREDENTIALS_PATH
+            with open(api_key_path, mode='r', encoding='utf-8') as f:
+                ibm_cred_yaml = yaml.safe_load(f)
+                api_key = ibm_cred_yaml['iam_api_key']
+
+        headers = {
+            'Accept': 'application/json',
+        }
+        data = {
+            'grant_type': 'urn:ibm:params:oauth:grant-type:apikey',
+            'apikey': api_key,
+        }
+        response = requests.post(url=TOKEN_ENDPOINT, data=data, headers=headers)
+        return response.json()['access_token']
+    else:
+        return ibm_token
+
+
+def _fetch_regions(ibm_token: str) -> List[str]:
+    headers = {
+        'Authorization': f'Bearer {ibm_token}',
+        'Accept': 'application/json',
+    }
+    response = requests.get(url=REGIONS_ENDPOINT, headers=headers)
+    regions_json = response.json()
+
+    regions = [r['name'] for r in regions_json['regions']]
+
+    print(f'regions: {regions}')
+    return regions
+
+
+def _fetch_instance_profiles(regions: List[str],
+                             ibm_token: str) -> Dict[str, Tuple[List, List]]:
+    """Fetch instance profiles by region (map):
+    {
+        "region_name": (
+            [list of available zones in the region],
+            [list of available instance profiles in the region]
+        )
+    }
+    """
+    d = datetime.today().strftime('%Y-%m-%d')
+
+    result = {}
+    headers = {
+        'Authorization': f'Bearer {ibm_token}',
+        'Accept': 'application/json',
+    }
+    for r in regions:
+        az_endpoint = f'https://{r}.iaas.cloud.ibm.com/v1/regions/{r}/zones?version={d}&generation=2'  # pylint: disable=line-too-long
+        az_response = requests.get(url=az_endpoint, headers=headers)
+        az_response_json = az_response.json()
+        zones = [a['name'] for a in az_response_json['zones']]
+        print(f'Fetching instance profiles for region {r}, zones {zones}')
+
+        instances_endpoint = f'https://{r}.iaas.cloud.ibm.com/v1/instance/profiles?version={d}&generation=2'  # pylint: disable=line-too-long
+        instance_response = requests.get(url=instances_endpoint,
+                                         headers=headers)
+        instance_response_json = instance_response.json()
+        instance_profiles = instance_response_json['profiles']
+        result[r] = (zones, instance_profiles)
+
+    return result
+
+
+def create_catalog(region_profile: Dict[str, Tuple[List, List]],
+                   output_path: str) -> None:
+    print(f'Create catalog file {output_path} based on the fetched profiles')
+    with open(output_path, mode='w', encoding='utf-8') as f:
+        writer = csv.writer(f, delimiter=',', quotechar='"')
+        writer.writerow([
+            'InstanceType', 'AcceleratorName', 'AcceleratorCount', 'vCPUs',
+            'MemoryGiB', 'GpuInfo', 'Price', 'SpotPrice', 'Region',
+            'AvailabilityZone'
+        ])
+
+        for region, (zones, profiles) in region_profile.items():
+            print(f'  adding region {region} instances')
+            for profile in profiles:
+                vm = profile['name']
+                gpu: Optional[str] = None
+                gpu_cnt: Optional[int] = None
+                gpu_manufacturer: Optional[str] = None
+                gpu_memory: Optional[int] = None
+                if 'gpu_model' in profile:
+                    gpu = profile['gpu_model']['values'][0]
+                if 'gpu_count' in profile:
+                    gpu_cnt = int(profile['gpu_count']['value'])
+                if 'vcpu_count' in profile:
+                    vcpus = int(profile['vcpu_count']['value'])
+                if 'memory' in profile:
+                    mem = int(profile['memory']['value'])
+                if 'gpu_memory' in profile:
+                    gpu_memory = int(profile['gpu_memory']['value'])
+                if 'gpu_manufacturer' in profile:
+                    gpu_manufacturer = profile['gpu_manufacturer']['values'][0]
+                # TODO: How to fetch prices?
+                #       The pricing API doesn't return prices for instance.profile. # pylint: disable=line-too-long
+                #       https://cloud.ibm.com/docs/account?topic=account-getting-pricing-api # pylint: disable=line-too-long
+                #       https://globalcatalog.cloud.ibm.com/api/v1?q=kind:instance.profile # pylint: disable=line-too-long
+                #       https://globalcatalog.cloud.ibm.com/api/v1/gx2-16x128x1v100/plan # pylint: disable=line-too-long
+                price = 0.0
+                gpuinfo: Optional[str] = None
+                gpu_memory_mb: Optional[int] = None
+                gpu_memory_total_mb: Optional[int] = None
+                if gpu_memory is not None:
+                    gpu_memory_mb = gpu_memory * 1024
+                    if gpu_cnt is not None:
+                        gpu_memory_total_mb = gpu_memory_mb * gpu_cnt
+                # gpuinfo: Optional[str] = None
+                if gpu is not None:
+                    gpuinfo_dict = {
+                        'Gpus': [{
+                            'Name': gpu,
+                            'Manufacturer': gpu_manufacturer,
+                            'Count': gpu_cnt,
+                            'MemoryInfo': {
+                                'SizeInMiB': gpu_memory_mb
+                            },
+                        }],
+                        'TotalGpuMemoryInMiB': gpu_memory_total_mb
+                    }
+                    gpuinfo = json.dumps(gpuinfo_dict).replace('"', "'")  # pylint: disable=inconsistent-quotes,invalid-string-quote
+
+                for zone in zones:
+                    writer.writerow([
+                        vm, gpu, gpu_cnt, vcpus, mem, gpuinfo, price, '',
+                        region, zone
+                    ])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--api-key', help='IBM API key.')
+    parser.add_argument('--api-key-path',
+                        help='path of file containing IBM Credentials.')
+    args = parser.parse_args()
+    os.makedirs('ibm', exist_ok=True)
+    call_token = _fetch_token()
+    call_regions = _fetch_regions(call_token)
+    region_profiles_map = _fetch_instance_profiles(regions=call_regions,
+                                                   ibm_token=call_token)
+    create_catalog(region_profiles_map, 'ibm/vms.csv')
+    print('IBM Cloud catalog saved to ibm/vms.csv')

sky/core.py

@@ -1063,7 +1063,8 @@ def local_up(gpus: bool,
              ssh_user: Optional[str],
              ssh_key: Optional[str],
              cleanup: bool,
-             context_name: Optional[str] = None) -> None:
+             context_name: Optional[str] = None,
+             password: Optional[str] = None) -> None:
     """Creates a local or remote cluster."""
 
     def _validate_args(ips, ssh_user, ssh_key, cleanup):
@@ -1089,7 +1090,8 @@ def local_up(gpus: bool,
     if ips:
         assert ssh_user is not None and ssh_key is not None
         kubernetes_deploy_utils.deploy_remote_cluster(ips, ssh_user, ssh_key,
-                                                      cleanup, context_name)
+                                                      cleanup, context_name,
+                                                      password)
     else:
         # Run local deployment (kind) if no remote args are specified
         kubernetes_deploy_utils.deploy_local_cluster(gpus)

sky/data/storage.py

@@ -203,7 +203,7 @@ class StoreType(enum.Enum):
             return 'oci://'
         # Nebius storages use 's3://' as a prefix for various aws cli commands
         elif self == StoreType.NEBIUS:
-            return 's3://'
+            return 'nebius://'
         else:
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(f'Unknown store type: {self}')
@@ -4888,8 +4888,8 @@ class NebiusStore(AbstractStore):
                 raise exceptions.StorageBucketGetError(
                     'Attempted to use a non-existent bucket as a source: '
                     f'{self.source}. Consider using `aws s3 ls '
-                    f'{self.source} --endpoint={endpoint_url}`'
-                    f'--profile={nebius.NEBIUS_PROFILE_NAME} to debug.')
+                    f's3://{self.name} --endpoint={endpoint_url}'
+                    f'--profile={nebius.NEBIUS_PROFILE_NAME}` to debug.')
 
         # If bucket cannot be found in both private and public settings,
         # the bucket is to be created by Sky. However, creation is skipped if

sky/exceptions.py

@@ -180,6 +180,11 @@ class InvalidCloudConfigs(Exception):
     pass
 
 
+class InvalidCloudCredentials(Exception):
+    """Raised when the cloud credentials are invalid."""
+    pass
+
+
 class ProvisionPrechecksError(Exception):
     """Raised when a managed job fails prechecks before provision.
 

sky/jobs/server/server.py

@@ -71,8 +71,14 @@ async def logs(
         request_name='jobs.logs',
         request_body=jobs_logs_body,
         func=core.tail_logs,
-        schedule_type=api_requests.ScheduleType.SHORT
-        if jobs_logs_body.refresh else api_requests.ScheduleType.LONG,
+        # TODO(aylei): We have tail logs scheduled as SHORT request, because it
+        # should be responsive. However, it can be long running if the user's
+        # job keeps running, and we should avoid it taking the SHORT worker
+        # indefinitely.
+        # When refresh is True we schedule it as LONG because a controller
+        # restart might be needed.
+        schedule_type=api_requests.ScheduleType.LONG
+        if jobs_logs_body.refresh else api_requests.ScheduleType.SHORT,
         request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
     request_task = api_requests.get_request(request.state.request_id)

sky/optimizer.py

@@ -713,7 +713,7 @@ class Optimizer:
         if message_data:
             metric = 'COST ($)' if minimize_cost else 'TIME (s)'
             table = _create_table(['SOURCE', 'TARGET', 'SIZE (GB)', metric])
-            table.add_rows(reversed(message_data))
+            table.add_rows(list(reversed(message_data)))
             logger.info(f'Egress plan:\n{table}\n')
 
     @staticmethod

sky/provision/aws/utils.py

@@ -1,6 +1,7 @@
 """Utils for AWS provisioner."""
 import colorama
 
+from sky import exceptions
 from sky import sky_logging
 
 logger = sky_logging.init_logger(__name__)
@@ -24,18 +25,17 @@ def handle_boto_error(exc: Exception, msg: str) -> None:
     generic_message = (f'{msg}\nError code: {colorama.Style.BRIGHT}{error_code}'
                        f'{colorama.Style.RESET_ALL}')
 
-    # apparently
-    # ExpiredTokenException
-    # ExpiredToken
-    # RequestExpired
-    # are all the same pretty much
-    credentials_expiration_codes = [
+    # For invalid credentials, like expired tokens or
+    # invalid tokens, raise InvalidCloudCredentials exception
+    invalid_credentials_codes = [
         'ExpiredTokenException',
         'ExpiredToken',
         'RequestExpired',
+        'InvalidClientTokenId',
+        'InvalidClientToken',
     ]
 
-    if error_code in credentials_expiration_codes:
+    if error_code in invalid_credentials_codes:
         # 'An error occurred (ExpiredToken) when calling the
         # GetInstanceProfile operation: The security token
         # included in the request is expired'
@@ -73,9 +73,10 @@ def handle_boto_error(exc: Exception, msg: str) -> None:
             f'{colorama.Style.RESET_ALL}\n'
             f'You can find a script that automates this at:'
             f'{aws_session_script_url}')
-        # Do not re-raise the exception here because it looks awful
-        # and we already print all the info in verbose
-        raise SystemExit(1)
+        # Raise the InvalidCloudCredentials exception so that
+        # the provisioner can failover to other clouds
+        raise exceptions.InvalidCloudCredentials(
+            f'InvalidCloudCredentials: {generic_message}') from exc
 
     # todo: any other errors that we should catch separately?
 

sky/provision/azure/config.py

@@ -120,7 +120,7 @@ def bootstrap_instances(
             except azure.exceptions().ClientAuthenticationError as e:
                 message = (
                     'Failed to authenticate with Azure. Please check your '
-                    'Azure credentials. Error: '
+                    'Azure credentials. ClientAuthenticationError: '
                     f'{common_utils.format_exception(e)}').replace('\n', ' ')
                 logger.error(message)
                 raise exceptions.NoClusterLaunchedError(message) from e

sky/provision/kubernetes/utils.py

@@ -159,7 +159,10 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
                     # or 403 (Forbidden)
                     if (isinstance(e, kubernetes.api_exception()) and
                             e.status in (401, 403)):
-                        raise
+                        # Raise KubeAPIUnreachableError exception so that the
+                        # optimizer/provisioner can failover to other clouds.
+                        raise exceptions.KubeAPIUnreachableError(
+                            f'Kubernetes API error: {str(e)}') from e
                     if attempt < max_retries - 1:
                         sleep_time = backoff.current_backoff()
                         logger.debug(f'Kubernetes API call {func.__name__} '

sky/provision/provisioner.py

@@ -145,6 +145,10 @@ def bulk_provision(
         except exceptions.NoClusterLaunchedError:
             # Skip the teardown if the cluster was never launched.
             raise
+        except exceptions.InvalidCloudCredentials:
+            # Skip the teardown if the cloud config is expired and
+            # the provisioner should failover to other clouds.
+            raise
         except Exception:  # pylint: disable=broad-except
             zone_str = 'all zones'
             if zones:

sky/server/requests/payloads.py

@@ -464,6 +464,7 @@ class LocalUpBody(RequestBody):
     ssh_key: Optional[str] = None
     cleanup: bool = False
     context_name: Optional[str] = None
+    password: Optional[str] = None
 
 
 class ServeTerminateReplicaBody(RequestBody):

sky/utils/kubernetes/deploy_remote_cluster.sh

@@ -12,6 +12,7 @@ NC='\033[0m' # No color
 CLEANUP=false
 INSTALL_GPU=false
 POSITIONAL_ARGS=()
+PASSWORD=""
 
 # Process all arguments
 while [[ $# -gt 0 ]]; do
@@ -20,6 +21,11 @@ while [[ $# -gt 0 ]]; do
             CLEANUP=true
             shift
             ;;
+        --password)
+            PASSWORD=$2
+            shift
+            shift
+            ;;
         *)
             POSITIONAL_ARGS+=("$1")
             shift
@@ -36,11 +42,23 @@ USER=$2
 SSH_KEY=$3
 CONTEXT_NAME=${4:-default}
 K3S_TOKEN=mytoken  # Any string can be used as the token
+# Create temporary askpass script for sudo
+ASKPASS_BLOCK="# Create temporary askpass script
+ASKPASS_SCRIPT=\$(mktemp)
+trap 'rm -f \$ASKPASS_SCRIPT' EXIT INT TERM ERR QUIT
+cat > \$ASKPASS_SCRIPT << EOF
+#!/bin/bash
+echo $PASSWORD
+EOF
+chmod 700 \$ASKPASS_SCRIPT
+# Use askpass
+export SUDO_ASKPASS=\$ASKPASS_SCRIPT
+"
 
 # Basic argument checks
 if [ -z "$IPS_FILE" ] || [ -z "$USER" ] || [ -z "$SSH_KEY" ]; then
     >&2 echo -e "${RED}Error: Missing required arguments.${NC}"
-    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [context-name] [--cleanup]"
+    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [context-name] [--cleanup] [--password password]"
     exit 1
 fi
 
@@ -89,9 +107,10 @@ cleanup_server_node() {
     local NODE_IP=$1
     echo -e "${YELLOW}Cleaning up head node $NODE_IP...${NC}"
     run_remote "$NODE_IP" "
+        $ASKPASS_BLOCK
         echo 'Uninstalling k3s...' &&
-        /usr/local/bin/k3s-uninstall.sh || true &&
-        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+        sudo -A /usr/local/bin/k3s-uninstall.sh || true &&
+        sudo -A rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
     "
     echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
 }
@@ -101,9 +120,10 @@ cleanup_agent_node() {
     local NODE_IP=$1
     echo -e "${YELLOW}Cleaning up node $NODE_IP...${NC}"
     run_remote "$NODE_IP" "
+        $ASKPASS_BLOCK
         echo 'Uninstalling k3s...' &&
-        /usr/local/bin/k3s-agent-uninstall.sh || true &&
-        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+        sudo -A /usr/local/bin/k3s-agent-uninstall.sh || true &&
+        sudo -A rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
     "
     echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
 }
@@ -151,10 +171,11 @@ fi
 # Step 1: Install k3s on the head node
 progress_message "Deploying Kubernetes on head node ($HEAD_NODE)..."
 run_remote "$HEAD_NODE" "
-    curl -sfL https://get.k3s.io | K3S_TOKEN=$K3S_TOKEN sh - &&
+    $ASKPASS_BLOCK
+    curl -sfL https://get.k3s.io | K3S_TOKEN=$K3S_TOKEN sudo -E -A sh - &&
     mkdir -p ~/.kube &&
-    sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config &&
-    sudo chown \$(id -u):\$(id -g) ~/.kube/config &&
+    sudo -A cp /etc/rancher/k3s/k3s.yaml ~/.kube/config &&
+    sudo -A chown \$(id -u):\$(id -g) ~/.kube/config &&
     for i in {1..3}; do
         if kubectl wait --for=condition=ready node --all --timeout=2m --kubeconfig ~/.kube/config; then
             break
@@ -163,7 +184,7 @@ run_remote "$HEAD_NODE" "
             sleep 5
         fi
     done
-    if [ $i -eq 3 ]; then
+    if [ \$i -eq 3 ]; then
         echo 'Failed to wait for nodes to be ready after 3 attempts'
         exit 1
     fi"
@@ -184,7 +205,8 @@ echo -e "${GREEN}Master node internal IP: $MASTER_ADDR${NC}"
 for NODE in $WORKER_NODES; do
     progress_message "Deploying Kubernetes on worker node ($NODE)..."
     run_remote "$NODE" "
-        curl -sfL https://get.k3s.io | K3S_URL=https://$MASTER_ADDR:6443 K3S_TOKEN=$K3S_TOKEN sh -"
+        $ASKPASS_BLOCK
+        curl -sfL https://get.k3s.io | K3S_URL=https://$MASTER_ADDR:6443 K3S_TOKEN=$K3S_TOKEN sudo -E -A sh -"
     success_message "Kubernetes deployed on worker node ($NODE)."
 
     # Check if worker node has a GPU
@@ -247,12 +269,13 @@ echo "Cluster deployment completed. You can now run 'kubectl get nodes' to verif
 if [ "$INSTALL_GPU" == "true" ]; then
     echo -e "${YELLOW}GPU detected in the cluster. Installing Nvidia GPU Operator...${NC}"
     run_remote "$HEAD_NODE" "
+        $ASKPASS_BLOCK
         curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 &&
         chmod 700 get_helm.sh &&
         ./get_helm.sh &&
         helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update &&
         kubectl create namespace gpu-operator --kubeconfig ~/.kube/config || true &&
-        sudo ln -s /sbin/ldconfig /sbin/ldconfig.real || true &&
+        sudo -A ln -s /sbin/ldconfig /sbin/ldconfig.real || true &&
         helm install gpu-operator -n gpu-operator --create-namespace nvidia/gpu-operator \
         --set 'toolkit.env[0].name=CONTAINERD_CONFIG' \
         --set 'toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml' \

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -24,7 +24,8 @@ def deploy_remote_cluster(ip_list: List[str],
                           ssh_user: str,
                           ssh_key: str,
                           cleanup: bool,
-                          context_name: Optional[str] = None):
+                          context_name: Optional[str] = None,
+                          password: Optional[str] = None):
     success = False
     path_to_package = os.path.dirname(__file__)
     up_script_path = os.path.join(path_to_package, 'deploy_remote_cluster.sh')
@@ -47,6 +48,8 @@ def deploy_remote_cluster(ip_list: List[str],
                           f'{ssh_user} {key_file.name}')
         if context_name is not None:
             deploy_command += f' {context_name}'
+        if password is not None:
+            deploy_command += f' --password {password}'
         if cleanup:
             deploy_command += ' --cleanup'
 

{skypilot_nightly-1.0.0.dev20250326.dist-info → skypilot_nightly-1.0.0.dev20250328.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250326
+Version: 1.0.0.dev20250328
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250326.dist-info → skypilot_nightly-1.0.0.dev20250328.dist-info}/RECORD

@@ -1,16 +1,16 @@
-sky/__init__.py,sha256=ZQgnwZFx1OI8QGgvePoEaTVl6x9kc5Opq9Vsu0qdHWQ,6428
+sky/__init__.py,sha256=4GWeKYvInaF3mKBoHHRbOaX6jSHOfnjY0yHaGLpRmPQ,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
-sky/authentication.py,sha256=QWybUjX-O6TXzCDSu2vtSgyeQb-OY9B6gDpT-jwYK9I,22700
+sky/authentication.py,sha256=ND011K_-Ud1dVZF37A9KrwYir_ihJXcHc7iDWmuBc8Q,22872
 sky/check.py,sha256=iMGuM7yjUPRgDHc13Pf1_LSybBqIexq-6aXfnVcaI54,15898
-sky/cli.py,sha256=o1IHTY4YSJzsfkynTynaN1dB4RBPZIRZGWQ4rTnVVnQ,221956
+sky/cli.py,sha256=qEyX_VHT96vXQH40BzTqQZXv4iqBg8SFlT5ezed8X7o,222226
 sky/cloud_stores.py,sha256=yo8xQGr8iabYGmn0E6o6jHsfJiomXCTCcR-WMLnNYTQ,26718
-sky/core.py,sha256=BpZ5v-2aQEUkE3izBfMRSVFrmuTWY5_X_siHRf_ZRtI,47817
+sky/core.py,sha256=G3n6z0dyvoU4FJVGnnTu3kFdu_EtQC1l57er5voRAX0,47926
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
-sky/exceptions.py,sha256=wSLZlhXRTg4EmB1a0GIusaq5izhAReCV1krN_q5pG2s,16727
+sky/exceptions.py,sha256=EodMj6P0x0JYBiioLNuPoFMj12HzlQTQpfXNX6zHBoA,16837
 sky/execution.py,sha256=9L8NFOXNphtabnsL7mHGPJeGdw4n6gIIUEOzjW7CEHw,28294
 sky/global_user_state.py,sha256=uGmwtUL-HVKGqEwlEvIXFcQ_w2IxGE4msnffXUxCIGI,33574
 sky/models.py,sha256=4xSW05BdDPEjW8Ubvj3VlVOVnzv0TbrolsFvR5R5v1U,638
-sky/optimizer.py,sha256=nzM78zTq0zqs6DqF4eFyV3OqZy4wdorYDivr2tQFHyQ,60875
+sky/optimizer.py,sha256=Tv7Qz7uOEQWRP3CboHSOixOl7IXn8dicpn4ypsWhVUw,60881
 sky/resources.py,sha256=2qc5U09MFDaJjI1dHcThcRodpMGY7HyXzQn8eC4lvbE,72402
 sky/sky_logging.py,sha256=pID2RINjH62n7SZpv70DuN8BSFYdCfTJ2ScGQpVmugg,5725
 sky/skypilot_config.py,sha256=CdaIbPL_7ECG5laOARca4p9df_6NLhT-bO8WnalxZAY,8839
@@ -23,7 +23,7 @@ sky/adaptors/common.py,sha256=o17TOFNLm1t0lQvZSJCF1C6RjD6ykKrQ-q1BLPX41XI,3027
 sky/adaptors/cudo.py,sha256=WGvIQrlzJkGDe02Ve7pygA56tHwUc4kwS3XHW8kMFAA,239
 sky/adaptors/do.py,sha256=dJ0BYbkQoUWVu6_9Pxq3fOu6PngjZyyCQzgjnODXLCA,777
 sky/adaptors/docker.py,sha256=_kzpZ0fkWHqqQAVVl0llTsCE31KYz3Sjn8psTBQHVkA,468
-sky/adaptors/gcp.py,sha256=EEjiF_n0HGH3TEjSbY6PQ8nczlhV2kSdUCXe0l8DmJg,3316
+sky/adaptors/gcp.py,sha256=oEb9jClEtApw6PQnxdxDYxOCYsedvM3aiko1EW1FDVo,3501
 sky/adaptors/ibm.py,sha256=7YbHrWbYcZsJDgxMBNZr1yBI03mjs_C3pnCTCz-MNtQ,5068
 sky/adaptors/kubernetes.py,sha256=BfbUYMc9AMPvtoPX8f3qVI_tY2mLfGEJI9wSXLJftQs,6678
 sky/adaptors/nebius.py,sha256=B3KxPYhTJA3zjQ7x9k8_sqFTAkPYXldnM7PAuwy02gA,7512
@@ -34,7 +34,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
 sky/backends/backend_utils.py,sha256=ndY4IPs1F9QovyiKAnB1FNYGWm52_ylwf_K7wY50cv0,134922
-sky/backends/cloud_vm_ray_backend.py,sha256=8uMkhOzJqeHDkh1212RUe8HrT5fLYTdxe-beTXgygR8,248154
+sky/backends/cloud_vm_ray_backend.py,sha256=rWz9eUSq9TauA0HWhwYWm2DaGT2p8yDRxXvigcgx-uA,249578
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=meypuMaygSXXjGdXfq6dhWl-OrpAybg9KVRoup4D0wU,9098
@@ -43,15 +43,15 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=7rf-iHt6RXZ_pnBBWOMwcdodHQW69x27xNyx0yVog1U,26385
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=o1IHTY4YSJzsfkynTynaN1dB4RBPZIRZGWQ4rTnVVnQ,221956
+sky/client/cli.py,sha256=qEyX_VHT96vXQH40BzTqQZXv4iqBg8SFlT5ezed8X7o,222226
 sky/client/common.py,sha256=YMgYoSYlV3B_scfCIJRKHvDq6JKvnSvRZDhlRjRmzu0,14780
-sky/client/sdk.py,sha256=G-nrHjs6-Xq66zX9CHbDOuzVU-VM_rj2NkCGzS0eS0o,68524
+sky/client/sdk.py,sha256=36tvJou2IaG8H5lrqMicv-gSJn6VUDHUtB7V20t4lA8,68620
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
 sky/clouds/aws.py,sha256=-feXZ72UMUjuZz8dV4qMTjMMTC9pwfkQQT3KJDcIv5A,54633
 sky/clouds/azure.py,sha256=Zpo6ftWz_B30mX7N-An7JVO-8v7aU3f9cw1iH9phvwE,32251
 sky/clouds/cloud.py,sha256=OoSyFNYtby2Y0h2TpfMB_lEeolIZOQcfKgIn6AvRC68,36694
 sky/clouds/cudo.py,sha256=_UkLEtwJsfDMKlmJfML5W3rA8VArba4x8YGIdnvgZoM,13226
-sky/clouds/do.py,sha256=AHbdq5HqxqxKYp3sOCvywXme_eLckYtn1x-1yhlNH7U,11522
+sky/clouds/do.py,sha256=ZCJa3PTwnnO-mnX-97gD-7vSmo4tsXLLQU4KfnHogMo,11512
 sky/clouds/fluidstack.py,sha256=jIqW1MLe55MVME1PATZm8e6_FsiTnJawW7OdytPW0aM,12666
 sky/clouds/gcp.py,sha256=sUJ9LXUnMxYm6OYZ5P-z1dJHxgVILuC3OW3eFSTNCv8,56919
 sky/clouds/ibm.py,sha256=XtuPN8QgrwJdb1qb_b-7KwAE2tf_N9wh9eEfi2tcg-s,22013
@@ -91,6 +91,7 @@ sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=7YVnoGDGGZI2TK02b
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=hsqpQi_YUI-qil3zLCEGatrR7BkWzywr4otRdHrd-4k,7350
 sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=JnugFifzHPQITlbDKoKexE8NqgagOEfQWTxon7P6vJ0,30935
+sky/clouds/service_catalog/data_fetchers/fetch_ibm.py,sha256=i8zW9TFI_McedGoTe7Mnw6bP30W6dnCEbGJV1ta1VV0,7481
 sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=MUzogyLruLQmIt-To6TsfnGPgv_nnlp49XYbeshsd7I,5003
 sky/clouds/service_catalog/data_fetchers/fetch_vast.py,sha256=MRxk52FUeG-R2hPUbkH44HXRPou73dxXWYAHDEXg3xU,5016
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=Opp2r3KSzXPtwk3lKNbO8IX9QzjoRSwy1kW3jPjtS1c,21453
@@ -104,7 +105,7 @@ sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=-JcnVa_LT0kQejcSCnBwYtxhuuaNDPf_Q5oz62p186c,11973
 sky/data/data_utils.py,sha256=m1rrspfkgux7awWwTvQLFBjVix2KbKgRwgnhJDv3pcw,29895
 sky/data/mounting_utils.py,sha256=gLplBNKRQvVUBdkIRZO-wW4wpfY_F8xmkMyMfSZasTQ,16832
-sky/data/storage.py,sha256=FSWTfNqtRHoz5TFHD8R6lRqxat0RVLA_Dqi1NtR7mU0,233449
+sky/data/storage.py,sha256=AfSZ5gAtfpBXmo5mXHYk1thvq8l-I3CZbcax8C4OPmE,233456
 sky/data/storage_utils.py,sha256=GEERbieI4RoQ4LvzVBieVuvUaDDvfP958edjp8tiWNc,12935
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
 sky/jobs/constants.py,sha256=1XiIqdR5dEgGgepLKWkZCRT3MYSsMBR-dO7N4RTsjwg,3088
@@ -121,7 +122,7 @@ sky/jobs/dashboard/templates/index.html,sha256=NrlTDiEHJDt7sViwWgXUSxVCyVl_IEukE
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 sky/jobs/server/core.py,sha256=IHbGMEEbEdNS_ivIStv3rYVH6XodFgufrzZb3HGF9uo,25257
 sky/jobs/server/dashboard_utils.py,sha256=2Mbx40W1pQqPEPHsSDbHeaF0j5cgyKy-_A9Owdwp_AQ,2315
-sky/jobs/server/server.py,sha256=UIg2C0SkeGc_GXGudLWXoJePfPNO30n8qvqrcbgZCec,8018
+sky/jobs/server/server.py,sha256=pLobr0ITZJmhWUX-QrUwpZMgm7qy4_PHFuAnp4Xikac,8386
 sky/provision/__init__.py,sha256=LzOo5LjkRXwSf29dUqN14YbjzQu3liXLQcmweTeZ4dE,6457
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
@@ -129,14 +130,14 @@ sky/provision/docker_utils.py,sha256=ENm0LkyrYWic3Ikyacho8X5uDMvGsbkZQsb6kNH1DuI
 sky/provision/instance_setup.py,sha256=8ePBfK8qVb9IWmK7vKL8huXrBj5IdN2WsEbiQ6FtGqQ,24388
 sky/provision/logging.py,sha256=_sx_TH6nLt0FF3myS5pEZbiMhXyl4s1XwMidu_TTBUw,2091
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
-sky/provision/provisioner.py,sha256=tJEk5PcZ0i61j_4LfcMMbRhnONti5omRammzXswpfkU,29679
+sky/provision/provisioner.py,sha256=XBUnDxPALMj0CiBMtJY8s_1Q-V5rWWL07olk4jzua_g,29878
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
 sky/provision/aws/config.py,sha256=fGOkkn89EpVFYp4x0LTIauBem2Vsqcy8faEadS8l0nE,25979
 sky/provision/aws/instance.py,sha256=3okAJ1WkZ0YemH9kvUgA6HpHN6BP7unhDL-CHny1qFI,40849
-sky/provision/aws/utils.py,sha256=m49pS-SHGW7Au3bhDeTPsL8N5iRzbwOXzyEWRCc1Vho,3238
+sky/provision/aws/utils.py,sha256=LrjeQ09zA7GoMv9Nt8TlL2A3VqqChsgJ9bL-Q5VLaao,3401
 sky/provision/azure/__init__.py,sha256=87cgk1_Ws7n9rqaDDPv-HpfrkVeSQMdFQnhnXwyx9g4,548
 sky/provision/azure/azure-config-template.json,sha256=jrjAgOtpe0e6FSg3vsVqHKQqJe0w-HeWOFT1HuwzS2c,4712
-sky/provision/azure/config.py,sha256=V5-0Zelt4Xo0vcqnD6PpsnaCS7vc3xosDelILDAKSW4,8885
+sky/provision/azure/config.py,sha256=4MdatCG7-38vq8azy9kNDYhNdmndIBTEoQQzeQqPlJ8,8905
 sky/provision/azure/instance.py,sha256=BQ4stZl_m76iULPY_TFEN4WDwNPWI1jmw-LswxxqXPg,47495
 sky/provision/cudo/__init__.py,sha256=KAEl26MVPsk7IoP9Gg-MOJJRIV6-X9B0fbyHdyJWdLo,741
 sky/provision/cudo/config.py,sha256=RYOVkV0MoUqVBJRZiKhBZhjFygeyFs7eUdVMdPg1vds,327
@@ -165,7 +166,7 @@ sky/provision/kubernetes/constants.py,sha256=dZCUV8FOO9Gct80sdqeubKnxeW3CGl-u5mx
 sky/provision/kubernetes/instance.py,sha256=oag17OtuiqU-1RjkgW9NvEpxSGUFIYdI7M61S-YmPu8,50503
 sky/provision/kubernetes/network.py,sha256=AtcOM8wPs_-UlQJhGEQGP6Lh4HIgdx63Y0iWEhP5jyc,12673
 sky/provision/kubernetes/network_utils.py,sha256=6uck1aBkgtm-gGBitU3_hEUp8j14ZuG_4Xo70ReZYXs,11654
-sky/provision/kubernetes/utils.py,sha256=x14q_0SZ5RGdyXELaLtYx7UAptcXdq4MnbepGbFEKX0,124368
+sky/provision/kubernetes/utils.py,sha256=37esnQgQgeXfxwWyMqAo6SXCz0EyVvKECy2DOWISDEo,124630
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -237,7 +238,7 @@ sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/event_loop.py,sha256=OhpPbuce65bbjpGRlcJa78AVnYSm08SzFKt70ypCUuQ,1211
 sky/server/requests/executor.py,sha256=BNJqkTQ3swYeRO5YVW-dTmobL2CYnDDf_m-kY7__n40,21684
-sky/server/requests/payloads.py,sha256=6egfR6QSpd1WZP7K0joCQOrA0K6n7iTQbKL02T_RCOM,16494
+sky/server/requests/payloads.py,sha256=K9CzEqRLDdU2R7s3yxPDD2vC-h6beD_RPdSRpjoQzu4,16529
 sky/server/requests/preconditions.py,sha256=ipxIb_3JXG6S3-ymcOdqQNb7VDvoPqADxu9ZK7-nQWc,7179
 sky/server/requests/requests.py,sha256=9ovdQE-zv_Mvc6IbGATHVyQlOxSKjg_OankZbgDVGeE,21338
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -338,19 +339,19 @@ sky/utils/cli_utils/status_utils.py,sha256=LwGXzMgvnQeGR1fCC24q38hRLuAPeeSDkQ387
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=VLXfazav9XCMQmeKVqhuOQzt2vM6G1jgnvvb0SHUFno,7773
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
-sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=SGnqa5ks5Og-F96S_PIUbKzuKAtTF5cYOrXIg5XWx_E,10179
+sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=EQMBC0VUqe3UaiYvwkNq4P6U9bkiBwjTNTE0z-MsTBc,10785
 sky/utils/kubernetes/exec_kubeconfig_converter.py,sha256=fE1SnteoxI05EaugnWeV82hXwZTVHmbXsh1aaZAgF3c,2548
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=MBvXJio0PeujZSCXiRKE_pa6HCTiU9qBzR1WrXccVSY,10477
 sky/utils/kubernetes/gpu_labeler.py,sha256=9VZkC4UuR4vtE6TAz5P0i279Rn8KEaHE35JA9OVHxqI,7006
 sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
-sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=NtfbovAECN2A_R8x4FiNPaBVPWeuwaOVuuGsnVOSk2g,10231
+sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=HPVgNt-wbCVPd9dpDFiA7t2mzQLpjXHJ61eiwRbEr-c,10378
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250326.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250326.dist-info/METADATA,sha256=V5UlwHyFmeBcPPaUBEzuhCvdyoXArUCqy4zdQrckbMg,18657
-skypilot_nightly-1.0.0.dev20250326.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-skypilot_nightly-1.0.0.dev20250326.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250326.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250326.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250328.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250328.dist-info/METADATA,sha256=Ean7z4vIG2Xc5ExZ6qUOl-9-84h1smKzaRcmvV4ec18,18657
+skypilot_nightly-1.0.0.dev20250328.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250328.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250328.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250328.dist-info/RECORD,,