skypilot-nightly 1.0.0.dev20240927__py3-none-any.whl → 1.0.0.dev20240929__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'e6b8d2c086544ab5cfdb877ad414eafddaa49cb4'
+_SKYPILOT_COMMIT_SHA = 'e6a3b830fb2a12871815773af6171d42e0416e89'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240927'
+__version__ = '1.0.0.dev20240929'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/authentication.py

@@ -380,6 +380,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     secret_field_name = clouds.Kubernetes().ssh_key_secret_field_name
     context = config['provider'].get(
         'context', kubernetes_utils.get_current_kube_config_context_name())
+    if context == kubernetes_utils.IN_CLUSTER_REGION:
+        # If the context is set to IN_CLUSTER_REGION, we are running in a pod
+        # with in-cluster configuration. We need to set the context to None
+        # to use the mounted service account.
+        context = None
     namespace = config['provider'].get(
         'namespace',
         kubernetes_utils.get_kube_config_context_namespace(context))

sky/cli.py

@@ -5072,15 +5072,7 @@ def local():
     pass
 
 
-@click.option('--gpus/--no-gpus',
-              default=True,
-              is_flag=True,
-              help='Launch cluster without GPU support even '
-              'if GPUs are detected on the host.')
-@local.command('up', cls=_DocumentedCodeCommand)
-@usage_lib.entrypoint
-def local_up(gpus: bool):
-    """Creates a local cluster."""
+def _deploy_local_cluster(gpus: bool):
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -5206,6 +5198,124 @@ def local_up(gpus: bool):
             f'{gpu_hint}')
 
 
+def _deploy_remote_cluster(ip_file: str, ssh_user: str, ssh_key_path: str,
+                           cleanup: bool):
+    success = False
+    path_to_package = os.path.dirname(os.path.dirname(__file__))
+    up_script_path = os.path.join(path_to_package, 'sky/utils/kubernetes',
+                                  'deploy_remote_cluster.sh')
+    # Get directory of script and run it from there
+    cwd = os.path.dirname(os.path.abspath(up_script_path))
+
+    deploy_command = f'{up_script_path} {ip_file} {ssh_user} {ssh_key_path}'
+    if cleanup:
+        deploy_command += ' --cleanup'
+
+    # Convert the command to a format suitable for subprocess
+    deploy_command = shlex.split(deploy_command)
+
+    # Setup logging paths
+    run_timestamp = backend_utils.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_up.log')
+    tail_cmd = 'tail -n100 -f ' + log_path
+
+    # Check if ~/.kube/config exists:
+    if os.path.exists(os.path.expanduser('~/.kube/config')):
+        click.echo('Found existing kube config. '
+                   'It will be backed up to ~/.kube/config.bak.')
+    style = colorama.Style
+    click.echo('To view detailed progress: '
+               f'{style.BRIGHT}{tail_cmd}{style.RESET_ALL}')
+    if cleanup:
+        msg_str = 'Cleaning up remote cluster...'
+    else:
+        msg_str = 'Deploying remote cluster...'
+    with rich_utils.safe_status(f'[bold cyan]{msg_str}'):
+        returncode, _, stderr = log_lib.run_with_log(
+            cmd=deploy_command,
+            log_path=log_path,
+            require_outputs=True,
+            stream_logs=False,
+            line_processor=log_utils.SkyRemoteUpLineProcessor(),
+            cwd=cwd)
+    if returncode == 0:
+        success = True
+    else:
+        with ux_utils.print_exception_no_traceback():
+            raise RuntimeError(
+                'Failed to deploy remote cluster. '
+                f'Full log: {log_path}'
+                f'\nError: {style.BRIGHT}{stderr}{style.RESET_ALL}')
+
+    if success:
+        if cleanup:
+            click.echo(f'{colorama.Fore.GREEN}'
+                       '🎉 Remote cluster cleaned up successfully.'
+                       f'{style.RESET_ALL}')
+        else:
+            click.echo('Cluster deployment done. You can now run tasks on '
+                       'this cluster.\nE.g., run a task with: '
+                       'sky launch --cloud kubernetes -- echo hello world.'
+                       f'\n{colorama.Fore.GREEN}🎉 Remote cluster deployed '
+                       f'successfully. {style.RESET_ALL}')
+
+
+@click.option('--gpus/--no-gpus',
+              default=True,
+              is_flag=True,
+              help='Launch cluster without GPU support even '
+              'if GPUs are detected on the host.')
+@click.option(
+    '--ips',
+    type=str,
+    required=False,
+    help='Path to the file containing IP addresses of remote machines.')
+@click.option('--ssh-user',
+              type=str,
+              required=False,
+              help='SSH username for accessing remote machines.')
+@click.option('--ssh-key-path',
+              type=str,
+              required=False,
+              help='Path to the SSH private key.')
+@click.option('--cleanup',
+              is_flag=True,
+              help='Clean up the remote cluster instead of deploying it.')
+@local.command('up', cls=_DocumentedCodeCommand)
+@usage_lib.entrypoint
+def local_up(gpus: bool, ips: str, ssh_user: str, ssh_key_path: str,
+             cleanup: bool):
+    """Creates a local or remote cluster."""
+
+    def _validate_args(ips, ssh_user, ssh_key_path, cleanup):
+        # If any of --ips, --ssh-user, or --ssh-key-path is specified,
+        # all must be specified
+        if bool(ips) or bool(ssh_user) or bool(ssh_key_path):
+            if not (ips and ssh_user and ssh_key_path):
+                raise click.BadParameter(
+                    'All --ips, --ssh-user, and --ssh-key-path '
+                    'must be specified together.')
+
+        # --cleanup can only be used if --ips, --ssh-user and --ssh-key-path
+        # are all provided
+        if cleanup and not (ips and ssh_user and ssh_key_path):
+            raise click.BadParameter('--cleanup can only be used with '
+                                     '--ips, --ssh-user and --ssh-key-path.')
+
+    _validate_args(ips, ssh_user, ssh_key_path, cleanup)
+
+    # If remote deployment arguments are specified, run remote up script
+    if ips and ssh_user and ssh_key_path:
+        # Convert ips and ssh_key_path to absolute paths
+        ips = os.path.abspath(ips)
+        ssh_key_path = os.path.abspath(ssh_key_path)
+        _deploy_remote_cluster(ips, ssh_user, ssh_key_path, cleanup)
+    else:
+        # Run local deployment (kind) if no remote args are specified
+        _deploy_local_cluster(gpus)
+
+
 @local.command('down', cls=_DocumentedCodeCommand)
 @usage_lib.entrypoint
 def local_down():

sky/clouds/kubernetes.py

@@ -129,11 +129,24 @@ class Kubernetes(clouds.Cloud):
                 'Ignoring these contexts.')
 
     @classmethod
-    def _existing_allowed_contexts(cls) -> List[str]:
-        """Get existing allowed contexts."""
+    def _existing_allowed_contexts(cls) -> List[Optional[str]]:
+        """Get existing allowed contexts.
+
+        If None is returned in the list, it means that we are running in a pod
+        with in-cluster auth. In this case, we specify None context, which will
+        use the service account mounted in the pod.
+        """
         all_contexts = kubernetes_utils.get_all_kube_config_context_names()
-        if all_contexts is None:
+        if len(all_contexts) == 0:
             return []
+        if all_contexts == [None]:
+            # If only one context is found and it is None, we are running in a
+            # pod with in-cluster auth. In this case, we allow it to be used
+            # without checking against allowed_contexts.
+            # TODO(romilb): We may want check in-cluster auth against
+            #  allowed_contexts in the future by adding a special context name
+            #  for in-cluster auth.
+            return [None]
         all_contexts = set(all_contexts)
 
         allowed_contexts = skypilot_config.get_nested(
@@ -164,7 +177,15 @@ class Kubernetes(clouds.Cloud):
         del accelerators, zone, use_spot  # unused
         existing_contexts = cls._existing_allowed_contexts()
 
-        regions = [clouds.Region(context) for context in existing_contexts]
+        regions = []
+        for context in existing_contexts:
+            if context is None:
+                # If running in-cluster, we allow the region to be set to the
+                # singleton region since there is no context name available.
+                regions.append(clouds.Region(
+                    kubernetes_utils.IN_CLUSTER_REGION))
+            else:
+                regions.append(clouds.Region(context))
 
         if region is not None:
             regions = [r for r in regions if r.name == region]
@@ -541,13 +562,20 @@ class Kubernetes(clouds.Cloud):
     def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
         if region == self._LEGACY_SINGLETON_REGION:
             # For backward compatibility, we allow the region to be set to the
-            # legacy singletonton region.
+            # legacy singleton region.
             # TODO: Remove this after 0.9.0.
             return region, zone
 
+        if region == kubernetes_utils.IN_CLUSTER_REGION:
+            # If running incluster, we set region to IN_CLUSTER_REGION
+            # since there is no context name available.
+            return region, zone
+
         all_contexts = kubernetes_utils.get_all_kube_config_context_names()
-        if all_contexts is None:
-            all_contexts = []
+        if all_contexts == [None]:
+            # If [None] context is returned, use the singleton region since we
+            # are running in a pod with in-cluster auth.
+            all_contexts = [kubernetes_utils.IN_CLUSTER_REGION]
         if region not in all_contexts:
             raise ValueError(
                 f'Context {region} not found in kubeconfig. Kubernetes only '

sky/provision/kubernetes/config.py

@@ -247,7 +247,8 @@ def _get_resource(container_resources: Dict[str, Any], resource_name: str,
 
 
 def _configure_autoscaler_service_account(
-        namespace: str, context: str, provider_config: Dict[str, Any]) -> None:
+        namespace: str, context: Optional[str],
+        provider_config: Dict[str, Any]) -> None:
     account_field = 'autoscaler_service_account'
     if account_field not in provider_config:
         logger.info('_configure_autoscaler_service_account: '
@@ -281,7 +282,7 @@ def _configure_autoscaler_service_account(
                 f'{created_msg(account_field, name)}')
 
 
-def _configure_autoscaler_role(namespace: str, context: str,
+def _configure_autoscaler_role(namespace: str, context: Optional[str],
                                provider_config: Dict[str, Any],
                                role_field: str) -> None:
     """ Reads the role from the provider config, creates if it does not exist.
@@ -330,7 +331,7 @@ def _configure_autoscaler_role(namespace: str, context: str,
 
 def _configure_autoscaler_role_binding(
         namespace: str,
-        context: str,
+        context: Optional[str],
         provider_config: Dict[str, Any],
         binding_field: str,
         override_name: Optional[str] = None,
@@ -620,7 +621,7 @@ def _configure_fuse_mounting(provider_config: Dict[str, Any]) -> None:
                 f'in namespace {fuse_device_manager_namespace!r}')
 
 
-def _configure_services(namespace: str, context: str,
+def _configure_services(namespace: str, context: Optional[str],
                         provider_config: Dict[str, Any]) -> None:
     service_field = 'services'
     if service_field not in provider_config:

sky/provision/kubernetes/instance.py

@@ -302,7 +302,8 @@ def _wait_for_pods_to_run(namespace, context, new_nodes):
         time.sleep(1)
 
 
-def _set_env_vars_in_pods(namespace: str, context: str, new_pods: List):
+def _set_env_vars_in_pods(namespace: str, context: Optional[str],
+                          new_pods: List):
     """Setting environment variables in pods.
 
     Once all containers are ready, we can exec into them and set env vars.
@@ -330,7 +331,7 @@ def _set_env_vars_in_pods(namespace: str, context: str, new_pods: List):
                                      new_pod.metadata.name, rc, stdout)
 
 
-def _check_user_privilege(namespace: str, context: str,
+def _check_user_privilege(namespace: str, context: Optional[str],
                           new_nodes: List) -> None:
     # Checks if the default user has sufficient privilege to set up
     # the kubernetes instance pod.
@@ -366,7 +367,8 @@ def _check_user_privilege(namespace: str, context: str,
                 'from the image.')
 
 
-def _setup_ssh_in_pods(namespace: str, context: str, new_nodes: List) -> None:
+def _setup_ssh_in_pods(namespace: str, context: Optional[str],
+                       new_nodes: List) -> None:
     # Setting up ssh for the pod instance. This is already setup for
     # the jump pod so it does not need to be run for it.
     set_k8s_ssh_cmd = (
@@ -410,7 +412,7 @@ def _setup_ssh_in_pods(namespace: str, context: str, new_nodes: List) -> None:
         logger.info(f'{"-"*20}End: Set up SSH in pod {pod_name!r} {"-"*20}')
 
 
-def _label_pod(namespace: str, context: str, pod_name: str,
+def _label_pod(namespace: str, context: Optional[str], pod_name: str,
                label: Dict[str, str]) -> None:
     """Label a pod."""
     kubernetes.core_api(context).patch_namespaced_pod(
@@ -647,7 +649,8 @@ def stop_instances(
     raise NotImplementedError()
 
 
-def _terminate_node(namespace: str, context: str, pod_name: str) -> None:
+def _terminate_node(namespace: str, context: Optional[str],
+                    pod_name: str) -> None:
     """Terminate a pod."""
     logger.debug('terminate_instances: calling delete_namespaced_pod')
     try:

sky/provision/kubernetes/network_utils.py

@@ -132,7 +132,7 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int,
 
 
 def create_or_replace_namespaced_ingress(
-        namespace: str, context: str, ingress_name: str,
+        namespace: str, context: Optional[str], ingress_name: str,
         ingress_spec: Dict[str, Union[str, int]]) -> None:
     """Creates an ingress resource for the specified service."""
     networking_api = kubernetes.networking_api(context)
@@ -156,7 +156,7 @@ def create_or_replace_namespaced_ingress(
         _request_timeout=kubernetes.API_TIMEOUT)
 
 
-def delete_namespaced_ingress(namespace: str, context: str,
+def delete_namespaced_ingress(namespace: str, context: Optional[str],
                               ingress_name: str) -> None:
     """Deletes an ingress resource."""
     networking_api = kubernetes.networking_api(context)
@@ -171,7 +171,7 @@ def delete_namespaced_ingress(namespace: str, context: str,
 
 
 def create_or_replace_namespaced_service(
-        namespace: str, context: str, service_name: str,
+        namespace: str, context: Optional[str], service_name: str,
         service_spec: Dict[str, Union[str, int]]) -> None:
     """Creates a service resource for the specified service."""
     core_api = kubernetes.core_api(context)
@@ -208,7 +208,7 @@ def delete_namespaced_service(namespace: str, service_name: str) -> None:
         raise e
 
 
-def ingress_controller_exists(context: str,
+def ingress_controller_exists(context: Optional[str],
                               ingress_class_name: str = 'nginx') -> bool:
     """Checks if an ingress controller exists in the cluster."""
     networking_api = kubernetes.networking_api(context)
@@ -220,7 +220,7 @@ def ingress_controller_exists(context: str,
 
 
 def get_ingress_external_ip_and_ports(
-    context: str,
+    context: Optional[str],
     namespace: str = 'ingress-nginx'
 ) -> Tuple[Optional[str], Optional[Tuple[int, int]]]:
     """Returns external ip and ports for the ingress controller."""
@@ -258,7 +258,7 @@ def get_ingress_external_ip_and_ports(
     return external_ip, None
 
 
-def get_loadbalancer_ip(context: str,
+def get_loadbalancer_ip(context: Optional[str],
                         namespace: str,
                         service_name: str,
                         timeout: int = 0) -> Optional[str]:
@@ -284,7 +284,8 @@ def get_loadbalancer_ip(context: str,
     return ip
 
 
-def get_pod_ip(context: str, namespace: str, pod_name: str) -> Optional[str]:
+def get_pod_ip(context: Optional[str], namespace: str,
+               pod_name: str) -> Optional[str]:
     """Returns the IP address of the pod."""
     core_api = kubernetes.core_api(context)
     pod = core_api.read_namespaced_pod(pod_name,

sky/provision/kubernetes/utils.py

@@ -33,6 +33,7 @@ if typing.TYPE_CHECKING:
 
 # TODO(romilb): Move constants to constants.py
 DEFAULT_NAMESPACE = 'default'
+IN_CLUSTER_REGION = 'in-cluster'
 
 DEFAULT_SERVICE_ACCOUNT_NAME = 'skypilot-service-account'
 
@@ -310,7 +311,7 @@ AUTOSCALER_TO_LABEL_FORMATTER = {
 
 @functools.lru_cache()
 def detect_gpu_label_formatter(
-    context: str
+    context: Optional[str]
 ) -> Tuple[Optional[GPULabelFormatter], Dict[str, List[Tuple[str, str]]]]:
     """Detects the GPU label formatter for the Kubernetes cluster
 
@@ -342,7 +343,7 @@ def detect_gpu_label_formatter(
 
 
 @functools.lru_cache(maxsize=10)
-def detect_gpu_resource(context: str) -> Tuple[bool, Set[str]]:
+def detect_gpu_resource(context: Optional[str]) -> Tuple[bool, Set[str]]:
     """Checks if the Kubernetes cluster has nvidia.com/gpu resource.
 
     If nvidia.com/gpu resource is missing, that typically means that the
@@ -402,7 +403,7 @@ def get_all_pods_in_kubernetes_cluster(
     return pods
 
 
-def check_instance_fits(context: str,
+def check_instance_fits(context: Optional[str],
                         instance: str) -> Tuple[bool, Optional[str]]:
     """Checks if the instance fits on the Kubernetes cluster.
 
@@ -488,7 +489,7 @@ def check_instance_fits(context: str,
         return fits, reason
 
 
-def get_gpu_label_key_value(context: str,
+def get_gpu_label_key_value(context: Optional[str],
                             acc_type: str,
                             check_mode=False) -> Tuple[str, str]:
     """Returns the label key and value for the given GPU type.
@@ -651,11 +652,14 @@ def get_external_ip(network_mode: Optional[
     return parsed_url.hostname
 
 
-def check_credentials(context: str, timeout: int = kubernetes.API_TIMEOUT) -> \
+def check_credentials(context: Optional[str],
+                      timeout: int = kubernetes.API_TIMEOUT) -> \
         Tuple[bool, Optional[str]]:
     """Check if the credentials in kubeconfig file are valid
 
     Args:
+        context (Optional[str]): The Kubernetes context to use. If none, uses
+            in-cluster auth to check credentials, if available.
         timeout (int): Timeout in seconds for the test API call
 
     Returns:
@@ -817,22 +821,42 @@ def get_current_kube_config_context_name() -> Optional[str]:
         return None
 
 
-def get_all_kube_config_context_names() -> Optional[List[str]]:
+def is_incluster_config_available() -> bool:
+    """Check if in-cluster auth is available.
+
+    Note: We cannot use load_incluster_config() to check if in-cluster config
+    is available because it will load the in-cluster config (if available)
+    and modify the current global kubernetes config. We simply check if the
+    service account token file exists to determine if in-cluster config may
+    be available.
+    """
+    return os.path.exists('/var/run/secrets/kubernetes.io/serviceaccount/token')
+
+
+def get_all_kube_config_context_names() -> List[Optional[str]]:
     """Get all kubernetes context names from the kubeconfig file.
 
+    If running in-cluster, returns [None] to indicate in-cluster config.
+
     We should not cache the result of this function as the admin policy may
     update the contexts.
 
     Returns:
-        List[str] | None: The list of kubernetes context names if it exists,
-            None otherwise
+        List[Optional[str]]: The list of kubernetes context names if
+            available, an empty list otherwise. If running in-cluster,
+            returns [None] to indicate in-cluster config.
     """
     k8s = kubernetes.kubernetes
     try:
         all_contexts, _ = k8s.config.list_kube_config_contexts()
+        # all_contexts will always have at least one context. If kubeconfig
+        # does not have any contexts defined, it will raise ConfigException.
         return [context['name'] for context in all_contexts]
     except k8s.config.config_exception.ConfigException:
-        return None
+        # If running in cluster, return [None] to indicate in-cluster config
+        if is_incluster_config_available():
+            return [None]
+        return []
 
 
 @functools.lru_cache()
@@ -1046,7 +1070,7 @@ def get_ssh_proxy_command(
     k8s_ssh_target: str,
     network_mode: kubernetes_enums.KubernetesNetworkingMode,
     private_key_path: str,
-    context: str,
+    context: Optional[str],
     namespace: str,
 ) -> str:
     """Generates the SSH proxy command to connect to the pod.
@@ -1144,7 +1168,8 @@ def create_proxy_command_script() -> str:
     return port_fwd_proxy_cmd_path
 
 
-def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str, context: str,
+def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str,
+                       context: Optional[str],
                        service_type: kubernetes_enums.KubernetesServiceType):
     """Sets up Kubernetes service resource to access for SSH jump pod.
 
@@ -1216,7 +1241,8 @@ def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str, context: str,
 
 
 def setup_ssh_jump_pod(ssh_jump_name: str, ssh_jump_image: str,
-                       ssh_key_secret: str, namespace: str, context: str):
+                       ssh_key_secret: str, namespace: str,
+                       context: Optional[str]):
     """Sets up Kubernetes RBAC and pod for SSH jump host.
 
     Our Kubernetes implementation uses a SSH jump pod to reach SkyPilot clusters
@@ -1296,7 +1322,8 @@ def setup_ssh_jump_pod(ssh_jump_name: str, ssh_jump_image: str,
         logger.info(f'Created SSH Jump Host {ssh_jump_name}.')
 
 
-def clean_zombie_ssh_jump_pod(namespace: str, context: str, node_id: str):
+def clean_zombie_ssh_jump_pod(namespace: str, context: Optional[str],
+                              node_id: str):
     """Analyzes SSH jump pod and removes if it is in a bad state
 
     Prevents the existence of a dangling SSH jump pod. This could happen
@@ -1618,7 +1645,8 @@ def check_nvidia_runtime_class(context: Optional[str] = None) -> bool:
     return nvidia_exists
 
 
-def check_secret_exists(secret_name: str, namespace: str, context: str) -> bool:
+def check_secret_exists(secret_name: str, namespace: str,
+                        context: Optional[str]) -> bool:
     """Checks if a secret exists in a namespace
 
     Args:
@@ -1836,7 +1864,7 @@ def get_namespace_from_config(provider_config: Dict[str, Any]) -> str:
 
 
 def filter_pods(namespace: str,
-                context: str,
+                context: Optional[str],
                 tag_filters: Dict[str, str],
                 status_filters: Optional[List[str]] = None) -> Dict[str, Any]:
     """Filters pods by tags and status."""
@@ -1962,6 +1990,11 @@ def set_autodown_annotations(handle: 'backends.CloudVmRayResourceHandle',
                                    context=context)
 
 
-def get_context_from_config(provider_config: Dict[str, Any]) -> str:
-    return provider_config.get('context',
-                               get_current_kube_config_context_name())
+def get_context_from_config(provider_config: Dict[str, Any]) -> Optional[str]:
+    context = provider_config.get('context',
+                                  get_current_kube_config_context_name())
+    if context == IN_CLUSTER_REGION:
+        # If the context (also used as the region) is set to IN_CLUSTER_REGION
+        # we need to use in-cluster auth.
+        context = None
+    return context

sky/utils/command_runner.py

@@ -653,7 +653,7 @@ class KubernetesCommandRunner(CommandRunner):
 
     def __init__(
         self,
-        node: Tuple[Tuple[str, str], str],
+        node: Tuple[Tuple[str, Optional[str]], str],
         **kwargs,
     ):
         """Initialize KubernetesCommandRunner.

sky/utils/command_runner.pyi

@@ -204,7 +204,7 @@ class KubernetesCommandRunner(CommandRunner):
 
     def __init__(
         self,
-        node: Tuple[Tuple[str, str], str],
+        node: Tuple[Tuple[str, Optional[str]], str],
     ) -> None:
         ...
 

sky/utils/controller_utils.py

@@ -363,6 +363,14 @@ def shared_controller_vars_to_fill(
         # again on the controller. This is required since admin_policy is not
         # installed on the controller.
         local_user_config.pop('admin_policy', None)
+        # Remove allowed_contexts from local_user_config since the controller
+        # may be running in a Kubernetes cluster with in-cluster auth and may
+        # not have kubeconfig available to it. This is the typical case since
+        # remote_identity default for Kubernetes is SERVICE_ACCOUNT.
+        # TODO(romilb): We should check the cloud the controller is running on
+        # before popping allowed_contexts. If it is not on Kubernetes,
+        # we may be able to use allowed_contexts.
+        local_user_config.pop('allowed_contexts', None)
         with tempfile.NamedTemporaryFile(
                 delete=False,
                 suffix=_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX) as temp_file:

sky/utils/kubernetes/deploy_remote_cluster.sh

@@ -0,0 +1,243 @@
+#!/bin/bash
+# Refer to https://skypilot.readthedocs.io/en/latest/reservations/existing-machines.html for details on how to use this script.
+set -e
+
+# Colors for nicer UX
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No color
+
+# Variables
+IPS_FILE=$1
+USER=$2
+SSH_KEY=$3
+K3S_TOKEN=mytoken  # Any string can be used as the token
+CLEANUP=false
+INSTALL_GPU=false
+
+if [[ "$4" == "--cleanup" ]]; then
+    CLEANUP=true
+fi
+
+# Basic argument checks
+if [ -z "$IPS_FILE" ] || [ -z "$USER" ] || [ -z "$SSH_KEY" ]; then
+    >&2 echo -e "${RED}Error: Missing required arguments.${NC}"
+    >&2 echo "Usage: ./deploy_remote_cluster.sh ips.txt username path/to/ssh/key [--cleanup]"
+    exit 1
+fi
+
+# Check if SSH key exists
+if [ ! -f "$SSH_KEY" ]; then
+    >&2 echo -e "${RED}Error: SSH key not found: $SSH_KEY${NC}"
+    exit 1
+fi
+
+# Check if IPs file exists
+if [ ! -f "$IPS_FILE" ]; then
+    >&2 echo -e "${RED}Error: IPs file not found: $IPS_FILE${NC}"
+    exit 1
+fi
+
+# Get head node and worker nodes from the IPs file
+HEAD_NODE=$(head -n 1 "$IPS_FILE")
+WORKER_NODES=$(tail -n +2 "$IPS_FILE")
+
+# Check if the IPs file is empty or not formatted correctly
+if [ -z "$HEAD_NODE" ]; then
+    >&2 echo -e "${RED}Error: IPs file is empty or not formatted correctly.${NC}"
+    exit 1
+fi
+
+# Function to show a progress message
+progress_message() {
+    echo -e "${YELLOW}➜ $1${NC}"
+}
+
+# Step to display success
+success_message() {
+    echo -e "${GREEN}✔ $1${NC}"
+}
+
+# Function to run a command on a remote machine via SSH
+run_remote() {
+    local NODE_IP=$1
+    local CMD=$2
+    # echo -e "${YELLOW}Running command on $NODE_IP...${NC}"
+    ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" "$USER@$NODE_IP" "$CMD"
+}
+
+# Function to uninstall k3s and clean up the state on a remote machine
+cleanup_server_node() {
+    local NODE_IP=$1
+    echo -e "${YELLOW}Cleaning up head node $NODE_IP...${NC}"
+    run_remote "$NODE_IP" "
+        echo 'Uninstalling k3s...' &&
+        /usr/local/bin/k3s-uninstall.sh || true &&
+        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+    "
+    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
+}
+
+# Function to uninstall k3s and clean up the state on a remote machine
+cleanup_agent_node() {
+    local NODE_IP=$1
+    echo -e "${YELLOW}Cleaning up node $NODE_IP...${NC}"
+    run_remote "$NODE_IP" "
+        echo 'Uninstalling k3s...' &&
+        /usr/local/bin/k3s-agent-uninstall.sh || true &&
+        sudo rm -rf /etc/rancher /var/lib/rancher /var/lib/kubelet /etc/kubernetes ~/.kube
+    "
+    echo -e "${GREEN}Node $NODE_IP cleaned up successfully.${NC}"
+}
+
+check_gpu() {
+    local NODE_IP=$1
+    run_remote "$NODE_IP" "
+        if command -v nvidia-smi &> /dev/null; then
+            nvidia-smi --list-gpus | grep 'GPU 0'
+        fi
+    "
+}
+
+# Pre-flight checks
+run_remote "$HEAD_NODE" "echo 'SSH connection successful'"
+# TODO: Add more pre-flight checks here, including checking if port 6443 is accessible
+
+# If --cleanup flag is set, uninstall k3s and exit
+if [ "$CLEANUP" == "true" ]; then
+    echo -e "${YELLOW}Starting cleanup...${NC}"
+
+    # Clean up head node
+    cleanup_server_node "$HEAD_NODE"
+
+    # Clean up worker nodes
+    for NODE in $WORKER_NODES; do
+        cleanup_agent_node "$NODE"
+    done
+
+    echo -e "${GREEN}Cleanup completed successfully.${NC}"
+    exit 0
+fi
+
+# Step 1: Install k3s on the head node
+progress_message "Deploying Kubernetes on head node ($HEAD_NODE)..."
+run_remote "$HEAD_NODE" "
+    curl -sfL https://get.k3s.io | K3S_TOKEN=$K3S_TOKEN sh - &&
+    mkdir -p ~/.kube &&
+    sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config &&
+    sudo chown \$(id -u):\$(id -g) ~/.kube/config &&
+    for i in {1..3}; do
+        if kubectl wait --for=condition=ready node --all --timeout=2m --kubeconfig ~/.kube/config; then
+            break
+        else
+            echo 'Waiting for nodes to be ready...'
+            sleep 5
+        fi
+    done
+    if [ $i -eq 3 ]; then
+        echo 'Failed to wait for nodes to be ready after 3 attempts'
+        exit 1
+    fi"
+success_message "K3s deployed on head node."
+
+# Check if head node has a GPU
+if check_gpu "$HEAD_NODE"; then
+    echo -e "${YELLOW}GPU detected on head node ($HEAD_NODE).${NC}"
+    INSTALL_GPU=true
+fi
+
+# Fetch the head node's internal IP (this will be passed to worker nodes)
+MASTER_ADDR=$(run_remote "$HEAD_NODE" "hostname -I | awk '{print \$1}'")
+
+echo -e "${GREEN}Master node internal IP: $MASTER_ADDR${NC}"
+
+# Step 2: Install k3s on worker nodes and join them to the master node
+for NODE in $WORKER_NODES; do
+    progress_message "Deploying Kubernetes on worker node ($NODE)..."
+    run_remote "$NODE" "
+        curl -sfL https://get.k3s.io | K3S_URL=https://$MASTER_ADDR:6443 K3S_TOKEN=$K3S_TOKEN sh -"
+    success_message "Kubernetes deployed on worker node ($NODE)."
+
+    # Check if worker node has a GPU
+    if check_gpu "$NODE"; then
+        echo -e "${YELLOW}GPU detected on worker node ($NODE).${NC}"
+        INSTALL_GPU=true
+    fi
+done
+# Step 3: Configure local kubectl to connect to the cluster
+progress_message "Configuring local kubectl to connect to the cluster..."
+scp -o StrictHostKeyChecking=no -i "$SSH_KEY" "$USER@$HEAD_NODE":~/.kube/config ~/.kube/config
+
+# Back up the original kubeconfig file if it exists
+KUBECONFIG_FILE="$HOME/.kube/config"
+if [[ -f "$KUBECONFIG_FILE" ]]; then
+    echo "Backing up existing kubeconfig to $KUBECONFIG_FILE.bak"
+    cp "$KUBECONFIG_FILE" "$KUBECONFIG_FILE.bak"
+fi
+
+# Update kubeconfig for the local machine to use the master node's IP
+# Temporary file to hold the modified kubeconfig
+TEMP_FILE=$(mktemp)
+
+# Remove the certificate-authority-data, and replace the server with the master address
+awk '
+  BEGIN { in_cluster = 0 }
+  /^clusters:/ { in_cluster = 1 }
+  /^users:/ { in_cluster = 0 }
+  in_cluster && /^ *certificate-authority-data:/ { next }
+  in_cluster && /^ *server:/ {
+    print "    server: https://'${HEAD_NODE}:6443'"
+    print "    insecure-skip-tls-verify: true"
+    next
+  }
+  { print }
+' "$KUBECONFIG_FILE" > "$TEMP_FILE"
+
+# Replace the original kubeconfig with the modified one
+mv "$TEMP_FILE" "$KUBECONFIG_FILE"
+
+success_message "kubectl configured to connect to the cluster."
+
+echo "Cluster deployment completed. You can now run 'kubectl get nodes' to verify the setup."
+
+# Install GPU operator if a GPU was detected on any node
+if [ "$INSTALL_GPU" == "true" ]; then
+    echo -e "${YELLOW}GPU detected in the cluster. Installing Nvidia GPU Operator...${NC}"
+    run_remote "$HEAD_NODE" "
+        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 &&
+        chmod 700 get_helm.sh &&
+        ./get_helm.sh &&
+        helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update &&
+        kubectl create namespace gpu-operator --kubeconfig ~/.kube/config || true &&
+        sudo ln -s /sbin/ldconfig /sbin/ldconfig.real || true &&
+        helm install gpu-operator -n gpu-operator --create-namespace nvidia/gpu-operator \
+        --set 'toolkit.env[0].name=CONTAINERD_CONFIG' \
+        --set 'toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml' \
+        --set 'toolkit.env[1].name=CONTAINERD_SOCKET' \
+        --set 'toolkit.env[1].value=/run/k3s/containerd/containerd.sock' \
+        --set 'toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS' \
+        --set 'toolkit.env[2].value=nvidia' &&
+        echo 'Waiting for GPU operator installation...' &&
+        while ! kubectl describe nodes --kubeconfig ~/.kube/config | grep -q 'nvidia.com/gpu:'; do
+            echo 'Waiting for GPU operator...'
+            sleep 5
+        done
+        echo 'GPU operator installed successfully.'"
+    success_message "GPU Operator installed."
+else
+    echo -e "${YELLOW}No GPUs detected. Skipping GPU Operator installation.${NC}"
+fi
+
+# Configure SkyPilot
+progress_message "Configuring SkyPilot..."
+sky check kubernetes
+success_message "SkyPilot configured successfully."
+
+# Display final success message
+echo -e "${GREEN}==== 🎉 Kubernetes cluster deployment completed successfully 🎉 ====${NC}"
+echo "You can now interact with your Kubernetes cluster through SkyPilot: "
+echo "  • List available GPUs: sky show-gpus --cloud kubernetes"
+echo "  • Launch a GPU development pod: sky launch -c devbox --cloud kubernetes --gpus A100:1"
+echo "  • Connect to pod with SSH: ssh devbox"
+echo "  • Connect to pod with VSCode: code --remote ssh-remote+devbox '/'"

sky/utils/log_utils.py

@@ -1,6 +1,7 @@
 """Logging utils."""
 import enum
-from typing import List, Optional
+import types
+from typing import List, Optional, Type
 
 import colorama
 import pendulum
@@ -15,13 +16,15 @@ logger = sky_logging.init_logger(__name__)
 class LineProcessor(object):
     """A processor for log lines."""
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         pass
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         pass
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         pass
 
@@ -34,12 +37,12 @@ class RayUpLineProcessor(LineProcessor):
         RUNTIME_SETUP = 1
         PULLING_DOCKER_IMAGES = 2
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         self.state = self.ProvisionStatus.LAUNCH
         self.status_display = rich_utils.safe_status('[bold cyan]Launching')
         self.status_display.start()
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         if ('Success.' in log_line and
                 self.state == self.ProvisionStatus.LAUNCH):
             logger.info(f'{colorama.Fore.GREEN}Head node is up.'
@@ -60,7 +63,9 @@ class RayUpLineProcessor(LineProcessor):
                 '[bold cyan]Launching - Preparing SkyPilot runtime')
             self.state = self.ProvisionStatus.RUNTIME_SETUP
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         self.status_display.stop()
 
@@ -68,13 +73,13 @@ class RayUpLineProcessor(LineProcessor):
 class SkyLocalUpLineProcessor(LineProcessor):
     """A processor for `sky local up` log lines."""
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         status = rich_utils.safe_status('[bold cyan]Creating local cluster - '
                                         'initializing Kubernetes')
         self.status_display = status
         self.status_display.start()
 
-    def process_line(self, log_line):
+    def process_line(self, log_line: str) -> None:
         if 'Kind cluster created.' in log_line:
             logger.info(f'{colorama.Fore.GREEN}Kubernetes is running.'
                         f'{colorama.Style.RESET_ALL}')
@@ -124,7 +129,80 @@ class SkyLocalUpLineProcessor(LineProcessor):
                 f'{colorama.Fore.GREEN}Nginx Ingress Controller installed.'
                 f'{colorama.Style.RESET_ALL}')
 
-    def __exit__(self, except_type, except_value, traceback):
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
+        del except_type, except_value, traceback  # unused
+        self.status_display.stop()
+
+
+class SkyRemoteUpLineProcessor(LineProcessor):
+    """A processor for deploy_remote_cluster.sh log lines."""
+
+    def __enter__(self) -> None:
+        status = rich_utils.safe_status('[bold cyan]Creating remote cluster')
+        self.status_display = status
+        self.status_display.start()
+
+    def process_line(self, log_line: str) -> None:
+        # Pre-flight checks
+        if 'SSH connection successful' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}SSH connection established.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Kubernetes installation steps
+        if 'Deploying Kubernetes on head node' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'deploying Kubernetes on head node')
+        if 'K3s deployed on head node.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ K3s successfully deployed on head node.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Worker nodes
+        if 'Deploying Kubernetes on worker node' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'deploying Kubernetes on worker nodes')
+        if 'Kubernetes deployed on worker node' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ K3s successfully deployed on worker node.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Cluster configuration
+        if 'Configuring local kubectl to connect to the cluster...' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'configuring local kubectl')
+        if 'kubectl configured to connect to the cluster.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ kubectl configured for the remote cluster.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # GPU operator installation
+        if 'Installing Nvidia GPU Operator...' in log_line:
+            self.status_display.update('[bold cyan]Creating remote cluster - '
+                                       'installing Nvidia GPU Operator')
+        if 'GPU Operator installed.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        '✔ Nvidia GPU Operator installed successfully.'
+                        f'{colorama.Style.RESET_ALL}')
+
+        # Cleanup steps
+        if 'Cleaning up head node' in log_line:
+            self.status_display.update('[bold cyan]Cleaning up head node')
+        if 'Cleaning up node' in log_line:
+            self.status_display.update('[bold cyan]Cleaning up worker node')
+        if 'cleaned up successfully' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}'
+                        f'{log_line.strip()}{colorama.Style.RESET_ALL}')
+
+        # Final status
+        if 'Cluster deployment completed.' in log_line:
+            logger.info(f'{colorama.Fore.GREEN}✔ Remote k3s is running.'
+                        f'{colorama.Style.RESET_ALL}')
+
+    def __exit__(self, except_type: Optional[Type[BaseException]],
+                 except_value: Optional[BaseException],
+                 traceback: Optional[types.TracebackType]) -> None:
         del except_type, except_value, traceback  # unused
         self.status_display.stop()
 

{skypilot_nightly-1.0.0.dev20240927.dist-info → skypilot_nightly-1.0.0.dev20240929.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240927
+Version: 1.0.0.dev20240929
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240927.dist-info → skypilot_nightly-1.0.0.dev20240929.dist-info}/RECORD

@@ -1,8 +1,8 @@
-sky/__init__.py,sha256=S-XaZNVM-9OM5oGtcUfWmQC9CLW7HvN9ckCN-KCbPio,5854
+sky/__init__.py,sha256=8FioKRx3X_BHtQt6BCrINW2IHHhrCWkiT7cb8NcaRjY,5854
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
-sky/authentication.py,sha256=o8ZhUf4VSN8WtjWcUUGYg-HVskaqaoMK4ZobHC-HVYU,20697
+sky/authentication.py,sha256=TfKkVnmRIetATSEVQFp-rOOIRGqVig2i8faSQQt_ixA,20974
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
-sky/cli.py,sha256=DMnZ-vLBuFazKmiMavidSYkQvv_YvXnZALJkHaLveDM,201714
+sky/cli.py,sha256=9h4yO8p962960qUjvQ-xSusrtdh8TXNNQ1sfV0OqgZc,206262
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
 sky/core.py,sha256=YF_6kwj8Ja171Oycb8L25SZ7V_ylZYovFS_jpnjwGo0,34408
 sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
@@ -48,7 +48,7 @@ sky/clouds/cudo.py,sha256=H4VyMo5wWGAv2MXZ3xsbWjlZA_cZYnt4ecNlTOOao8Y,13147
 sky/clouds/fluidstack.py,sha256=iOmoOx52yTrHKMzwBDaxFJCfNo79M61d5tj-Np24Lyc,12436
 sky/clouds/gcp.py,sha256=CrSsaSXd83tM78foKH9viBfW1cQsjve3aUQbshsqvDg,54033
 sky/clouds/ibm.py,sha256=M8QdjeSFlwssfoY2aOodxG4q5R3eT9K-4lTPDHYvEYI,21476
-sky/clouds/kubernetes.py,sha256=DyGkJusl5YMy_sIeogSKAcJ8XaUgxUx7Gc90dRi2bZU,27251
+sky/clouds/kubernetes.py,sha256=aWoXWR-S4puZHzuUHroLKxLdTpkqU7j75dQlXECnsmE,28679
 sky/clouds/lambda_cloud.py,sha256=2Al3qCSl-I4iTi7pPPNXcbaLyVfCUgTl__vYBunLB6k,12439
 sky/clouds/oci.py,sha256=ozVEa-9IkfI-RxyXDs_aLG5G0toLBRdtwUtaU-y7bH4,26382
 sky/clouds/paperspace.py,sha256=lmUZPYAblaqiBmGQwCunccMiTF_dVA1o3vqY9Q_Nc28,10921
@@ -137,11 +137,11 @@ sky/provision/gcp/instance.py,sha256=l2-1nHj4pUoHqOu8HMN1hT1bwd4Q96X8MXgOPsNJUN8
 sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHaSvY7c8,70943
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
-sky/provision/kubernetes/config.py,sha256=gC1FeW-cyeebphY6sq2BGVF8QKZujUKyH7qe9TAAoPM,29024
-sky/provision/kubernetes/instance.py,sha256=YdcZ2vhxJPXzT1D8FuCIUyjdkK6VjsG4_qm3dDbygGw,38204
+sky/provision/kubernetes/config.py,sha256=WEKcFXXhe89bLGAvoMiBvTDxdxkpTIA6ezrj2vmzldc,29072
+sky/provision/kubernetes/instance.py,sha256=MdgyGcMUbhsSRdaTRV3IgHmiAj5goCDVhzDZ2PDVs_Y,38323
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
-sky/provision/kubernetes/network_utils.py,sha256=AZ8dkVyRHxdbJ8Lm_zXYc2y9a3O9PJWLL1PH8IjwAW8,11314
-sky/provision/kubernetes/utils.py,sha256=JO54OXmalzjXkiE6VgirTJoJ6s-3uBH1zt0rBqPz_Yk,82014
+sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMshsvsYmnMUcq8k,11399
+sky/provision/kubernetes/utils.py,sha256=iULhot4naFOsyzp53x4Q4qpsHXvz5-DMOIFFTR8ap9s,83609
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/paperspace/__init__.py,sha256=1nbUPWio7UA5gCQkO_rfEDfgXT17u5OtuByxQx4Ez6g,598
@@ -244,15 +244,15 @@ sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
 sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
-sky/utils/command_runner.py,sha256=4A7IbhyAiHHeYA85MulyRqldkQyDmc4BhRhjbfVlwE4,33850
-sky/utils/command_runner.pyi,sha256=1khh14BhdOpMxvk9Ydnd3OFdas5Nha6dSOzy5xLBUU4,7710
+sky/utils/command_runner.py,sha256=NpBe7VHlzxGxuWJeDbRWwy2p64qefqz8c6Bar7KaRnc,33860
+sky/utils/command_runner.pyi,sha256=G6DHTQ9DhjYmGf_hDoyeWdWuktXqkQyJ7U-wAJTcLiw,7720
 sky/utils/common_utils.py,sha256=O6PlZTCNhbuXOzjuV2DKw43niWE_qPfYZNGhnMtZzQg,24028
-sky/utils/controller_utils.py,sha256=VtdjKH9u1kWwUOMzPUxuLpT-XXQ2gCLCLOldB-vdh_8,37483
+sky/utils/controller_utils.py,sha256=32pVORm2cd42tg0srxGvmYV0kYTl67IFsw2EdXbdoR8,38042
 sky/utils/dag_utils.py,sha256=gjGZiJj4_GYsraXX67e6ElvbmOByJcyjSfvVgYZiXvs,5588
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
 sky/utils/env_options.py,sha256=1VXyd3bhiUgGfCpmmTqM9PagRo1ILBH4-pzIxmIeE6E,861
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
-sky/utils/log_utils.py,sha256=W7FYK7xzvbq4V-8R-ihLtz939ryvtABug6O-4DFrjho,8139
+sky/utils/log_utils.py,sha256=yVu3etgKhiVYX8UG-JFPWZujxWBT4kwxZ5oAPIdjtGs,12054
 sky/utils/resources_utils.py,sha256=snByBxgx3Hnjfch2uysdAA3D-OAwrnuzTDHug36s5H4,6515
 sky/utils/rich_utils.py,sha256=5ZVhzlFx-nhqMXwv00eO9xC4rz7ibDlfD2lmGhZrJEY,1581
 sky/utils/schemas.py,sha256=QT0Fxri2o0SiWkky1DlZhA1dzQRQoB5OdVaej0wJvhc,28787
@@ -265,6 +265,7 @@ sky/utils/cli_utils/status_utils.py,sha256=9odkfXiXLMD14XJsqve6sGvHpe7ThHXpC6ic9
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=rv5Lz6AR00yBJMRyScfMSQiGKptMhtHWRsvyG20-u9c,7764
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
+sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=vGj0mD0tejHDRy8ulwKOvOF2mfLyT5J8fp7GVqEe_EY,8478
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
 sky/utils/kubernetes/generate_kubeconfig.sh,sha256=AcYhuuG5jXWGHUmyRuH-oKy5qcn92gXhu6bXOt6eD6g,9274
 sky/utils/kubernetes/gpu_labeler.py,sha256=MEUv0U4ACDcNwtFVltlv017XJMjxx1Bndf6fL0i6eqg,6960
@@ -272,9 +273,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=Ma-N9a271fTfdgP5-8XIQL7KPf8IPUo-uY004PCdUFo,747
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240927.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240927.dist-info/METADATA,sha256=GXz5qTbQuxyKpPHIVPT5vsvlpo3bRyataK8Vtj6rovw,18948
-skypilot_nightly-1.0.0.dev20240927.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-skypilot_nightly-1.0.0.dev20240927.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240927.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240927.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240929.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240929.dist-info/METADATA,sha256=WfIgFJj_CNWtVZ39aN4Bx7l35R3rE4vdr6tJR9w9m-A,18948
+skypilot_nightly-1.0.0.dev20240929.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+skypilot_nightly-1.0.0.dev20240929.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240929.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240929.dist-info/RECORD,,