skypilot-nightly 1.0.0.dev20240928__py3-none-any.whl → 1.0.0.dev20240929__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'dacf27348ae1446c3c93d0ee2fc57702c5366eac'
+_SKYPILOT_COMMIT_SHA = 'e6a3b830fb2a12871815773af6171d42e0416e89'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240928'
+__version__ = '1.0.0.dev20240929'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/authentication.py

@@ -380,6 +380,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     secret_field_name = clouds.Kubernetes().ssh_key_secret_field_name
     context = config['provider'].get(
         'context', kubernetes_utils.get_current_kube_config_context_name())
+    if context == kubernetes_utils.IN_CLUSTER_REGION:
+        # If the context is set to IN_CLUSTER_REGION, we are running in a pod
+        # with in-cluster configuration. We need to set the context to None
+        # to use the mounted service account.
+        context = None
     namespace = config['provider'].get(
         'namespace',
         kubernetes_utils.get_kube_config_context_namespace(context))

sky/clouds/kubernetes.py

@@ -129,11 +129,24 @@ class Kubernetes(clouds.Cloud):
                 'Ignoring these contexts.')
 
     @classmethod
-    def _existing_allowed_contexts(cls) -> List[str]:
-        """Get existing allowed contexts."""
+    def _existing_allowed_contexts(cls) -> List[Optional[str]]:
+        """Get existing allowed contexts.
+
+        If None is returned in the list, it means that we are running in a pod
+        with in-cluster auth. In this case, we specify None context, which will
+        use the service account mounted in the pod.
+        """
         all_contexts = kubernetes_utils.get_all_kube_config_context_names()
-        if all_contexts is None:
+        if len(all_contexts) == 0:
             return []
+        if all_contexts == [None]:
+            # If only one context is found and it is None, we are running in a
+            # pod with in-cluster auth. In this case, we allow it to be used
+            # without checking against allowed_contexts.
+            # TODO(romilb): We may want check in-cluster auth against
+            #  allowed_contexts in the future by adding a special context name
+            #  for in-cluster auth.
+            return [None]
         all_contexts = set(all_contexts)
 
         allowed_contexts = skypilot_config.get_nested(
@@ -164,7 +177,15 @@ class Kubernetes(clouds.Cloud):
         del accelerators, zone, use_spot  # unused
         existing_contexts = cls._existing_allowed_contexts()
 
-        regions = [clouds.Region(context) for context in existing_contexts]
+        regions = []
+        for context in existing_contexts:
+            if context is None:
+                # If running in-cluster, we allow the region to be set to the
+                # singleton region since there is no context name available.
+                regions.append(clouds.Region(
+                    kubernetes_utils.IN_CLUSTER_REGION))
+            else:
+                regions.append(clouds.Region(context))
 
         if region is not None:
             regions = [r for r in regions if r.name == region]
@@ -541,13 +562,20 @@ class Kubernetes(clouds.Cloud):
     def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
         if region == self._LEGACY_SINGLETON_REGION:
             # For backward compatibility, we allow the region to be set to the
-            # legacy singletonton region.
+            # legacy singleton region.
             # TODO: Remove this after 0.9.0.
             return region, zone
 
+        if region == kubernetes_utils.IN_CLUSTER_REGION:
+            # If running incluster, we set region to IN_CLUSTER_REGION
+            # since there is no context name available.
+            return region, zone
+
         all_contexts = kubernetes_utils.get_all_kube_config_context_names()
-        if all_contexts is None:
-            all_contexts = []
+        if all_contexts == [None]:
+            # If [None] context is returned, use the singleton region since we
+            # are running in a pod with in-cluster auth.
+            all_contexts = [kubernetes_utils.IN_CLUSTER_REGION]
         if region not in all_contexts:
             raise ValueError(
                 f'Context {region} not found in kubeconfig. Kubernetes only '

sky/provision/kubernetes/config.py

@@ -247,7 +247,8 @@ def _get_resource(container_resources: Dict[str, Any], resource_name: str,
 
 
 def _configure_autoscaler_service_account(
-        namespace: str, context: str, provider_config: Dict[str, Any]) -> None:
+        namespace: str, context: Optional[str],
+        provider_config: Dict[str, Any]) -> None:
     account_field = 'autoscaler_service_account'
     if account_field not in provider_config:
         logger.info('_configure_autoscaler_service_account: '
@@ -281,7 +282,7 @@ def _configure_autoscaler_service_account(
                 f'{created_msg(account_field, name)}')
 
 
-def _configure_autoscaler_role(namespace: str, context: str,
+def _configure_autoscaler_role(namespace: str, context: Optional[str],
                                provider_config: Dict[str, Any],
                                role_field: str) -> None:
     """ Reads the role from the provider config, creates if it does not exist.
@@ -330,7 +331,7 @@ def _configure_autoscaler_role(namespace: str, context: str,
 
 def _configure_autoscaler_role_binding(
         namespace: str,
-        context: str,
+        context: Optional[str],
         provider_config: Dict[str, Any],
         binding_field: str,
         override_name: Optional[str] = None,
@@ -620,7 +621,7 @@ def _configure_fuse_mounting(provider_config: Dict[str, Any]) -> None:
                 f'in namespace {fuse_device_manager_namespace!r}')
 
 
-def _configure_services(namespace: str, context: str,
+def _configure_services(namespace: str, context: Optional[str],
                         provider_config: Dict[str, Any]) -> None:
     service_field = 'services'
     if service_field not in provider_config:

sky/provision/kubernetes/instance.py

@@ -302,7 +302,8 @@ def _wait_for_pods_to_run(namespace, context, new_nodes):
         time.sleep(1)
 
 
-def _set_env_vars_in_pods(namespace: str, context: str, new_pods: List):
+def _set_env_vars_in_pods(namespace: str, context: Optional[str],
+                          new_pods: List):
     """Setting environment variables in pods.
 
     Once all containers are ready, we can exec into them and set env vars.
@@ -330,7 +331,7 @@ def _set_env_vars_in_pods(namespace: str, context: str, new_pods: List):
                                      new_pod.metadata.name, rc, stdout)
 
 
-def _check_user_privilege(namespace: str, context: str,
+def _check_user_privilege(namespace: str, context: Optional[str],
                           new_nodes: List) -> None:
     # Checks if the default user has sufficient privilege to set up
     # the kubernetes instance pod.
@@ -366,7 +367,8 @@ def _check_user_privilege(namespace: str, context: str,
                 'from the image.')
 
 
-def _setup_ssh_in_pods(namespace: str, context: str, new_nodes: List) -> None:
+def _setup_ssh_in_pods(namespace: str, context: Optional[str],
+                       new_nodes: List) -> None:
     # Setting up ssh for the pod instance. This is already setup for
     # the jump pod so it does not need to be run for it.
     set_k8s_ssh_cmd = (
@@ -410,7 +412,7 @@ def _setup_ssh_in_pods(namespace: str, context: str, new_nodes: List) -> None:
         logger.info(f'{"-"*20}End: Set up SSH in pod {pod_name!r} {"-"*20}')
 
 
-def _label_pod(namespace: str, context: str, pod_name: str,
+def _label_pod(namespace: str, context: Optional[str], pod_name: str,
                label: Dict[str, str]) -> None:
     """Label a pod."""
     kubernetes.core_api(context).patch_namespaced_pod(
@@ -647,7 +649,8 @@ def stop_instances(
     raise NotImplementedError()
 
 
-def _terminate_node(namespace: str, context: str, pod_name: str) -> None:
+def _terminate_node(namespace: str, context: Optional[str],
+                    pod_name: str) -> None:
     """Terminate a pod."""
     logger.debug('terminate_instances: calling delete_namespaced_pod')
     try:

sky/provision/kubernetes/network_utils.py

@@ -132,7 +132,7 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int,
 
 
 def create_or_replace_namespaced_ingress(
-        namespace: str, context: str, ingress_name: str,
+        namespace: str, context: Optional[str], ingress_name: str,
         ingress_spec: Dict[str, Union[str, int]]) -> None:
     """Creates an ingress resource for the specified service."""
     networking_api = kubernetes.networking_api(context)
@@ -156,7 +156,7 @@ def create_or_replace_namespaced_ingress(
         _request_timeout=kubernetes.API_TIMEOUT)
 
 
-def delete_namespaced_ingress(namespace: str, context: str,
+def delete_namespaced_ingress(namespace: str, context: Optional[str],
                               ingress_name: str) -> None:
     """Deletes an ingress resource."""
     networking_api = kubernetes.networking_api(context)
@@ -171,7 +171,7 @@ def delete_namespaced_ingress(namespace: str, context: str,
 
 
 def create_or_replace_namespaced_service(
-        namespace: str, context: str, service_name: str,
+        namespace: str, context: Optional[str], service_name: str,
         service_spec: Dict[str, Union[str, int]]) -> None:
     """Creates a service resource for the specified service."""
     core_api = kubernetes.core_api(context)
@@ -208,7 +208,7 @@ def delete_namespaced_service(namespace: str, service_name: str) -> None:
         raise e
 
 
-def ingress_controller_exists(context: str,
+def ingress_controller_exists(context: Optional[str],
                               ingress_class_name: str = 'nginx') -> bool:
     """Checks if an ingress controller exists in the cluster."""
     networking_api = kubernetes.networking_api(context)
@@ -220,7 +220,7 @@ def ingress_controller_exists(context: str,
 
 
 def get_ingress_external_ip_and_ports(
-    context: str,
+    context: Optional[str],
     namespace: str = 'ingress-nginx'
 ) -> Tuple[Optional[str], Optional[Tuple[int, int]]]:
     """Returns external ip and ports for the ingress controller."""
@@ -258,7 +258,7 @@ def get_ingress_external_ip_and_ports(
     return external_ip, None
 
 
-def get_loadbalancer_ip(context: str,
+def get_loadbalancer_ip(context: Optional[str],
                         namespace: str,
                         service_name: str,
                         timeout: int = 0) -> Optional[str]:
@@ -284,7 +284,8 @@ def get_loadbalancer_ip(context: str,
     return ip
 
 
-def get_pod_ip(context: str, namespace: str, pod_name: str) -> Optional[str]:
+def get_pod_ip(context: Optional[str], namespace: str,
+               pod_name: str) -> Optional[str]:
     """Returns the IP address of the pod."""
     core_api = kubernetes.core_api(context)
     pod = core_api.read_namespaced_pod(pod_name,

sky/provision/kubernetes/utils.py

@@ -33,6 +33,7 @@ if typing.TYPE_CHECKING:
 
 # TODO(romilb): Move constants to constants.py
 DEFAULT_NAMESPACE = 'default'
+IN_CLUSTER_REGION = 'in-cluster'
 
 DEFAULT_SERVICE_ACCOUNT_NAME = 'skypilot-service-account'
 
@@ -310,7 +311,7 @@ AUTOSCALER_TO_LABEL_FORMATTER = {
 
 @functools.lru_cache()
 def detect_gpu_label_formatter(
-    context: str
+    context: Optional[str]
 ) -> Tuple[Optional[GPULabelFormatter], Dict[str, List[Tuple[str, str]]]]:
     """Detects the GPU label formatter for the Kubernetes cluster
 
@@ -342,7 +343,7 @@ def detect_gpu_label_formatter(
 
 
 @functools.lru_cache(maxsize=10)
-def detect_gpu_resource(context: str) -> Tuple[bool, Set[str]]:
+def detect_gpu_resource(context: Optional[str]) -> Tuple[bool, Set[str]]:
     """Checks if the Kubernetes cluster has nvidia.com/gpu resource.
 
     If nvidia.com/gpu resource is missing, that typically means that the
@@ -402,7 +403,7 @@ def get_all_pods_in_kubernetes_cluster(
     return pods
 
 
-def check_instance_fits(context: str,
+def check_instance_fits(context: Optional[str],
                         instance: str) -> Tuple[bool, Optional[str]]:
     """Checks if the instance fits on the Kubernetes cluster.
 
@@ -488,7 +489,7 @@ def check_instance_fits(context: str,
         return fits, reason
 
 
-def get_gpu_label_key_value(context: str,
+def get_gpu_label_key_value(context: Optional[str],
                             acc_type: str,
                             check_mode=False) -> Tuple[str, str]:
     """Returns the label key and value for the given GPU type.
@@ -651,11 +652,14 @@ def get_external_ip(network_mode: Optional[
     return parsed_url.hostname
 
 
-def check_credentials(context: str, timeout: int = kubernetes.API_TIMEOUT) -> \
+def check_credentials(context: Optional[str],
+                      timeout: int = kubernetes.API_TIMEOUT) -> \
         Tuple[bool, Optional[str]]:
     """Check if the credentials in kubeconfig file are valid
 
     Args:
+        context (Optional[str]): The Kubernetes context to use. If none, uses
+            in-cluster auth to check credentials, if available.
         timeout (int): Timeout in seconds for the test API call
 
     Returns:
@@ -817,22 +821,42 @@ def get_current_kube_config_context_name() -> Optional[str]:
         return None
 
 
-def get_all_kube_config_context_names() -> Optional[List[str]]:
+def is_incluster_config_available() -> bool:
+    """Check if in-cluster auth is available.
+
+    Note: We cannot use load_incluster_config() to check if in-cluster config
+    is available because it will load the in-cluster config (if available)
+    and modify the current global kubernetes config. We simply check if the
+    service account token file exists to determine if in-cluster config may
+    be available.
+    """
+    return os.path.exists('/var/run/secrets/kubernetes.io/serviceaccount/token')
+
+
+def get_all_kube_config_context_names() -> List[Optional[str]]:
     """Get all kubernetes context names from the kubeconfig file.
 
+    If running in-cluster, returns [None] to indicate in-cluster config.
+
     We should not cache the result of this function as the admin policy may
     update the contexts.
 
     Returns:
-        List[str] | None: The list of kubernetes context names if it exists,
-            None otherwise
+        List[Optional[str]]: The list of kubernetes context names if
+            available, an empty list otherwise. If running in-cluster,
+            returns [None] to indicate in-cluster config.
     """
     k8s = kubernetes.kubernetes
     try:
         all_contexts, _ = k8s.config.list_kube_config_contexts()
+        # all_contexts will always have at least one context. If kubeconfig
+        # does not have any contexts defined, it will raise ConfigException.
         return [context['name'] for context in all_contexts]
     except k8s.config.config_exception.ConfigException:
-        return None
+        # If running in cluster, return [None] to indicate in-cluster config
+        if is_incluster_config_available():
+            return [None]
+        return []
 
 
 @functools.lru_cache()
@@ -1046,7 +1070,7 @@ def get_ssh_proxy_command(
     k8s_ssh_target: str,
     network_mode: kubernetes_enums.KubernetesNetworkingMode,
     private_key_path: str,
-    context: str,
+    context: Optional[str],
     namespace: str,
 ) -> str:
     """Generates the SSH proxy command to connect to the pod.
@@ -1144,7 +1168,8 @@ def create_proxy_command_script() -> str:
     return port_fwd_proxy_cmd_path
 
 
-def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str, context: str,
+def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str,
+                       context: Optional[str],
                        service_type: kubernetes_enums.KubernetesServiceType):
     """Sets up Kubernetes service resource to access for SSH jump pod.
 
@@ -1216,7 +1241,8 @@ def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str, context: str,
 
 
 def setup_ssh_jump_pod(ssh_jump_name: str, ssh_jump_image: str,
-                       ssh_key_secret: str, namespace: str, context: str):
+                       ssh_key_secret: str, namespace: str,
+                       context: Optional[str]):
     """Sets up Kubernetes RBAC and pod for SSH jump host.
 
     Our Kubernetes implementation uses a SSH jump pod to reach SkyPilot clusters
@@ -1296,7 +1322,8 @@ def setup_ssh_jump_pod(ssh_jump_name: str, ssh_jump_image: str,
         logger.info(f'Created SSH Jump Host {ssh_jump_name}.')
 
 
-def clean_zombie_ssh_jump_pod(namespace: str, context: str, node_id: str):
+def clean_zombie_ssh_jump_pod(namespace: str, context: Optional[str],
+                              node_id: str):
     """Analyzes SSH jump pod and removes if it is in a bad state
 
     Prevents the existence of a dangling SSH jump pod. This could happen
@@ -1618,7 +1645,8 @@ def check_nvidia_runtime_class(context: Optional[str] = None) -> bool:
     return nvidia_exists
 
 
-def check_secret_exists(secret_name: str, namespace: str, context: str) -> bool:
+def check_secret_exists(secret_name: str, namespace: str,
+                        context: Optional[str]) -> bool:
     """Checks if a secret exists in a namespace
 
     Args:
@@ -1836,7 +1864,7 @@ def get_namespace_from_config(provider_config: Dict[str, Any]) -> str:
 
 
 def filter_pods(namespace: str,
-                context: str,
+                context: Optional[str],
                 tag_filters: Dict[str, str],
                 status_filters: Optional[List[str]] = None) -> Dict[str, Any]:
     """Filters pods by tags and status."""
@@ -1962,6 +1990,11 @@ def set_autodown_annotations(handle: 'backends.CloudVmRayResourceHandle',
                                    context=context)
 
 
-def get_context_from_config(provider_config: Dict[str, Any]) -> str:
-    return provider_config.get('context',
-                               get_current_kube_config_context_name())
+def get_context_from_config(provider_config: Dict[str, Any]) -> Optional[str]:
+    context = provider_config.get('context',
+                                  get_current_kube_config_context_name())
+    if context == IN_CLUSTER_REGION:
+        # If the context (also used as the region) is set to IN_CLUSTER_REGION
+        # we need to use in-cluster auth.
+        context = None
+    return context

sky/utils/command_runner.py

@@ -653,7 +653,7 @@ class KubernetesCommandRunner(CommandRunner):
 
     def __init__(
         self,
-        node: Tuple[Tuple[str, str], str],
+        node: Tuple[Tuple[str, Optional[str]], str],
         **kwargs,
     ):
         """Initialize KubernetesCommandRunner.

sky/utils/command_runner.pyi

@@ -204,7 +204,7 @@ class KubernetesCommandRunner(CommandRunner):
 
     def __init__(
         self,
-        node: Tuple[Tuple[str, str], str],
+        node: Tuple[Tuple[str, Optional[str]], str],
     ) -> None:
         ...
 

sky/utils/controller_utils.py

@@ -363,6 +363,14 @@ def shared_controller_vars_to_fill(
         # again on the controller. This is required since admin_policy is not
         # installed on the controller.
         local_user_config.pop('admin_policy', None)
+        # Remove allowed_contexts from local_user_config since the controller
+        # may be running in a Kubernetes cluster with in-cluster auth and may
+        # not have kubeconfig available to it. This is the typical case since
+        # remote_identity default for Kubernetes is SERVICE_ACCOUNT.
+        # TODO(romilb): We should check the cloud the controller is running on
+        # before popping allowed_contexts. If it is not on Kubernetes,
+        # we may be able to use allowed_contexts.
+        local_user_config.pop('allowed_contexts', None)
         with tempfile.NamedTemporaryFile(
                 delete=False,
                 suffix=_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX) as temp_file:

{skypilot_nightly-1.0.0.dev20240928.dist-info → skypilot_nightly-1.0.0.dev20240929.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240928
+Version: 1.0.0.dev20240929
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240928.dist-info → skypilot_nightly-1.0.0.dev20240929.dist-info}/RECORD

@@ -1,6 +1,6 @@
-sky/__init__.py,sha256=8BEk3x0IPkFli8tjp7axkkM5mwQ1GuCABWwTMppkPcc,5854
+sky/__init__.py,sha256=8FioKRx3X_BHtQt6BCrINW2IHHhrCWkiT7cb8NcaRjY,5854
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
-sky/authentication.py,sha256=o8ZhUf4VSN8WtjWcUUGYg-HVskaqaoMK4ZobHC-HVYU,20697
+sky/authentication.py,sha256=TfKkVnmRIetATSEVQFp-rOOIRGqVig2i8faSQQt_ixA,20974
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
 sky/cli.py,sha256=9h4yO8p962960qUjvQ-xSusrtdh8TXNNQ1sfV0OqgZc,206262
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
@@ -48,7 +48,7 @@ sky/clouds/cudo.py,sha256=H4VyMo5wWGAv2MXZ3xsbWjlZA_cZYnt4ecNlTOOao8Y,13147
 sky/clouds/fluidstack.py,sha256=iOmoOx52yTrHKMzwBDaxFJCfNo79M61d5tj-Np24Lyc,12436
 sky/clouds/gcp.py,sha256=CrSsaSXd83tM78foKH9viBfW1cQsjve3aUQbshsqvDg,54033
 sky/clouds/ibm.py,sha256=M8QdjeSFlwssfoY2aOodxG4q5R3eT9K-4lTPDHYvEYI,21476
-sky/clouds/kubernetes.py,sha256=DyGkJusl5YMy_sIeogSKAcJ8XaUgxUx7Gc90dRi2bZU,27251
+sky/clouds/kubernetes.py,sha256=aWoXWR-S4puZHzuUHroLKxLdTpkqU7j75dQlXECnsmE,28679
 sky/clouds/lambda_cloud.py,sha256=2Al3qCSl-I4iTi7pPPNXcbaLyVfCUgTl__vYBunLB6k,12439
 sky/clouds/oci.py,sha256=ozVEa-9IkfI-RxyXDs_aLG5G0toLBRdtwUtaU-y7bH4,26382
 sky/clouds/paperspace.py,sha256=lmUZPYAblaqiBmGQwCunccMiTF_dVA1o3vqY9Q_Nc28,10921
@@ -137,11 +137,11 @@ sky/provision/gcp/instance.py,sha256=l2-1nHj4pUoHqOu8HMN1hT1bwd4Q96X8MXgOPsNJUN8
 sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHaSvY7c8,70943
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
-sky/provision/kubernetes/config.py,sha256=gC1FeW-cyeebphY6sq2BGVF8QKZujUKyH7qe9TAAoPM,29024
-sky/provision/kubernetes/instance.py,sha256=YdcZ2vhxJPXzT1D8FuCIUyjdkK6VjsG4_qm3dDbygGw,38204
+sky/provision/kubernetes/config.py,sha256=WEKcFXXhe89bLGAvoMiBvTDxdxkpTIA6ezrj2vmzldc,29072
+sky/provision/kubernetes/instance.py,sha256=MdgyGcMUbhsSRdaTRV3IgHmiAj5goCDVhzDZ2PDVs_Y,38323
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
-sky/provision/kubernetes/network_utils.py,sha256=AZ8dkVyRHxdbJ8Lm_zXYc2y9a3O9PJWLL1PH8IjwAW8,11314
-sky/provision/kubernetes/utils.py,sha256=JO54OXmalzjXkiE6VgirTJoJ6s-3uBH1zt0rBqPz_Yk,82014
+sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMshsvsYmnMUcq8k,11399
+sky/provision/kubernetes/utils.py,sha256=iULhot4naFOsyzp53x4Q4qpsHXvz5-DMOIFFTR8ap9s,83609
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/paperspace/__init__.py,sha256=1nbUPWio7UA5gCQkO_rfEDfgXT17u5OtuByxQx4Ez6g,598
@@ -244,10 +244,10 @@ sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
 sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
-sky/utils/command_runner.py,sha256=4A7IbhyAiHHeYA85MulyRqldkQyDmc4BhRhjbfVlwE4,33850
-sky/utils/command_runner.pyi,sha256=1khh14BhdOpMxvk9Ydnd3OFdas5Nha6dSOzy5xLBUU4,7710
+sky/utils/command_runner.py,sha256=NpBe7VHlzxGxuWJeDbRWwy2p64qefqz8c6Bar7KaRnc,33860
+sky/utils/command_runner.pyi,sha256=G6DHTQ9DhjYmGf_hDoyeWdWuktXqkQyJ7U-wAJTcLiw,7720
 sky/utils/common_utils.py,sha256=O6PlZTCNhbuXOzjuV2DKw43niWE_qPfYZNGhnMtZzQg,24028
-sky/utils/controller_utils.py,sha256=VtdjKH9u1kWwUOMzPUxuLpT-XXQ2gCLCLOldB-vdh_8,37483
+sky/utils/controller_utils.py,sha256=32pVORm2cd42tg0srxGvmYV0kYTl67IFsw2EdXbdoR8,38042
 sky/utils/dag_utils.py,sha256=gjGZiJj4_GYsraXX67e6ElvbmOByJcyjSfvVgYZiXvs,5588
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
 sky/utils/env_options.py,sha256=1VXyd3bhiUgGfCpmmTqM9PagRo1ILBH4-pzIxmIeE6E,861
@@ -273,9 +273,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=Ma-N9a271fTfdgP5-8XIQL7KPf8IPUo-uY004PCdUFo,747
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240928.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240928.dist-info/METADATA,sha256=AT9cnsY7Uj7BK0COu8mOXiCtfyCFrjtk7OBQvqx-_Nk,18948
-skypilot_nightly-1.0.0.dev20240928.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-skypilot_nightly-1.0.0.dev20240928.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240928.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240928.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240929.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240929.dist-info/METADATA,sha256=WfIgFJj_CNWtVZ39aN4Bx7l35R3rE4vdr6tJR9w9m-A,18948
+skypilot_nightly-1.0.0.dev20240929.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+skypilot_nightly-1.0.0.dev20240929.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240929.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240929.dist-info/RECORD,,