skypilot-nightly 1.0.0.dev20250413__py3-none-any.whl → 1.0.0.dev20250417__py3-none-any.whl

sky/utils/kubernetes/gpu_labeler.py

@@ -3,8 +3,9 @@ import argparse
 import hashlib
 import os
 import subprocess
-from typing import Optional, Tuple
+from typing import Dict, Optional, Tuple
 
+import colorama
 import yaml
 
 import sky
@@ -13,6 +14,10 @@ from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.utils import rich_utils
 
 
+def _format_string(str_to_format: str, colorama_format: str) -> str:
+    return f'{colorama_format}{str_to_format}{colorama.Style.RESET_ALL}'
+
+
 def cleanup(context: Optional[str] = None) -> Tuple[bool, str]:
     """Deletes all Kubernetes resources created by this script
 
@@ -45,7 +50,7 @@ def get_node_hash(node_name: str):
     return md5_hash[:32]
 
 
-def label(context: Optional[str] = None):
+def label(context: Optional[str] = None, wait_for_completion: bool = True):
     deletion_success, reason = cleanup(context=context)
     if not deletion_success:
         print(reason)
@@ -60,8 +65,10 @@ def label(context: Optional[str] = None):
               'in their capacity.')
         return
 
-    print(f'Found {len(unlabeled_gpu_nodes)} '
-          'unlabeled GPU nodes in the cluster')
+    print(
+        _format_string(
+            f'Found {len(unlabeled_gpu_nodes)} '
+            'unlabeled GPU nodes in the cluster', colorama.Fore.YELLOW))
 
     sky_dir = os.path.dirname(sky.__file__)
     manifest_dir = os.path.join(sky_dir, 'utils/kubernetes')
@@ -80,6 +87,7 @@ def label(context: Optional[str] = None):
             print('Error setting up GPU labeling: ' + output)
             return
 
+    jobs_to_node_names: Dict[str, str] = {}
     with rich_utils.client_status('Creating GPU labeler jobs'):
         batch_v1 = kubernetes.batch_api(context=context)
         # Load the job manifest
@@ -113,8 +121,11 @@ def label(context: Optional[str] = None):
             node_name = node.metadata.name
 
             # Modify the job manifest for the current node
-            job_manifest['metadata']['name'] = ('sky-gpu-labeler-'
-                                                f'{get_node_hash(node_name)}')
+            job_name = ('sky-gpu-labeler-'
+                        f'{get_node_hash(node_name)}')
+            jobs_to_node_names[job_name] = node_name
+            job_manifest['metadata']['name'] = job_name
+
             job_manifest['spec']['template']['spec']['nodeSelector'] = {
                 'kubernetes.io/hostname': node_name
             }
@@ -122,17 +133,85 @@ def label(context: Optional[str] = None):
 
             # Create the job for this node`
             batch_v1.create_namespaced_job(namespace, job_manifest)
-            print(f'Created GPU labeler job for node {node_name}')
+            print(
+                _format_string(f'Created GPU labeler job for node {node_name}',
+                               colorama.Style.DIM))
 
     context_str = f' --context {context}' if context else ''
-    print(f'GPU labeling started - this may take 10 min or more to complete.'
-          '\nTo check the status of GPU labeling jobs, run '
-          f'`kubectl get jobs -n kube-system '
-          f'-l job=sky-gpu-labeler{context_str}`'
-          '\nYou can check if nodes have been labeled by running '
-          f'`kubectl describe nodes{context_str}` '
-          'and looking for labels of the format '
-          '`skypilot.co/accelerator: <gpu_name>`. ')
+
+    if wait_for_completion:
+        # Wait for the job to complete
+        with rich_utils.client_status(
+                'Waiting for GPU labeler jobs to complete'):
+            success = wait_for_jobs_completion(jobs_to_node_names,
+                                               'kube-system',
+                                               context=context)
+        if success:
+            print(
+                _format_string('✅ GPU labeling completed successfully',
+                               colorama.Fore.GREEN))
+        else:
+            print(_format_string('❌ GPU labeling failed', colorama.Fore.RED))
+        cleanup(context=context)
+    else:
+        print(
+            f'GPU labeling started - this may take 10 min or more to complete.'
+            '\nTo check the status of GPU labeling jobs, run '
+            f'`kubectl get jobs -n kube-system '
+            f'-l job=sky-gpu-labeler{context_str}`'
+            '\nYou can check if nodes have been labeled by running '
+            f'`kubectl describe nodes{context_str}` '
+            'and looking for labels of the format '
+            '`skypilot.co/accelerator: <gpu_name>`. ')
+
+
+def wait_for_jobs_completion(jobs_to_node_names: Dict[str, str],
+                             namespace: str,
+                             context: Optional[str] = None,
+                             timeout: int = 60 * 20):
+    """Waits for a Kubernetes Job to complete or fail.
+
+    Args:
+        jobs_to_node_names: A dictionary mapping job names to node names.
+        namespace: The namespace the Job is in (default: "default").
+        timeout: Timeout in seconds (default: 1200 seconds = 20 minutes).
+
+    Returns:
+        True if the Job completed successfully, False if it failed or timed out.
+    """
+    batch_v1 = kubernetes.batch_api(context=context)
+    w = kubernetes.watch()
+    completed_jobs = []
+    for event in w.stream(func=batch_v1.list_namespaced_job,
+                          namespace=namespace,
+                          timeout_seconds=timeout):
+        job = event['object']
+        job_name = job.metadata.name
+        if job_name in jobs_to_node_names:
+            node_name = jobs_to_node_names[job_name]
+            if job.status and job.status.completion_time:
+                print(
+                    _format_string(
+                        f'GPU labeler job for node {node_name} '
+                        'completed successfully', colorama.Style.DIM))
+                completed_jobs.append(job_name)
+                num_remaining_jobs = len(jobs_to_node_names) - len(
+                    completed_jobs)
+                if num_remaining_jobs == 0:
+                    w.stop()
+                    return True
+            elif job.status and job.status.failed:
+                print(
+                    _format_string(
+                        f'GPU labeler job for node {node_name} failed',
+                        colorama.Style.DIM))
+                w.stop()
+                return False
+    print(
+        _format_string(
+            f'Timed out after waiting {timeout} seconds '
+            'for job to complete', colorama.Style.DIM))
+    return False  #Timed out
 
 
 def main():
@@ -151,6 +230,10 @@ def main():
     parser.add_argument('--context',
                         type=str,
                         help='the context to use for the Kubernetes cluster.')
+    parser.add_argument('--async',
+                        dest='async_completion',
+                        action='store_true',
+                        help='do not wait for the GPU labeling to complete.')
     args = parser.parse_args()
     context = None
     if args.context:
@@ -165,7 +248,7 @@ def main():
     if args.cleanup:
         cleanup(context=context)
     else:
-        label(context=context)
+        label(context=context, wait_for_completion=not args.async_completion)
 
 
 if __name__ == '__main__':

sky/utils/schemas.py

@@ -725,6 +725,29 @@ def get_config_schema():
         if k != '$schema'
     }
     resources_schema['properties'].pop('ports')
+    autostop_schema = {
+        'anyOf': [
+            {
+                # Use boolean to disable autostop completely, e.g.
+                #   autostop: false
+                'type': 'boolean',
+            },
+            {
+                'type': 'object',
+                'required': [],
+                'additionalProperties': False,
+                'properties': {
+                    'idle_minutes': {
+                        'type': 'integer',
+                        'minimum': 0,
+                    },
+                    'down': {
+                        'type': 'boolean',
+                    },
+                },
+            },
+        ],
+    }
     controller_resources_schema = {
         'type': 'object',
         'required': [],
@@ -736,6 +759,7 @@ def get_config_schema():
                 'additionalProperties': False,
                 'properties': {
                     'resources': resources_schema,
+                    'autostop': autostop_schema,
                 }
             },
             'bucket': {

{skypilot_nightly-1.0.0.dev20250413.dist-info → skypilot_nightly-1.0.0.dev20250417.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250413
+Version: 1.0.0.dev20250417
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -47,6 +47,7 @@ Requires-Dist: python-multipart
 Requires-Dist: aiofiles
 Requires-Dist: httpx
 Requires-Dist: setproctitle
+Requires-Dist: omegaconf<2.5,>=2.4.0dev3
 Provides-Extra: aws
 Requires-Dist: urllib3<2; extra == "aws"
 Requires-Dist: awscli>=1.27.10; extra == "aws"

{skypilot_nightly-1.0.0.dev20250413.dist-info → skypilot_nightly-1.0.0.dev20250417.dist-info}/RECORD

@@ -1,19 +1,19 @@
-sky/__init__.py,sha256=Anba71nfyiYL3S1hnAFBp3HNfA5VPtmdOiFX22nDtxk,6428
+sky/__init__.py,sha256=n83lqng2g4DhzzxC8cTHADfh7bOSy9GB_kgGRj9tRWY,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
-sky/authentication.py,sha256=A9PkFyir809O3BkYoKedJKs0OynEwy5lAlkiW92vjag,22878
-sky/check.py,sha256=iXrVOqngw7M66HlU5r-Sf65I1s6QAJ5vkPwYxD3N9tg,16104
-sky/cli.py,sha256=A7J86zgCrO5gP4-6d3qr3swKA2RoF9LkbblcALYp8cc,222589
+sky/authentication.py,sha256=ND011K_-Ud1dVZF37A9KrwYir_ihJXcHc7iDWmuBc8Q,22872
+sky/check.py,sha256=PPNQnaaZBA9_aogJpN4gnG4XWnTqkd74c-rBYDkDRDY,16101
+sky/cli.py,sha256=nvgtywp4_hxkgp749uoy8hL5_lQdXg4Uk0BpNMfG6jA,225876
 sky/cloud_stores.py,sha256=Ln5GBpel-sEs7rVx7bBrMkfLwA_bctI05Rox2uoz7Lo,26388
-sky/core.py,sha256=BrYhuSL09slF5n1y9ZWExa99ZA98eYCVB8ACkMmEruE,47747
+sky/core.py,sha256=2D1AhdZ1hyD6bBdLyF0t8UJS4ObkgYMwEteWC9_6ysc,47900
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
-sky/exceptions.py,sha256=QQkcb6LSzcZazMZCrAcFl2yewYZoEFVcZVHRc-Ouquk,16856
+sky/exceptions.py,sha256=0PwVqsSz_ei48q7RSzrF1zVQmYrZ5vRfkYUpufZw1Yc,16960
 sky/execution.py,sha256=qJhDrgy4COdojU3SxYazcTE67QPWFrZdUhUvihloyng,28727
 sky/global_user_state.py,sha256=7HADn0mY-0omf5RbpmAZ88bjySzqKcmiD1MEmkrCoNU,33754
-sky/models.py,sha256=4Hq_JNpeQyvRlde_mf1T8H5iDS362TRORYXjtk2nmT4,735
+sky/models.py,sha256=bGMSATMkSMr_Kp6SCoiJVVeebwSdZuzjw_jrJzVWAAc,1603
 sky/optimizer.py,sha256=U9tTIMU4D-X9n03QmSldBMwwodUpWcPi1PhwgWHheMM,58483
-sky/resources.py,sha256=2qc5U09MFDaJjI1dHcThcRodpMGY7HyXzQn8eC4lvbE,72402
-sky/sky_logging.py,sha256=pID2RINjH62n7SZpv70DuN8BSFYdCfTJ2ScGQpVmugg,5725
-sky/skypilot_config.py,sha256=nXHTKi2ayMO9zA4vEJgJ_VuV-p1bah70SttCzjAHCkI,14448
+sky/resources.py,sha256=2CYV3AioOu7WKa_lYQVrT0ZXAOXjwiz6hs24LGdtLac,74281
+sky/sky_logging.py,sha256=Oro18TaHPAgfDxfzwSZP6ChLNCLY-QhXtdsEcuudJ5k,5695
+sky/skypilot_config.py,sha256=KhQA7KfBrau8GbMLNdXYfg-T1p0VSfPfAUTtq1BsUjA,18998
 sky/task.py,sha256=tM-DpB7Mtxtc280gFIw9QvJfKEHgiIoew5rnNxxaj04,56737
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=iH55Cm6eXWwufAG0Dgk7LTQcADawNa3ELrBH1m6yuSY,7617
@@ -25,7 +25,7 @@ sky/adaptors/do.py,sha256=dJ0BYbkQoUWVu6_9Pxq3fOu6PngjZyyCQzgjnODXLCA,777
 sky/adaptors/docker.py,sha256=_kzpZ0fkWHqqQAVVl0llTsCE31KYz3Sjn8psTBQHVkA,468
 sky/adaptors/gcp.py,sha256=oEb9jClEtApw6PQnxdxDYxOCYsedvM3aiko1EW1FDVo,3501
 sky/adaptors/ibm.py,sha256=7YbHrWbYcZsJDgxMBNZr1yBI03mjs_C3pnCTCz-MNtQ,5068
-sky/adaptors/kubernetes.py,sha256=pbrMAO_jaHFPdIKryNALY0jvXujC3nUKnYlq0Sj6I2M,6884
+sky/adaptors/kubernetes.py,sha256=ICkKQEFv5LahZYwX8pliCke1m9xj1aHrDmnvyAZ1tYw,7080
 sky/adaptors/nebius.py,sha256=PMDuqJ3SWLUUqtUgYLpCHOBJ-xucqptGforknSZr-54,6546
 sky/adaptors/oci.py,sha256=xJt6J9xBSFIENa6FwEt1V1sZE8puAZ_vPEoGlyQACPs,2839
 sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
@@ -33,8 +33,8 @@ sky/adaptors/vast.py,sha256=tpvmHi7IkQNzbbHVkeo04kUSajoEpSzXr2XgeO_I1LU,695
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=wrVNzPkxDjHVAh46-iejBNe0nqx4DB6-Awtaqx5j2zs,7723
-sky/backends/backend_utils.py,sha256=xmv-vSvpULmNmVeWvJxkzhbmQEkctF4x5nqHFFnOEew,134925
-sky/backends/cloud_vm_ray_backend.py,sha256=mSbGOhyDaoRnaUEITV3b5xYeo9COpBWvQmdKfQ3TSlk,251973
+sky/backends/backend_utils.py,sha256=8bpPl01XV7v4PY0yYanjoE2httHtFKEjfe26v12H1fg,134921
+sky/backends/cloud_vm_ray_backend.py,sha256=M_1U_xWwdLxMr0HKfDi-AkUPO3-Rz3A0Us2JbRZjeW0,251477
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=m0s6EZ9dTbPO8STy9FeLj8DvvtI_44MwDK9QByh2jwU,17048
 sky/backends/wheel_utils.py,sha256=meypuMaygSXXjGdXfq6dhWl-OrpAybg9KVRoup4D0wU,9098
@@ -43,11 +43,11 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=7rf-iHt6RXZ_pnBBWOMwcdodHQW69x27xNyx0yVog1U,26385
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=A7J86zgCrO5gP4-6d3qr3swKA2RoF9LkbblcALYp8cc,222589
+sky/client/cli.py,sha256=nvgtywp4_hxkgp749uoy8hL5_lQdXg4Uk0BpNMfG6jA,225876
 sky/client/common.py,sha256=KlV1SDWqSdnXyBd1VMNRoUyVk1430D2PL27rD6fk5fY,14913
-sky/client/sdk.py,sha256=nkN3WCYLkwJEFuT-C-PK0yfXaZ5pCgDwpp8jVNIvZNw,70657
+sky/client/sdk.py,sha256=5vElYUysXWg5oP6OkJPAR-0cVRL3GVJT_f1uQvDKB0s,70609
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
-sky/clouds/aws.py,sha256=Op79pTZpMTIKznLbaZEnl7cs7J_xvJDnO5BOlCv4eQ4,54639
+sky/clouds/aws.py,sha256=-feXZ72UMUjuZz8dV4qMTjMMTC9pwfkQQT3KJDcIv5A,54633
 sky/clouds/azure.py,sha256=Zpo6ftWz_B30mX7N-An7JVO-8v7aU3f9cw1iH9phvwE,32251
 sky/clouds/cloud.py,sha256=22sfoOyGYjsBj88RHp-eHHs6aqi0xDb8q_9o_v6SIFM,36690
 sky/clouds/cudo.py,sha256=_UkLEtwJsfDMKlmJfML5W3rA8VArba4x8YGIdnvgZoM,13226
@@ -55,10 +55,10 @@ sky/clouds/do.py,sha256=-jVrq5qXxaOROT2R2U0XoYiMfLT0J1wGNodzjzcTUSI,11586
 sky/clouds/fluidstack.py,sha256=jIqW1MLe55MVME1PATZm8e6_FsiTnJawW7OdytPW0aM,12666
 sky/clouds/gcp.py,sha256=aDkVXRl2hjrYVp8S0BLkkbvRF4Fl996uAsv7yOkN9wQ,57108
 sky/clouds/ibm.py,sha256=XtuPN8QgrwJdb1qb_b-7KwAE2tf_N9wh9eEfi2tcg-s,22013
-sky/clouds/kubernetes.py,sha256=UCEy6_kLFyYY1XoWYJdixbBbli_Nx8DSRELh47z8c_w,37045
+sky/clouds/kubernetes.py,sha256=K2O44w3LsIb6SqJFYyDmWd7HpK4K-cGHgrZ2BacdJJE,36754
 sky/clouds/lambda_cloud.py,sha256=H32vd30OfjXriH9SibVATrJZfZcCBTiWAXHfliGeMZk,12804
 sky/clouds/nebius.py,sha256=5n_LTFZjhsCTOFIC7pTvpOdpYuRlxx2VFiS2NXa8cMk,15086
-sky/clouds/oci.py,sha256=TW2bZJnZkl_Ih_1heovvi39s7iHRLM_ZLjSjz-6fRo0,27625
+sky/clouds/oci.py,sha256=mZrtMGoU9uTv4eJcbc1bWTIp67fGioDB_YzVOb-bdVo,27622
 sky/clouds/paperspace.py,sha256=s_sZcGqmABEqzu0MTus8__NuHo5hIUvq2FqVZCRkTzE,11070
 sky/clouds/runpod.py,sha256=P486CMN-Mt3R-7jMVd3XIGjLWYy0X5B74dK_IgAp2Cg,12149
 sky/clouds/scp.py,sha256=6OucFxDIOZFA1Z_QZwGqblW4zdBljhUjhamssvchu_0,15971
@@ -108,7 +108,7 @@ sky/data/mounting_utils.py,sha256=fmJUDs7WEARYpq-cHQ5OfPRVFp_siAnqSutTLe3PROA,21
 sky/data/storage.py,sha256=AczQdfC37N686FOYP63yk9xysBy4YRzFQkz6HK6k7aQ,236136
 sky/data/storage_utils.py,sha256=u8PTKUkE7qYwr1GgAJ45pI5YUkhUaPQPRUz_CZZo_HI,13785
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
-sky/jobs/constants.py,sha256=1XiIqdR5dEgGgepLKWkZCRT3MYSsMBR-dO7N4RTsjwg,3088
+sky/jobs/constants.py,sha256=FLPAT02CmG7PLVPpYOUlbw1s_LgRdQiwdg2YBg8joFA,3305
 sky/jobs/controller.py,sha256=d5qQYHadesfFgU7-dYtt2trZwyd5IzvlVJeNh5O8OiA,31386
 sky/jobs/recovery_strategy.py,sha256=RLrqq8B1likxTknPzt3_BqO26sFVpoatxzUuGfwc18A,26170
 sky/jobs/scheduler.py,sha256=luQgrCDaDP6bI7oIbaqzxg4qMJtUIVswypnOGUklGtw,13270
@@ -120,7 +120,7 @@ sky/jobs/dashboard/dashboard.py,sha256=JKg8cCH_Y0sf3MoDTx85BghVEXWpp8ItPLshp09-_
 sky/jobs/dashboard/static/favicon.ico,sha256=uYlvgxSM7gjBmXpZ8wydvZUPAbJiiix-rc2Xe5mma9s,15086
 sky/jobs/dashboard/templates/index.html,sha256=NrlTDiEHJDt7sViwWgXUSxVCyVl_IEukE5jdvN8WhtQ,33132
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-sky/jobs/server/core.py,sha256=IHbGMEEbEdNS_ivIStv3rYVH6XodFgufrzZb3HGF9uo,25257
+sky/jobs/server/core.py,sha256=PqWvCo_5BTi6ZnSSqTSFxpYc9oRYFw__9_G9LZFpYF4,25338
 sky/jobs/server/dashboard_utils.py,sha256=2Mbx40W1pQqPEPHsSDbHeaF0j5cgyKy-_A9Owdwp_AQ,2315
 sky/jobs/server/server.py,sha256=LqBWzYVMMJ6rRcGeDsqCQco6pG_CcDCAHzmVbt57IQM,8618
 sky/provision/__init__.py,sha256=LzOo5LjkRXwSf29dUqN14YbjzQu3liXLQcmweTeZ4dE,6457
@@ -161,12 +161,12 @@ sky/provision/gcp/instance.py,sha256=47jDHLbIAI5M1MZIQTCiKGfwc9QzPOyjApkShqBRczE
 sky/provision/gcp/instance_utils.py,sha256=T0AVT8lMn128snPp3MvqmhXOihlZSC8-c1QpgYT4_FA,71377
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
-sky/provision/kubernetes/config.py,sha256=ma5_bTbbt-VLTP2MvKzfMYgD7VDXlP0KZu7vOLwF2jw,29626
+sky/provision/kubernetes/config.py,sha256=-6ddvMro9LZ-y7K-hUPo8GdUGZ-01YyBzTAdkKblGXY,29623
 sky/provision/kubernetes/constants.py,sha256=dZCUV8FOO9Gct80sdqeubKnxeW3CGl-u5mxKeIb-B0M,411
-sky/provision/kubernetes/instance.py,sha256=tzy_eEJoQD6BFENzJiz263rtnfJlzMmNRdAFO4Mn7RQ,50256
+sky/provision/kubernetes/instance.py,sha256=Cf6er0uH8knqMxJXuGXiK9EKF5oBN6hiW_OM2vDsT1w,50323
 sky/provision/kubernetes/network.py,sha256=AtcOM8wPs_-UlQJhGEQGP6Lh4HIgdx63Y0iWEhP5jyc,12673
-sky/provision/kubernetes/network_utils.py,sha256=fBAYHfc-h3PWw0ORQLK9AlTcWxcMe97Ef6GOEKMNcQ4,11657
-sky/provision/kubernetes/utils.py,sha256=edXivU2XfSo13hx3I5t2Kjcu9NzKN777Bn_LMW3YxR8,127523
+sky/provision/kubernetes/network_utils.py,sha256=6uck1aBkgtm-gGBitU3_hEUp8j14ZuG_4Xo70ReZYXs,11654
+sky/provision/kubernetes/utils.py,sha256=SBlC0u5gRwOURYEPefBUZ83uuZWJZABxvBm1otp3TnI,128678
 sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml,sha256=S87GNAbDqgTrLuxF-afPAqQ0V-i41El4s_9KBZMuaag,1331
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
 sky/provision/lambda_cloud/config.py,sha256=jq1iLzp4Up61r4JGxvtpVbJlgXnea3LHYQhCQyyl7ik,272
@@ -213,12 +213,12 @@ sky/provision/vsphere/common/vapiconnect.py,sha256=piU29ZfdYCJQMV5n-5J0EQ8G56AWO
 sky/provision/vsphere/common/vim_utils.py,sha256=y_i_bh6jxAEfaoVKbgVB02vWE0HbxP9oM83rANT8kBE,17871
 sky/serve/__init__.py,sha256=dNMxQ21dBqXu9BJiZz-r4yq1hUd1_eqcAlRnHeS5470,1740
 sky/serve/autoscalers.py,sha256=qPe3kR81mS3Q3Rx5IN9O-i3dlBGb8U3MWWNoLSDtqcc,32441
-sky/serve/constants.py,sha256=H_cVwI49HkS4JRg2k6TWje1vJtfAAnOivRtVrh8qjwQ,4806
+sky/serve/constants.py,sha256=TdavR0D3uNISK9GXasZhBvIp3wBbRKifRrN9cWOzXDU,5002
 sky/serve/controller.py,sha256=jtzWHsLHnVPQ727ZpDZTUpGTtIOssbnQpXeWOyAuW_s,11886
 sky/serve/load_balancer.py,sha256=2nkMPRvy-h7hJL4Qq__tkT8nIAVC_nmjyXf8mMGYEFk,13658
-sky/serve/load_balancing_policies.py,sha256=XVj76qBgqh7h6wfx53RKQFzBefDWTE4TCdCEtFLLtI4,5398
+sky/serve/load_balancing_policies.py,sha256=Ssr0GtHKyzzAIEOUAMCLvND5rapmrpxI-OQemf3dgxU,5177
 sky/serve/replica_managers.py,sha256=mdZ7pEK6TzYx54YybE9u4CKv4VO_QnHBtCMsvDuuh80,62221
-sky/serve/serve_state.py,sha256=Do-MITtijw1z8dDUyJZMAGnjZCgdUFtJ2nwhhcjpOGI,20056
+sky/serve/serve_state.py,sha256=P1aR_bHfmL8MZdMhjLNu0bfYxsIWw8lWg4QFSme0ZLc,19690
 sky/serve/serve_utils.py,sha256=e6piFZ_6Mg1Wkrg5ObtqV4MFQ2MG7bGQmGhd3Kmms58,45947
 sky/serve/service.py,sha256=JlLyhwHbBUztLgKrfBpVRIUhGwGizemfxL7txs9dpcc,12539
 sky/serve/service_spec.py,sha256=t7wh7sx68DKjPyrk1Ub-c1ZjkWD8ehG-YOj9CNdC9qQ,18552
@@ -226,12 +226,12 @@ sky/serve/spot_placer.py,sha256=LZk-VFPLyKpSJHFlNj-OkTN8C3KppD-IOocp5BHb6X4,1107
 sky/serve/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/serve/client/sdk.py,sha256=4lcil59KNmJSPUpPaq7x1oj6KsdXxCw-GUMU9QiTli8,11967
 sky/serve/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/serve/server/core.py,sha256=eShbugECWzJoSUDq5tZt-pePG0hgMY4dqY8AWNZ9_9s,33537
+sky/serve/server/core.py,sha256=4iFfxG1epVEL3AFMHBvKtTETa1v9b1aqaoKVPfG5kAA,33710
 sky/serve/server/server.py,sha256=gQGVU9nHYdGbaLhGjIUNIYn4xwKjRASRJkiiTL5AI1Y,3283
 sky/server/__init__.py,sha256=MPPBqFzXz6Jv5QSk6td_IcvnfXfNErDZVcizu4MLRow,27
-sky/server/common.py,sha256=0Ob8MfIUhjL8XGXm2DiKfJtEvNfb0Ez4m2HX0820uRI,19879
-sky/server/constants.py,sha256=hYY00IKxBQGveZMuWATOv4vHsgJNN8sCnI-t5G78sAc,948
-sky/server/server.py,sha256=7aK3Nv_aHIYX2ZB_NFy-5Cn-DbSxUPMeiWaDY_2DLXE,44674
+sky/server/common.py,sha256=-fRaJnxueJec8DSON4AWp2MEmKq10LWrUTSOaN-ebRs,23627
+sky/server/constants.py,sha256=L6W9H0YWOnPOF6WJkst11Ev1nBt3Y7dfyFSuaCSrwqA,948
+sky/server/server.py,sha256=9m5bE83fsOUHJ0SiHZ9TSroOyZe8014KKf5KeHA5eJA,44685
 sky/server/stream_utils.py,sha256=4JMHgtoXPpCT8JwtqyUcDQ9IdZFir9om0JaCRr8rvbQ,5849
 sky/server/uvicorn.py,sha256=wajwPHJ3IEEP3GMNOCc0S81-1v2qT5F-ejUkLFVhUzk,2953
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
@@ -246,19 +246,19 @@ sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 sky/server/requests/queues/local_queue.py,sha256=X6VkBiUmgd_kfqIK1hCtMWG1b8GiZbY70TBiBR6c6GY,416
 sky/server/requests/queues/mp_queue.py,sha256=jDqP4Jd28U3ibSFyMR1DF9I2OWZrPZqFJrG5S6RFpyw,3403
 sky/server/requests/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/server/requests/serializers/decoders.py,sha256=0cpg80uAqkdK_LqcQPkpKswhcNUUztG9luDLm_0eUow,6811
-sky/server/requests/serializers/encoders.py,sha256=i4SAb5Oyp00CyMkyidbdA9dtxAzxZl40KTpL_x6pH0w,5679
+sky/server/requests/serializers/decoders.py,sha256=F7eqENzt7hH92BPYUGnSJ_-XGobA0whPezTh5EDLCQs,6729
+sky/server/requests/serializers/encoders.py,sha256=4bQV5yTg8RTPT_HkRyQpjaBY_uUvBJ4NH189W0-6Pi0,5578
 sky/setup_files/MANIFEST.in,sha256=cHYG6IdIp7RsDapL4Lrs-WTeYJftHn6qystSolmyyk8,581
-sky/setup_files/dependencies.py,sha256=adacO1H3Y0LUJgcCOlTVnbaDSUBgXLV42Qr4OW0amb0,6456
+sky/setup_files/dependencies.py,sha256=ZNn608y0Yb2oTz9prWDxgPEeH9QEmfJ3CzlIcE021QQ,6489
 sky/setup_files/setup.py,sha256=Q9f0RvsdPC0FLvyTKW-upQtRuA81jRO4TtN3VK-mP-Y,7436
 sky/skylet/LICENSE,sha256=BnFrJSvUFpMUoH5mOpWnEvaC5R6Uux8W6WXgrte8iYg,12381
 sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=kGUnHm-jpF4zl3UJfB-4pnoldWpnVeR96WwYGSw7em0,4630
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=963O5hk98tw-osqGx2BNtFkb4Z5nHnii1_wfSJeVUSI,18556
+sky/skylet/constants.py,sha256=kCUODpKMTIYn1x8hjhqJPnI__ZzKEpiMizzoY_pViNA,18446
 sky/skylet/events.py,sha256=pnV3ZiwWhXqTHpU5B5Y9Xwam_7FQDI6IrxgSx7X_NVA,12743
-sky/skylet/job_lib.py,sha256=FUpaCSg5yl4ptFb3_GOeUunHTz4qJvwJJjeqrpiRUTk,44470
+sky/skylet/job_lib.py,sha256=LInoIK4M37ZPSb5c7u5d8h990B68O2imiayDIH4wEaA,44473
 sky/skylet/log_lib.py,sha256=eQT_nbdPVp6wLtFxeoYL5Vt-VFIdmV588G08iEljncc,21053
 sky/skylet/log_lib.pyi,sha256=rRk4eUX0RHGs1QL9CXsJq6RE7FqqxZlfuPJOLXTvg7I,4453
 sky/skylet/skylet.py,sha256=mWmqCvxSlfdVU_L8NL6P52jmCt3smd8K0HdyNBfMPeI,1234
@@ -291,7 +291,7 @@ sky/templates/jobs-controller.yaml.j2,sha256=LbeWGkarOcrRbbdvkZv_ZfyydEcJBohKItC
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=iw7mypHszg6Ggq9MbyiYMFOkSlXaQZulaxqC5IWYGCc,3381
-sky/templates/kubernetes-ray.yml.j2,sha256=JHAIdZJwhCwR33aci1b2IbbHhwdGGf4U8xY8gE4ZaJ8,31780
+sky/templates/kubernetes-ray.yml.j2,sha256=N99HvRVr9S6OgAwKZ8vsAd2OpxjapiNElwQdvfY21_g,31778
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
 sky/templates/lambda-ray.yml.j2,sha256=IWUFROlaVV9qsH4tYInn0xgyE1WAP_dHZBnGkNpmpYM,4713
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
@@ -318,9 +318,9 @@ sky/utils/command_runner.py,sha256=aEBs4Km8b6PqDklNc63tVYMK0w3PBGQEEP21_wmhG1k,3
 sky/utils/command_runner.pyi,sha256=mJOzCgcYZAfHwnY_6Wf1YwlTEJGb9ihzc2f0rE0Kw98,7751
 sky/utils/common.py,sha256=P4oVXFATUYgkruHX92cN12SJBtfb8DiOOYZtbN1kvP0,1927
 sky/utils/common_utils.py,sha256=UM2eSQNdXRvAzlbfC839E7-7DXC9BMMUkquLsmYpu8w,31619
-sky/utils/config_utils.py,sha256=7zEdqq1voSrlpDyZsBAhuTzdsOGzL4WpjWKkjEEd1Eg,9036
+sky/utils/config_utils.py,sha256=DZ_NN46ZZyavlkkM077lp-6FqzlUOB2cTeu50lz8NZc,10395
 sky/utils/control_master_utils.py,sha256=iD4M0onjYOdZ2RuxjwMBl4KhafHXJzuHjvqlBUnu-VE,1450
-sky/utils/controller_utils.py,sha256=6JpqPATZGvwN7P-qGn4jZ_6ka4mZVXYgZtTg49U3YD8,49808
+sky/utils/controller_utils.py,sha256=JnEaO7tWijMONjaqEXuodAUDhuvCHjvCaW7YL59sQZM,51612
 sky/utils/dag_utils.py,sha256=sAus0aL1wtuuFZSDnpO4LY-6WK4u5iJY952oWQzHo3Y,7532
 sky/utils/db_utils.py,sha256=K2-OHPg0FeHCarevMdWe0IWzm6wWumViEeYeJuGoFUE,3747
 sky/utils/env_options.py,sha256=aaD6GoYK0LaZIqjOEZ-R7eccQuiRriW3EuLWtOI5En8,1578
@@ -331,30 +331,30 @@ sky/utils/registry.py,sha256=I08nS0rvCF-xR5GEZoHEVgN1jcOeglz77h7xPpBCIjU,4179
 sky/utils/resources_utils.py,sha256=GbmD95DPwPB7xCnczhEpcopY16PUD6a37Xlq3U8mAe4,10530
 sky/utils/rich_console_utils.py,sha256=wPvAlshaFHuMZSjiDnaK3OSBppZLBjAn-lj7AvxNBQk,553
 sky/utils/rich_utils.py,sha256=PK5nVUFRMvg7ngYoIxZU6Ldcyiy3VfXriOLgNFkv53g,12980
-sky/utils/schemas.py,sha256=1TXAqql1S4WRYIgzTwR_qJklK6uzd11Ote5Q-lIOSOk,31403
+sky/utils/schemas.py,sha256=eLJDOoqUAc59q81VtC8MeVAlddP6sIFZ3o1bQKhKLco,32110
 sky/utils/status_lib.py,sha256=zn_MSuRYQdNKF8pnFOGQ54X_s_R7dyqWS6Q3a9zENw8,1512
 sky/utils/subprocess_utils.py,sha256=yM2WumV49gSKuZs0v6E3R8XKl5Q9b6veIzi6us5ORU8,15927
 sky/utils/timeline.py,sha256=ob6s3bc7nwAuSI76yLKBrSR5bzOHnOhbozz1avwoet4,4070
 sky/utils/ux_utils.py,sha256=R-ddrqcwKngziZz5haHufxiUnABaMMbmRVsaUljrPBg,10181
 sky/utils/validator.py,sha256=yo5cPUjGxqfa0ZxGyEYZMCWZ8O35G-k3VOEAtAoA_3w,856
 sky/utils/cli_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/utils/cli_utils/status_utils.py,sha256=LwGXzMgvnQeGR1fCC24q38hRLuAPeeSDkQ387eG6YSs,13495
+sky/utils/cli_utils/status_utils.py,sha256=aQRD03tbLqbpIwTRsUFngaNWKDwFzNBV1uUmUwoIzrQ,13876
 sky/utils/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/kubernetes/create_cluster.sh,sha256=VLXfazav9XCMQmeKVqhuOQzt2vM6G1jgnvvb0SHUFno,7773
 sky/utils/kubernetes/delete_cluster.sh,sha256=BSccHF43GyepDNf-FZcenzHzpXXATkVD92vgn1lWPgk,927
 sky/utils/kubernetes/deploy_remote_cluster.sh,sha256=EQMBC0VUqe3UaiYvwkNq4P6U9bkiBwjTNTE0z-MsTBc,10785
 sky/utils/kubernetes/exec_kubeconfig_converter.py,sha256=fE1SnteoxI05EaugnWeV82hXwZTVHmbXsh1aaZAgF3c,2548
 sky/utils/kubernetes/generate_kind_config.py,sha256=_TNLnifA_r7-CRq083IP1xjelYqiLjzQX9ohuqYpDH8,3187
-sky/utils/kubernetes/generate_kubeconfig.sh,sha256=4vcZ2kBikYbwoPF8JyxzGDsbdRirSSmRWSnXqVE_rZ0,10483
-sky/utils/kubernetes/gpu_labeler.py,sha256=MKsEiWitn-LOG3-OEdBZ-MXJd_L3ztdB9JrhD6D2T1w,6912
+sky/utils/kubernetes/generate_kubeconfig.sh,sha256=MBvXJio0PeujZSCXiRKE_pa6HCTiU9qBzR1WrXccVSY,10477
+sky/utils/kubernetes/gpu_labeler.py,sha256=5BGv7m5Tb_TfP7qu9kix5A1uEIa-tK_sbAQ70aP0VwU,10321
 sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7ZWF5gdVIZPupCCo9A,1224
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=HPVgNt-wbCVPd9dpDFiA7t2mzQLpjXHJ61eiwRbEr-c,10378
 sky/utils/kubernetes/rsync_helper.sh,sha256=eaQOyvDq_RmcRHBqB9tH6LyDuYC310IVp97C5nqrTQg,1793
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250413.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250413.dist-info/METADATA,sha256=xqyWNBCx-R5uIjJfyGJZ5LE_j5Y0sDB2jO_pd227LPs,18650
-skypilot_nightly-1.0.0.dev20250413.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-skypilot_nightly-1.0.0.dev20250413.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250413.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250413.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250417.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250417.dist-info/METADATA,sha256=3V6Pzkw34VZOuZH2O-2iB27VNTk2fYXNpQSMOkJmmog,18691
+skypilot_nightly-1.0.0.dev20250417.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250417.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250417.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250417.dist-info/RECORD,,