skypilot-nightly 1.0.0.dev20250403__py3-none-any.whl → 1.0.0.dev20250405__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '7149e721d257cfc890526c9738290724882ce82e'
+_SKYPILOT_COMMIT_SHA = '5025c819176d48aa8643fe9ec058158ce2bece9a'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250403'
+__version__ = '1.0.0.dev20250405'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -655,7 +655,10 @@ class RayCodeGen:
         rclone_flush_script = {rclone_flush_script!r}
         if run_fn is not None:
             script = run_fn({gang_scheduling_id}, gang_scheduling_id_to_ip)
-        script += rclone_flush_script
+        if script is not None:
+            script += rclone_flush_script
+        else:
+            script = rclone_flush_script
 
         if script is not None:
             sky_env_vars_dict['{constants.SKYPILOT_NUM_GPUS_PER_NODE}'] = {int(math.ceil(num_gpus))!r}

sky/check.py

@@ -289,6 +289,12 @@ def _print_checked_cloud(
         cloud_tuple: The cloud to print the capabilities for.
         cloud_capabilities: The capabilities for the cloud.
     """
+
+    def _yellow_color(str_to_format: str) -> str:
+        return (f'{colorama.Fore.LIGHTYELLOW_EX}'
+                f'{str_to_format}'
+                f'{colorama.Style.RESET_ALL}')
+
     cloud_repr, cloud = cloud_tuple
     # Print the capabilities for the cloud.
     # consider cloud enabled if any capability is enabled.
@@ -319,7 +325,7 @@ def _print_checked_cloud(
     if activated_account is not None:
         echo(f'    Activated account: {activated_account}')
     for reason, caps in hints_to_capabilities.items():
-        echo(f'    Hint [{", ".join(caps)}]: {reason}')
+        echo(f'    Hint [{", ".join(caps)}]: {_yellow_color(reason)}')
     for reason, caps in reasons_to_capabilities.items():
         echo(f'    Reason [{", ".join(caps)}]: {reason}')
 

sky/cli.py

@@ -3062,10 +3062,15 @@ def _down_or_stop_clusters(
                 'Letting --all take effect.')
         # We should not remove controllers when --all is specified.
         # Otherwise, it would be very easy to accidentally delete a controller.
+
+        # do not select already stopped clusters for stop command.
+        # stopped clusters are still included for down or autostop commands.
         names = [
             record['name']
             for record in all_clusters
             if controller_utils.Controllers.from_name(record['name']) is None
+            and (down or idle_minutes_to_autostop is not None or
+                 record['status'] != status_lib.ClusterStatus.STOPPED)
         ]
 
     clusters = names

sky/client/cli.py

@@ -3062,10 +3062,15 @@ def _down_or_stop_clusters(
                 'Letting --all take effect.')
         # We should not remove controllers when --all is specified.
         # Otherwise, it would be very easy to accidentally delete a controller.
+
+        # do not select already stopped clusters for stop command.
+        # stopped clusters are still included for down or autostop commands.
         names = [
             record['name']
             for record in all_clusters
             if controller_utils.Controllers.from_name(record['name']) is None
+            and (down or idle_minutes_to_autostop is not None or
+                 record['status'] != status_lib.ClusterStatus.STOPPED)
         ]
 
     clusters = names

sky/clouds/kubernetes.py

@@ -679,7 +679,8 @@ class Kubernetes(clouds.Cloud):
         success = False
         for context in existing_allowed_contexts:
             try:
-                check_result = kubernetes_utils.check_credentials(context)
+                check_result = kubernetes_utils.check_credentials(
+                    context, run_optional_checks=True)
                 if check_result[0]:
                     success = True
                     if check_result[1] is not None:
@@ -689,15 +690,6 @@ class Kubernetes(clouds.Cloud):
             except Exception as e:  # pylint: disable=broad-except
                 return (False, f'Credential check failed for {context}: '
                         f'{common_utils.format_exception(e)}')
-            unlabeled_nodes = kubernetes_utils.get_unlabeled_accelerator_nodes(
-                context)
-            if len(unlabeled_nodes) > 0:
-                hints.append(f'Context {context} has {len(unlabeled_nodes)} '
-                             f'unlabeled nodes with accelerators. '
-                             f'To label these nodes, run '
-                             f'`python -m sky.utils.kubernetes.gpu_labeler '
-                             f'--context {context}` from the project root '
-                             f'directory.')
         if success:
             return (True, cls._format_credential_check_results(hints, reasons))
         return (False, 'Failed to find available context with working '

sky/clouds/service_catalog/__init__.py

@@ -328,9 +328,9 @@ def get_common_gpus() -> List[str]:
         'A100',
         'A100-80GB',
         'H100',
+        'H200',
         'L4',
         'L40S',
-        'P100',
         'T4',
         'V100',
         'V100-32GB',

sky/provision/kubernetes/utils.py

@@ -272,7 +272,8 @@ def get_gke_accelerator_name(accelerator: str) -> str:
     if accelerator == 'H100':
         # H100 is named as H100-80GB in GKE.
         accelerator = 'H100-80GB'
-    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB'):
+    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB',
+                       'B200'):
         # A100-80GB, L4, H100-80GB and H100-MEGA-80GB
         # have a different name pattern.
         return 'nvidia-{}'.format(accelerator.lower())
@@ -1096,7 +1097,9 @@ def get_accelerator_label_key_value(
         ResourcesUnavailableError: Can be raised from the following conditions:
             - The cluster does not have GPU/TPU resources
                 (nvidia.com/gpu, google.com/tpu)
-            - The cluster does not have GPU/TPU labels setup correctly
+            - The cluster has GPU/TPU resources, but no node in the cluster has
+              an accelerator label.
+            - The cluster has a node with an invalid accelerator label value.
             - The cluster doesn't have any nodes with acc_type GPU/TPU
     """
     # Check if the cluster has GPU resources
@@ -1291,7 +1294,8 @@ def get_external_ip(network_mode: Optional[
 
 
 def check_credentials(context: Optional[str],
-                      timeout: int = kubernetes.API_TIMEOUT) -> \
+                      timeout: int = kubernetes.API_TIMEOUT,
+                      run_optional_checks: bool = False) -> \
         Tuple[bool, Optional[str]]:
     """Check if the credentials in kubeconfig file are valid
 
@@ -1333,6 +1337,9 @@ def check_credentials(context: Optional[str],
                        f'{common_utils.format_exception(e, use_bracket=True)}')
 
     # If we reach here, the credentials are valid and Kubernetes cluster is up.
+    if not run_optional_checks:
+        return True, None
+
     # We now do softer checks to check if exec based auth is used and to
     # see if the cluster is GPU-enabled.
 
@@ -1344,16 +1351,36 @@ def check_credentials(context: Optional[str],
     # `sky launch --gpus <gpu>` and the optimizer does not list Kubernetes as a
     # provider if their cluster GPUs are not setup correctly.
     gpu_msg = ''
-    try:
-        get_accelerator_label_key_value(context,
-                                        acc_type='',
-                                        acc_count=0,
-                                        check_mode=True)
-    except exceptions.ResourcesUnavailableError as e:
-        # If GPUs are not available, we return cluster as enabled (since it can
-        # be a CPU-only cluster) but we also return the exception message which
-        # serves as a hint for how to enable GPU access.
-        gpu_msg = str(e)
+    unlabeled_nodes = get_unlabeled_accelerator_nodes(context)
+    if unlabeled_nodes:
+        gpu_msg = (f'Cluster has {len(unlabeled_nodes)} nodes with '
+                   f'accelerators that are not labeled. '
+                   f'To label the nodes, run '
+                   f'`python -m sky.utils.kubernetes.gpu_labeler '
+                   f'--context {context}`')
+    else:
+        try:
+            # This function raises a ResourcesUnavailableError in three cases:
+            # 1. If no node in cluster has GPU/TPU resource in its capacity.
+            #    (e.g. google.com/tpu, nvidia.com/gpu)
+            # 2. If at least one node in cluster has GPU/TPU resource in its
+            #    capacity, but no node in the cluster has an accelerator label.
+            # 3. If an accelerator label on a node is invalid.
+            # Exception 2 is a special case of a cluster having at least one
+            # unlabelled node, which is caught in
+            # `get_unlabeled_accelerator_nodes`.
+            # Therefore, if `get_unlabeled_accelerator_nodes` detects unlabelled
+            # nodes, we skip this check.
+            get_accelerator_label_key_value(context,
+                                            acc_type='',
+                                            acc_count=0,
+                                            check_mode=True)
+        except exceptions.ResourcesUnavailableError as e:
+            # If GPUs are not available, we return cluster as enabled
+            # (since it can be a CPU-only cluster) but we also return the
+            # exception message which serves as a hint for how to enable
+            # GPU access.
+            gpu_msg = str(e)
     if exec_msg and gpu_msg:
         return True, f'{gpu_msg}\n    Additionally, {exec_msg}'
     elif gpu_msg:

sky/server/requests/executor.py

@@ -260,6 +260,12 @@ def _request_execution_wrapper(request_id: str,
                 f.flush()
         except KeyboardInterrupt:
             logger.info(f'Request {request_id} cancelled by user')
+            # Kill all children processes related to this request.
+            # Each executor handles a single request, so we can safely kill all
+            # children processes related to this request.
+            # This is required as python does not pass the KeyboardInterrupt
+            # to the threads that are not main thread.
+            subprocess_utils.kill_children_processes()
             _restore_output(original_stdout, original_stderr)
             return
         except (Exception, SystemExit) as e:  # pylint: disable=broad-except

{skypilot_nightly-1.0.0.dev20250403.dist-info → skypilot_nightly-1.0.0.dev20250405.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250403
+Version: 1.0.0.dev20250405
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250403.dist-info → skypilot_nightly-1.0.0.dev20250405.dist-info}/RECORD

@@ -1,8 +1,8 @@
-sky/__init__.py,sha256=jAriaLe3W4tnwwVPeiJ2hAxmMnsB0NRcf-MQ2yLMyQk,6428
+sky/__init__.py,sha256=ZkLN1hea20dguJClNeFn8F6wG99qD8HssxJoWkHxbR8,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=ND011K_-Ud1dVZF37A9KrwYir_ihJXcHc7iDWmuBc8Q,22872
-sky/check.py,sha256=oktScSPsHIyO7ZrVHy3QaybB6-s_D6eMEjmICAiUtDo,15902
-sky/cli.py,sha256=Zcio2ak6zX_5_N_lshDUqCvoV6NEOmGS6Tp6AgS9VAk,222446
+sky/check.py,sha256=PPNQnaaZBA9_aogJpN4gnG4XWnTqkd74c-rBYDkDRDY,16101
+sky/cli.py,sha256=DOMNgAjK8Iir8nQNoQDG2tIWg9Jf_Gh2DDFqHnjxflU,222727
 sky/cloud_stores.py,sha256=cmKdSoB4bmwrd-Z1NCZBFb6IIJt0jKVxkGPoX86280s,26606
 sky/core.py,sha256=G3n6z0dyvoU4FJVGnnTu3kFdu_EtQC1l57er5voRAX0,47926
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
@@ -34,7 +34,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
 sky/backends/backend_utils.py,sha256=ndY4IPs1F9QovyiKAnB1FNYGWm52_ylwf_K7wY50cv0,134922
-sky/backends/cloud_vm_ray_backend.py,sha256=uRz6XUvrTHGsF6LYIuSjFBUP51BSD5_yTj4R5-3DLo0,251922
+sky/backends/cloud_vm_ray_backend.py,sha256=ICo21xsKd1Ipy_nBHbP2FUWllOmdS0Pvr4mfypSYhXI,252012
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=meypuMaygSXXjGdXfq6dhWl-OrpAybg9KVRoup4D0wU,9098
@@ -43,7 +43,7 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=7rf-iHt6RXZ_pnBBWOMwcdodHQW69x27xNyx0yVog1U,26385
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=Zcio2ak6zX_5_N_lshDUqCvoV6NEOmGS6Tp6AgS9VAk,222446
+sky/client/cli.py,sha256=DOMNgAjK8Iir8nQNoQDG2tIWg9Jf_Gh2DDFqHnjxflU,222727
 sky/client/common.py,sha256=YMgYoSYlV3B_scfCIJRKHvDq6JKvnSvRZDhlRjRmzu0,14780
 sky/client/sdk.py,sha256=36tvJou2IaG8H5lrqMicv-gSJn6VUDHUtB7V20t4lA8,68620
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
@@ -55,7 +55,7 @@ sky/clouds/do.py,sha256=P38l4otp2AuDReUH9Ii621ht9s-NIyb7-R37jbtjHk8,11580
 sky/clouds/fluidstack.py,sha256=jIqW1MLe55MVME1PATZm8e6_FsiTnJawW7OdytPW0aM,12666
 sky/clouds/gcp.py,sha256=sUJ9LXUnMxYm6OYZ5P-z1dJHxgVILuC3OW3eFSTNCv8,56919
 sky/clouds/ibm.py,sha256=XtuPN8QgrwJdb1qb_b-7KwAE2tf_N9wh9eEfi2tcg-s,22013
-sky/clouds/kubernetes.py,sha256=VuPHstRysRZH0UTYZIExY2Gtd2ItQn9I5EboqxyuvV0,36717
+sky/clouds/kubernetes.py,sha256=SNRilcl_JYvaMDoYw0jnMtfaSi7FKQycCiJVmX6-4f4,36216
 sky/clouds/lambda_cloud.py,sha256=rR2YrZ6flEbKKpQAm60eKNjiMDYvH2hqzaCo3Hx4Ffw,12916
 sky/clouds/nebius.py,sha256=4D7C2NQYI-BNhXWNOyAXNAZj7-5nN33VQW1sxfSGt9w,14662
 sky/clouds/oci.py,sha256=YO4kjSsHBmAVH4z1TuVP72zfmC0BXte4E0xIyZir9N4,27622
@@ -64,7 +64,7 @@ sky/clouds/runpod.py,sha256=P486CMN-Mt3R-7jMVd3XIGjLWYy0X5B74dK_IgAp2Cg,12149
 sky/clouds/scp.py,sha256=6OucFxDIOZFA1Z_QZwGqblW4zdBljhUjhamssvchu_0,15971
 sky/clouds/vast.py,sha256=1AZaM71dI837oeuMZEXN4muHsH3CKRLjBAPcPy1QqV4,11296
 sky/clouds/vsphere.py,sha256=yRLQESpuSOdfs4KaVTMSQJqf_3mr4VysGqSo1zs1ZtU,12453
-sky/clouds/service_catalog/__init__.py,sha256=Fbm8kKpVTCoIumnY-FWL0X4xMpKu_qwYAxyuVXu4IBQ,14997
+sky/clouds/service_catalog/__init__.py,sha256=WCYMAhek36Ee1HjJL9LP287a70hv2NMA_8plcZDI4Z4,14997
 sky/clouds/service_catalog/aws_catalog.py,sha256=PbYD37rU_8m-Y_5xTglW21ppxI0GecM1sdO1yXuPwHE,13518
 sky/clouds/service_catalog/azure_catalog.py,sha256=5Q51x_WEKvQ2YSgJvZHRH3URlbwIstYuwpjaWW_wJlw,8149
 sky/clouds/service_catalog/common.py,sha256=6TCUE1kg2FvUZXpV2ktrnsRchUYNTGx8rq16m5Nztek,27766
@@ -166,7 +166,7 @@ sky/provision/kubernetes/constants.py,sha256=dZCUV8FOO9Gct80sdqeubKnxeW3CGl-u5mx
 sky/provision/kubernetes/instance.py,sha256=oag17OtuiqU-1RjkgW9NvEpxSGUFIYdI7M61S-YmPu8,50503
 sky/provision/kubernetes/network.py,sha256=AtcOM8wPs_-UlQJhGEQGP6Lh4HIgdx63Y0iWEhP5jyc,12673
 sky/provision/kubernetes/network_utils.py,sha256=6uck1aBkgtm-gGBitU3_hEUp8j14ZuG_4Xo70ReZYXs,11654
-sky/provision/kubernetes/utils.py,sha256=IR2AKbZA-ZuKtLRuqHZsiA1N2HmQTJKz5IqOQIVEk4U,125753
+sky/provision/kubernetes/utils.py,sha256=D7xfk6RjqLAsERpJMxQ8ozu-D3pRTz_bRszKQeMsVno,127188
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -238,7 +238,7 @@ sky/server/uvicorn.py,sha256=wajwPHJ3IEEP3GMNOCc0S81-1v2qT5F-ejUkLFVhUzk,2953
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/event_loop.py,sha256=OhpPbuce65bbjpGRlcJa78AVnYSm08SzFKt70ypCUuQ,1211
-sky/server/requests/executor.py,sha256=wW-8s8APEakYRJKNiWIhHg2vrn2UUteseUTsqm9q3Fg,21693
+sky/server/requests/executor.py,sha256=txzvCUKLafRzEoY4Snk9xVFbIdw5cnu7_wkHTldQdmE,22085
 sky/server/requests/payloads.py,sha256=3sF36Z9_PLzpEncW0AplJtOz-_nsn5PJaM5lS-3Y8bw,16558
 sky/server/requests/preconditions.py,sha256=ipxIb_3JXG6S3-ymcOdqQNb7VDvoPqADxu9ZK7-nQWc,7179
 sky/server/requests/requests.py,sha256=9ovdQE-zv_Mvc6IbGATHVyQlOxSKjg_OankZbgDVGeE,21338
@@ -350,9 +350,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=HPVgNt-wbCVPd9dpDFiA7t2mzQLpjXHJ61eiwRbEr-c,10378
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250403.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250403.dist-info/METADATA,sha256=YkgmWuWgAbuQRFP3Zfk3QpnqS_0KN7x50GuSoVI9JwU,18552
-skypilot_nightly-1.0.0.dev20250403.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-skypilot_nightly-1.0.0.dev20250403.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250403.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250403.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250405.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250405.dist-info/METADATA,sha256=wdtznc_1O5ewseMG9VlObIYMZmvjQO4-Cd7woHSdQV8,18552
+skypilot_nightly-1.0.0.dev20250405.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+skypilot_nightly-1.0.0.dev20250405.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250405.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250405.dist-info/RECORD,,