skypilot-nightly 1.0.0.dev20250403__tar.gz → 1.0.0.dev20250405__tar.gz

{skypilot_nightly-1.0.0.dev20250403/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250405}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250403
+Version: 1.0.0.dev20250405
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '7149e721d257cfc890526c9738290724882ce82e'
+_SKYPILOT_COMMIT_SHA = '5025c819176d48aa8643fe9ec058158ce2bece9a'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250403'
+__version__ = '1.0.0.dev20250405'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/backends/cloud_vm_ray_backend.py

@@ -655,7 +655,10 @@ class RayCodeGen:
         rclone_flush_script = {rclone_flush_script!r}
         if run_fn is not None:
             script = run_fn({gang_scheduling_id}, gang_scheduling_id_to_ip)
-        script += rclone_flush_script
+        if script is not None:
+            script += rclone_flush_script
+        else:
+            script = rclone_flush_script
 
         if script is not None:
             sky_env_vars_dict['{constants.SKYPILOT_NUM_GPUS_PER_NODE}'] = {int(math.ceil(num_gpus))!r}

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/check.py

@@ -289,6 +289,12 @@ def _print_checked_cloud(
         cloud_tuple: The cloud to print the capabilities for.
         cloud_capabilities: The capabilities for the cloud.
     """
+
+    def _yellow_color(str_to_format: str) -> str:
+        return (f'{colorama.Fore.LIGHTYELLOW_EX}'
+                f'{str_to_format}'
+                f'{colorama.Style.RESET_ALL}')
+
     cloud_repr, cloud = cloud_tuple
     # Print the capabilities for the cloud.
     # consider cloud enabled if any capability is enabled.
@@ -319,7 +325,7 @@ def _print_checked_cloud(
     if activated_account is not None:
         echo(f'    Activated account: {activated_account}')
     for reason, caps in hints_to_capabilities.items():
-        echo(f'    Hint [{", ".join(caps)}]: {reason}')
+        echo(f'    Hint [{", ".join(caps)}]: {_yellow_color(reason)}')
     for reason, caps in reasons_to_capabilities.items():
         echo(f'    Reason [{", ".join(caps)}]: {reason}')
 

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/cli.py

@@ -3062,10 +3062,15 @@ def _down_or_stop_clusters(
                 'Letting --all take effect.')
         # We should not remove controllers when --all is specified.
         # Otherwise, it would be very easy to accidentally delete a controller.
+
+        # do not select already stopped clusters for stop command.
+        # stopped clusters are still included for down or autostop commands.
         names = [
             record['name']
             for record in all_clusters
             if controller_utils.Controllers.from_name(record['name']) is None
+            and (down or idle_minutes_to_autostop is not None or
+                 record['status'] != status_lib.ClusterStatus.STOPPED)
         ]
 
     clusters = names

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/client/cli.py

@@ -3062,10 +3062,15 @@ def _down_or_stop_clusters(
                 'Letting --all take effect.')
         # We should not remove controllers when --all is specified.
         # Otherwise, it would be very easy to accidentally delete a controller.
+
+        # do not select already stopped clusters for stop command.
+        # stopped clusters are still included for down or autostop commands.
         names = [
             record['name']
             for record in all_clusters
             if controller_utils.Controllers.from_name(record['name']) is None
+            and (down or idle_minutes_to_autostop is not None or
+                 record['status'] != status_lib.ClusterStatus.STOPPED)
         ]
 
     clusters = names

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/clouds/kubernetes.py

@@ -679,7 +679,8 @@ class Kubernetes(clouds.Cloud):
         success = False
         for context in existing_allowed_contexts:
             try:
-                check_result = kubernetes_utils.check_credentials(context)
+                check_result = kubernetes_utils.check_credentials(
+                    context, run_optional_checks=True)
                 if check_result[0]:
                     success = True
                     if check_result[1] is not None:
@@ -689,15 +690,6 @@ class Kubernetes(clouds.Cloud):
             except Exception as e:  # pylint: disable=broad-except
                 return (False, f'Credential check failed for {context}: '
                         f'{common_utils.format_exception(e)}')
-            unlabeled_nodes = kubernetes_utils.get_unlabeled_accelerator_nodes(
-                context)
-            if len(unlabeled_nodes) > 0:
-                hints.append(f'Context {context} has {len(unlabeled_nodes)} '
-                             f'unlabeled nodes with accelerators. '
-                             f'To label these nodes, run '
-                             f'`python -m sky.utils.kubernetes.gpu_labeler '
-                             f'--context {context}` from the project root '
-                             f'directory.')
         if success:
             return (True, cls._format_credential_check_results(hints, reasons))
         return (False, 'Failed to find available context with working '

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/clouds/service_catalog/__init__.py

@@ -328,9 +328,9 @@ def get_common_gpus() -> List[str]:
         'A100',
         'A100-80GB',
         'H100',
+        'H200',
         'L4',
         'L40S',
-        'P100',
         'T4',
         'V100',
         'V100-32GB',

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/provision/kubernetes/utils.py

@@ -272,7 +272,8 @@ def get_gke_accelerator_name(accelerator: str) -> str:
     if accelerator == 'H100':
         # H100 is named as H100-80GB in GKE.
         accelerator = 'H100-80GB'
-    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB'):
+    if accelerator in ('A100-80GB', 'L4', 'H100-80GB', 'H100-MEGA-80GB',
+                       'B200'):
         # A100-80GB, L4, H100-80GB and H100-MEGA-80GB
         # have a different name pattern.
         return 'nvidia-{}'.format(accelerator.lower())
@@ -1096,7 +1097,9 @@ def get_accelerator_label_key_value(
         ResourcesUnavailableError: Can be raised from the following conditions:
             - The cluster does not have GPU/TPU resources
                 (nvidia.com/gpu, google.com/tpu)
-            - The cluster does not have GPU/TPU labels setup correctly
+            - The cluster has GPU/TPU resources, but no node in the cluster has
+              an accelerator label.
+            - The cluster has a node with an invalid accelerator label value.
             - The cluster doesn't have any nodes with acc_type GPU/TPU
     """
     # Check if the cluster has GPU resources
@@ -1291,7 +1294,8 @@ def get_external_ip(network_mode: Optional[
 
 
 def check_credentials(context: Optional[str],
-                      timeout: int = kubernetes.API_TIMEOUT) -> \
+                      timeout: int = kubernetes.API_TIMEOUT,
+                      run_optional_checks: bool = False) -> \
         Tuple[bool, Optional[str]]:
     """Check if the credentials in kubeconfig file are valid
 
@@ -1333,6 +1337,9 @@ def check_credentials(context: Optional[str],
                        f'{common_utils.format_exception(e, use_bracket=True)}')
 
     # If we reach here, the credentials are valid and Kubernetes cluster is up.
+    if not run_optional_checks:
+        return True, None
+
     # We now do softer checks to check if exec based auth is used and to
     # see if the cluster is GPU-enabled.
 
@@ -1344,16 +1351,36 @@ def check_credentials(context: Optional[str],
     # `sky launch --gpus <gpu>` and the optimizer does not list Kubernetes as a
     # provider if their cluster GPUs are not setup correctly.
     gpu_msg = ''
-    try:
-        get_accelerator_label_key_value(context,
-                                        acc_type='',
-                                        acc_count=0,
-                                        check_mode=True)
-    except exceptions.ResourcesUnavailableError as e:
-        # If GPUs are not available, we return cluster as enabled (since it can
-        # be a CPU-only cluster) but we also return the exception message which
-        # serves as a hint for how to enable GPU access.
-        gpu_msg = str(e)
+    unlabeled_nodes = get_unlabeled_accelerator_nodes(context)
+    if unlabeled_nodes:
+        gpu_msg = (f'Cluster has {len(unlabeled_nodes)} nodes with '
+                   f'accelerators that are not labeled. '
+                   f'To label the nodes, run '
+                   f'`python -m sky.utils.kubernetes.gpu_labeler '
+                   f'--context {context}`')
+    else:
+        try:
+            # This function raises a ResourcesUnavailableError in three cases:
+            # 1. If no node in cluster has GPU/TPU resource in its capacity.
+            #    (e.g. google.com/tpu, nvidia.com/gpu)
+            # 2. If at least one node in cluster has GPU/TPU resource in its
+            #    capacity, but no node in the cluster has an accelerator label.
+            # 3. If an accelerator label on a node is invalid.
+            # Exception 2 is a special case of a cluster having at least one
+            # unlabelled node, which is caught in
+            # `get_unlabeled_accelerator_nodes`.
+            # Therefore, if `get_unlabeled_accelerator_nodes` detects unlabelled
+            # nodes, we skip this check.
+            get_accelerator_label_key_value(context,
+                                            acc_type='',
+                                            acc_count=0,
+                                            check_mode=True)
+        except exceptions.ResourcesUnavailableError as e:
+            # If GPUs are not available, we return cluster as enabled
+            # (since it can be a CPU-only cluster) but we also return the
+            # exception message which serves as a hint for how to enable
+            # GPU access.
+            gpu_msg = str(e)
     if exec_msg and gpu_msg:
         return True, f'{gpu_msg}\n    Additionally, {exec_msg}'
     elif gpu_msg:

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405}/sky/server/requests/executor.py

@@ -260,6 +260,12 @@ def _request_execution_wrapper(request_id: str,
                 f.flush()
         except KeyboardInterrupt:
             logger.info(f'Request {request_id} cancelled by user')
+            # Kill all children processes related to this request.
+            # Each executor handles a single request, so we can safely kill all
+            # children processes related to this request.
+            # This is required as python does not pass the KeyboardInterrupt
+            # to the threads that are not main thread.
+            subprocess_utils.kill_children_processes()
             _restore_output(original_stdout, original_stderr)
             return
         except (Exception, SystemExit) as e:  # pylint: disable=broad-except

{skypilot_nightly-1.0.0.dev20250403 → skypilot_nightly-1.0.0.dev20250405/skypilot_nightly.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250403
+Version: 1.0.0.dev20250405
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0