skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/provisioner.py

@@ -15,15 +15,21 @@ import colorama
 import sky
 from sky import clouds
 from sky import exceptions
+from sky import global_user_state
+from sky import logs
 from sky import provision
+from sky import resources as resources_lib
 from sky import sky_logging
+from sky import skypilot_config
 from sky.adaptors import aws
 from sky.backends import backend_utils
+from sky.jobs.server import utils as server_jobs_utils
 from sky.provision import common as provision_common
 from sky.provision import instance_setup
 from sky.provision import logging as provision_logging
 from sky.provision import metadata_utils
 from sky.skylet import constants
+from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import message_utils
 from sky.utils import resources_utils
@@ -64,6 +70,7 @@ def _bulk_provision(
 
     provision_record = provision.run_instances(provider_name,
                                                region_name,
+                                               str(cluster_name),
                                                cluster_name.name_on_cloud,
                                                config=config)
 
@@ -71,7 +78,8 @@ def _bulk_provision(
     logger.debug(f'\nWaiting for instances of {cluster_name!r} to be ready...')
     rich_utils.force_update_status(
         ux_utils.spinner_message('Launching - Checking instance status',
-                                 str(provision_logging.config.log_path)))
+                                 str(provision_logging.config.log_path),
+                                 cluster_name=str(cluster_name)))
     # AWS would take a very short time (<<1s) updating the state of the
     # instance.
     time.sleep(1)
@@ -95,6 +103,12 @@ def _bulk_provision(
         f'\nProvisioning {cluster_name!r} took {time.time() - start:.2f} '
         f'seconds.')
 
+    # Add cluster event for provisioning completion.
+    global_user_state.add_cluster_event(
+        str(cluster_name), status_lib.ClusterStatus.INIT,
+        f'Instances launched on {cloud.display_name()} in {region}',
+        global_user_state.ClusterEventType.STATUS_CHANGE)
+
     return provision_record
 
 
@@ -117,7 +131,7 @@ def bulk_provision(
         Cloud specific exceptions: If the provisioning process failed, cloud-
             specific exceptions will be raised by the cloud APIs.
     """
-    original_config = common_utils.read_yaml(cluster_yaml)
+    original_config = global_user_state.get_cluster_yaml_dict(cluster_yaml)
     head_node_type = original_config['head_node_type']
     bootstrap_config = provision_common.ProvisionConfig(
         provider_config=original_config['provider'],
@@ -155,7 +169,7 @@ def bulk_provision(
             # This error is a user error instead of a provisioning failure.
             # And there is no possibility to fix it by teardown.
             raise
-        except Exception:  # pylint: disable=broad-except
+        except Exception as exc:  # pylint: disable=broad-except
             zone_str = 'all zones'
             if zones:
                 zone_str = ','.join(zone.name for zone in zones)
@@ -177,14 +191,18 @@ def bulk_provision(
                         provider_config=original_config['provider'])
                     break
                 except NotImplementedError as e:
-                    verb = 'terminate' if terminate else 'stop'
+                    assert not terminate, (
+                        'Terminating must be supported by all clouds')
+                    exc_msg = common_utils.format_exception(exc).replace(
+                        '\n', ' ')
                     # If the underlying cloud does not support stopping
                     # instances, we should stop failover as well.
                     raise provision_common.StopFailoverError(
-                        'During provisioner\'s failover, '
-                        f'{terminate_str.lower()} {cluster_name!r} failed. '
-                        f'We cannot {verb} the resources launched, as it is '
-                        f'not supported by {cloud}. Please try launching the '
+                        f'Provisioning cluster {cluster_name.display_name} '
+                        f'failed: {exc_msg}. Failover is stopped for safety '
+                        'because the cluster was previously in UP state but '
+                        f'{cloud} does not support stopping instances to '
+                        'preserve the cluster state. Please try launching the '
                         'cluster again, or terminate it with: '
                         f'sky down {cluster_name.display_name}') from e
                 except Exception as e:  # pylint: disable=broad-except
@@ -228,9 +246,9 @@ def _ssh_probe_command(ip: str,
                        ssh_port: int,
                        ssh_user: str,
                        ssh_private_key: str,
+                       ssh_probe_timeout: int,
                        ssh_proxy_command: Optional[str] = None) -> List[str]:
-    # NOTE: Ray uses 'uptime' command and 10s timeout, we use the same
-    # setting here.
+    # NOTE: Ray uses 'uptime' command, we use the same setting here.
     command = [
         'ssh',
         '-T',
@@ -244,7 +262,7 @@ def _ssh_probe_command(ip: str,
         '-o',
         'PasswordAuthentication=no',
         '-o',
-        'ConnectTimeout=10s',
+        f'ConnectTimeout={ssh_probe_timeout}s',
         '-o',
         f'UserKnownHostsFile={os.devnull}',
         '-o',
@@ -277,6 +295,7 @@ def _wait_ssh_connection_direct(ip: str,
                                 ssh_port: int,
                                 ssh_user: str,
                                 ssh_private_key: str,
+                                ssh_probe_timeout: int,
                                 ssh_control_name: Optional[str] = None,
                                 ssh_proxy_command: Optional[str] = None,
                                 **kwargs) -> Tuple[bool, str]:
@@ -305,6 +324,7 @@ def _wait_ssh_connection_direct(ip: str,
         if success:
             return _wait_ssh_connection_indirect(ip, ssh_port, ssh_user,
                                                  ssh_private_key,
+                                                 ssh_probe_timeout,
                                                  ssh_control_name,
                                                  ssh_proxy_command)
     except socket.timeout:  # this is the most expected exception
@@ -312,7 +332,7 @@ def _wait_ssh_connection_direct(ip: str,
     except Exception as e:  # pylint: disable=broad-except
         stderr = f'Error: {common_utils.format_exception(e)}'
     command = _ssh_probe_command(ip, ssh_port, ssh_user, ssh_private_key,
-                                 ssh_proxy_command)
+                                 ssh_probe_timeout, ssh_proxy_command)
     logger.debug(f'Waiting for SSH to {ip}. Try: '
                  f'{_shlex_join(command)}. '
                  f'{stderr}')
@@ -323,6 +343,7 @@ def _wait_ssh_connection_indirect(ip: str,
                                   ssh_port: int,
                                   ssh_user: str,
                                   ssh_private_key: str,
+                                  ssh_probe_timeout: int,
                                   ssh_control_name: Optional[str] = None,
                                   ssh_proxy_command: Optional[str] = None,
                                   **kwargs) -> Tuple[bool, str]:
@@ -333,14 +354,14 @@ def _wait_ssh_connection_indirect(ip: str,
     """
     del ssh_control_name, kwargs  # unused
     command = _ssh_probe_command(ip, ssh_port, ssh_user, ssh_private_key,
-                                 ssh_proxy_command)
+                                 ssh_probe_timeout, ssh_proxy_command)
     message = f'Waiting for SSH using command: {_shlex_join(command)}'
     logger.debug(message)
     try:
         proc = subprocess.run(command,
                               shell=False,
                               check=False,
-                              timeout=10,
+                              timeout=ssh_probe_timeout,
                               stdout=subprocess.DEVNULL,
                               stderr=subprocess.PIPE)
         if proc.returncode != 0:
@@ -383,8 +404,13 @@ def wait_for_ssh(cluster_info: provision_common.ClusterInfo,
     def _retry_ssh_thread(ip_ssh_port: Tuple[str, int]):
         ip, ssh_port = ip_ssh_port
         success = False
+        ssh_probe_timeout = skypilot_config.get_nested(
+            ('provision', 'ssh_timeout'), 10)
         while not success:
-            success, stderr = waiter(ip, ssh_port, **ssh_credentials)
+            success, stderr = waiter(ip,
+                                     ssh_port,
+                                     **ssh_credentials,
+                                     ssh_probe_timeout=ssh_probe_timeout)
             if not success and time.time() - start > timeout:
                 with ux_utils.print_exception_no_traceback():
                     raise RuntimeError(
@@ -403,16 +429,27 @@ def wait_for_ssh(cluster_info: provision_common.ClusterInfo,
 
 
 def _post_provision_setup(
-        cloud_name: str, cluster_name: resources_utils.ClusterName,
-        cluster_yaml: str, provision_record: provision_common.ProvisionRecord,
+        launched_resources: resources_lib.Resources,
+        cluster_name: resources_utils.ClusterName, handle_cluster_yaml: str,
+        provision_record: provision_common.ProvisionRecord,
         custom_resource: Optional[str]) -> provision_common.ClusterInfo:
-    config_from_yaml = common_utils.read_yaml(cluster_yaml)
+    config_from_yaml = global_user_state.get_cluster_yaml_dict(
+        handle_cluster_yaml)
     provider_config = config_from_yaml.get('provider')
+    cloud_name = repr(launched_resources.cloud)
     cluster_info = provision.get_cluster_info(cloud_name,
                                               provision_record.region,
                                               cluster_name.name_on_cloud,
                                               provider_config=provider_config)
 
+    # Update cluster info in handle so cluster instance ids are set. This
+    # allows us to expose provision logs to debug nodes that failed during post
+    # provision setup.
+    handle = global_user_state.get_handle_from_cluster_name(
+        cluster_name.display_name)
+    handle.cached_cluster_info = cluster_info
+    global_user_state.update_cluster_handle(cluster_name.display_name, handle)
+
     if cluster_info.num_instances > 1:
         # Only worker nodes have logs in the per-instance log directory. Head
         # node's log will be redirected to the main log file.
@@ -437,23 +474,24 @@ def _post_provision_setup(
     # TODO(suquark): Move wheel build here in future PRs.
     # We don't set docker_user here, as we are configuring the VM itself.
     ssh_credentials = backend_utils.ssh_credential_from_yaml(
-        cluster_yaml, ssh_user=cluster_info.ssh_user)
+        handle_cluster_yaml, ssh_user=cluster_info.ssh_user)
     docker_config = config_from_yaml.get('docker', {})
 
     with rich_utils.safe_status(
-            ux_utils.spinner_message(
-                'Launching - Waiting for SSH access',
-                provision_logging.config.log_path)) as status:
+            ux_utils.spinner_message('Launching - Waiting for SSH access',
+                                     provision_logging.config.log_path,
+                                     cluster_name=str(cluster_name))) as status:
         # If on Kubernetes, skip SSH check since the pods are guaranteed to be
         # ready by the provisioner, and we use kubectl instead of SSH to run the
         # commands and rsync on the pods. SSH will still be ready after a while
         # for the users to SSH into the pod.
-        if cloud_name.lower() != 'kubernetes':
+        is_k8s_cloud = cloud_name.lower() in ['kubernetes', 'ssh']
+        if not is_k8s_cloud:
             logger.debug(
                 f'\nWaiting for SSH to be available for {cluster_name!r} ...')
             wait_for_ssh(cluster_info, ssh_credentials)
             logger.debug(f'SSH Connection ready for {cluster_name!r}')
-        vm_str = 'Instance' if cloud_name.lower() != 'kubernetes' else 'Pod'
+        vm_str = 'Instance' if not is_k8s_cloud else 'Pod'
         plural = '' if len(cluster_info.instances) == 1 else 's'
         verb = 'is' if len(cluster_info.instances) == 1 else 'are'
         indent_str = (ux_utils.INDENT_SYMBOL
@@ -472,7 +510,8 @@ def _post_provision_setup(
             status.update(
                 ux_utils.spinner_message(
                     'Launching - Initializing docker container',
-                    provision_logging.config.log_path))
+                    provision_logging.config.log_path,
+                    cluster_name=str(cluster_name)))
             docker_user = instance_setup.initialize_docker(
                 cluster_name.name_on_cloud,
                 docker_config=docker_config,
@@ -489,6 +528,25 @@ def _post_provision_setup(
             logger.info(f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Style.DIM}'
                         f'Docker container is up.{colorama.Style.RESET_ALL}')
 
+        # Check version compatibility for jobs controller clusters
+        if cluster_name.display_name.startswith(common.JOB_CONTROLLER_PREFIX):
+            # TODO(zeping): remove this in v0.12.0
+            # This only happens in upgrade from <0.9.3 to > 0.10.0
+            # After 0.10.0 no incompatibility issue
+            # See https://github.com/skypilot-org/skypilot/pull/6096
+            # For more details
+            status.update(
+                ux_utils.spinner_message(
+                    'Checking controller version compatibility'))
+
+            try:
+                server_jobs_utils.check_version_mismatch_and_non_terminal_jobs()
+            except exceptions.ClusterNotUpError:
+                # Controller is not up yet during initial provisioning, that
+                # also means no non-terminal jobs, so no incompatibility in
+                # this case.
+                pass
+
         # We mount the metadata with sky wheel for speedup.
         # NOTE: currently we mount all credentials for all nodes, because
         # (1) jobs controllers need permission to launch/down nodes of
@@ -502,7 +560,8 @@ def _post_provision_setup(
 
         runtime_preparation_str = (ux_utils.spinner_message(
             'Preparing SkyPilot runtime ({step}/3 - {step_name})',
-            provision_logging.config.log_path))
+            provision_logging.config.log_path,
+            cluster_name=str(cluster_name)))
         status.update(
             runtime_preparation_str.format(step=1, step_name='initializing'))
         instance_setup.internal_file_mounts(cluster_name.name_on_cloud,
@@ -636,19 +695,32 @@ def _post_provision_setup(
             logger.debug('Ray cluster is ready. Skip starting ray cluster on '
                          'worker nodes.')
 
-        instance_setup.start_skylet_on_head_node(cluster_name.name_on_cloud,
-                                                 cluster_info, ssh_credentials)
+        logging_agent = logs.get_logging_agent()
+        if logging_agent:
+            status.update(
+                ux_utils.spinner_message('Setting up logging agent',
+                                         provision_logging.config.log_path,
+                                         cluster_name=str(cluster_name)))
+            instance_setup.setup_logging_on_cluster(logging_agent, cluster_name,
+                                                    cluster_info,
+                                                    ssh_credentials)
+
+        instance_setup.start_skylet_on_head_node(cluster_name, cluster_info,
+                                                 ssh_credentials,
+                                                 launched_resources)
 
     logger.info(
         ux_utils.finishing_message(f'Cluster launched: {cluster_name}.',
-                                   provision_logging.config.log_path))
+                                   provision_logging.config.log_path,
+                                   cluster_name=str(cluster_name)))
     return cluster_info
 
 
 @timeline.event
 def post_provision_runtime_setup(
-        cloud_name: str, cluster_name: resources_utils.ClusterName,
-        cluster_yaml: str, provision_record: provision_common.ProvisionRecord,
+        launched_resources: resources_lib.Resources,
+        cluster_name: resources_utils.ClusterName, handle_cluster_yaml: str,
+        provision_record: provision_common.ProvisionRecord,
         custom_resource: Optional[str],
         log_dir: str) -> provision_common.ClusterInfo:
     """Run internal setup commands after provisioning and before user setup.
@@ -659,6 +731,7 @@ def post_provision_runtime_setup(
        and other necessary files to the VM.
     3. Run setup commands to install dependencies.
     4. Start ray cluster and skylet.
+    5. (Optional) Setup logging agent.
 
     Raises:
         RuntimeError: If the setup process encounters any error.
@@ -666,11 +739,12 @@ def post_provision_runtime_setup(
     with provision_logging.setup_provision_logging(log_dir):
         try:
             logger.debug(_TITLE.format('System Setup After Provision'))
-            return _post_provision_setup(cloud_name,
-                                         cluster_name,
-                                         cluster_yaml=cluster_yaml,
-                                         provision_record=provision_record,
-                                         custom_resource=custom_resource)
+            return _post_provision_setup(
+                launched_resources,
+                cluster_name,
+                handle_cluster_yaml=handle_cluster_yaml,
+                provision_record=provision_record,
+                custom_resource=custom_resource)
         except Exception:  # pylint: disable=broad-except
             logger.error(
                 ux_utils.error_message(

sky/provision/runpod/__init__.py

@@ -9,3 +9,8 @@ from sky.provision.runpod.instance import run_instances
 from sky.provision.runpod.instance import stop_instances
 from sky.provision.runpod.instance import terminate_instances
 from sky.provision.runpod.instance import wait_instances
+from sky.provision.runpod.volume import apply_volume
+from sky.provision.runpod.volume import delete_volume
+from sky.provision.runpod.volume import get_all_volumes_usedby
+from sky.provision.runpod.volume import get_volume_usedby
+from sky.provision.runpod.volume import map_all_volumes_usedby

sky/provision/runpod/instance.py

@@ -1,6 +1,6 @@
 """RunPod instance provisioning."""
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from sky import sky_logging
 from sky.provision import common
@@ -44,10 +44,10 @@ def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['CREATED', 'RESTARTING']
 
     while True:
@@ -80,6 +80,21 @@ def run_instances(region: str, cluster_name_on_cloud: str,
             created_instance_ids=[])
 
     created_instance_ids = []
+    volume_mounts = config.node_config.get('VolumeMounts', [])
+    network_volume_id = None
+    volume_mount_path = None
+    if volume_mounts:
+        if len(volume_mounts) > 1:
+            logger.warning(
+                f'RunPod only supports one network volume mount, '
+                f'but {len(volume_mounts)} are specified. Only the first one '
+                f'will be used.')
+        volume_mount = volume_mounts[0]
+        network_volume_id = volume_mount.get('VolumeIdOnCloud')
+        volume_mount_path = volume_mount.get('MountPath')
+        if network_volume_id is None or volume_mount_path is None:
+            raise RuntimeError(
+                'Network volume ID and mount path must be specified.')
     for _ in range(to_start_count):
         node_type = 'head' if head_instance_id is None else 'worker'
         try:
@@ -97,6 +112,8 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                 bid_per_gpu=config.node_config['BidPerGPU'],
                 docker_login_config=config.provider_config.get(
                     'docker_login_config'),
+                network_volume_id=network_volume_id,
+                volume_mount_path=volume_mount_path,
             )
         except Exception as e:  # pylint: disable=broad-except
             logger.warning(f'run_instances error: {e}')
@@ -201,11 +218,14 @@ def get_cluster_info(
 
 
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     instances = _filter_instances(cluster_name_on_cloud, None)
 
@@ -215,12 +235,13 @@ def query_instances(
         'PAUSED': status_lib.ClusterStatus.INIT,
         'RUNNING': status_lib.ClusterStatus.UP,
     }
-    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst_id, inst in instances.items():
         status = status_map[inst['status']]
         if non_terminated_only and status is None:
             continue
-        statuses[inst_id] = status
+        statuses[inst_id] = (status, None)
     return statuses
 
 

sky/provision/runpod/utils.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from sky import sky_logging
 from sky.adaptors import runpod
 from sky.provision import docker_utils
-import sky.provision.runpod.api.commands as runpod_commands
+from sky.provision.runpod.api import commands as runpod_commands
 from sky.skylet import constants
 from sky.utils import common_utils
 
@@ -263,25 +263,36 @@ def _create_template_for_docker_login(
     return login_config.format_image(image_name), create_template_resp['id']
 
 
-def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
-           zone: str, disk_size: int, image_name: str,
-           ports: Optional[List[int]], public_key: str,
-           preemptible: Optional[bool], bid_per_gpu: float,
-           docker_login_config: Optional[Dict[str, str]]) -> str:
+def launch(
+    cluster_name: str,
+    node_type: str,
+    instance_type: str,
+    region: str,
+    zone: str,
+    disk_size: int,
+    image_name: str,
+    ports: Optional[List[int]],
+    public_key: str,
+    preemptible: Optional[bool],
+    bid_per_gpu: float,
+    docker_login_config: Optional[Dict[str, str]],
+    *,
+    network_volume_id: Optional[str] = None,
+    volume_mount_path: Optional[str] = None,
+) -> str:
     """Launches an instance with the given parameters.
 
-    Converts the instance_type to the RunPod GPU name, finds the specs for the
-    GPU, and launches the instance.
+    For CPU instances, we directly use the instance_type for launching the
+    instance.
+
+    For GPU instances, we convert the instance_type to the RunPod GPU name,
+    and finds the specs for the GPU, before launching the instance.
 
     Returns:
         instance_id: The instance ID.
     """
     name = f'{cluster_name}-{node_type}'
-    gpu_type = GPU_NAME_MAP[instance_type.split('_')[1]]
-    gpu_quantity = int(instance_type.split('_')[0].replace('x', ''))
-    cloud_type = instance_type.split('_')[2]
 
-    gpu_specs = runpod.runpod.get_gpu(gpu_type)
     # TODO(zhwu): keep this align with setups in
     # `provision.kuberunetes.instance.py`
     setup_cmd = (
@@ -329,12 +340,7 @@ def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
     params = {
         'name': name,
         'image_name': image_name_formatted,
-        'gpu_type_id': gpu_type,
-        'cloud_type': cloud_type,
         'container_disk_in_gb': disk_size,
-        'min_vcpu_count': 4 * gpu_quantity,
-        'min_memory_in_gb': gpu_specs['memoryInGb'] * gpu_quantity,
-        'gpu_count': gpu_quantity,
         'country_code': region,
         'data_center_id': zone,
         'ports': ports_str,
@@ -343,12 +349,39 @@ def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
         'template_id': template_id,
     }
 
+    # Optional network volume mount.
+    if volume_mount_path is not None:
+        params['volume_mount_path'] = volume_mount_path
+    if network_volume_id is not None:
+        params['network_volume_id'] = network_volume_id
+
+    # GPU instance types start with f'{gpu_count}x',
+    # CPU instance types start with 'cpu'.
+    is_cpu_instance = instance_type.startswith('cpu')
+    if is_cpu_instance:
+        # RunPod CPU instances can be uniquely identified by the instance_id.
+        params.update({
+            'instance_id': instance_type,
+        })
+    else:
+        gpu_type = GPU_NAME_MAP[instance_type.split('_')[1]]
+        gpu_quantity = int(instance_type.split('_')[0].replace('x', ''))
+        cloud_type = instance_type.split('_')[2]
+        gpu_specs = runpod.runpod.get_gpu(gpu_type)
+        params.update({
+            'gpu_type_id': gpu_type,
+            'cloud_type': cloud_type,
+            'min_vcpu_count': 4 * gpu_quantity,
+            'min_memory_in_gb': gpu_specs['memoryInGb'] * gpu_quantity,
+            'gpu_count': gpu_quantity,
+        })
+
     if preemptible is None or not preemptible:
         new_instance = runpod.runpod.create_pod(**params)
     else:
         new_instance = runpod_commands.create_spot_pod(
             bid_per_gpu=bid_per_gpu,
-            **params,
+            **params,  # type: ignore[arg-type]
         )
 
     return new_instance['id']