skypilot-nightly 1.0.0.dev20241109__py3-none-any.whl → 1.0.0.dev20241111__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '42c79e1d0a5e018e275705ada53957573f9a0181'
+_SKYPILOT_COMMIT_SHA = '91323d86baaeb1341c6953e15bbf19f2896b67ad'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241109'
+__version__ = '1.0.0.dev20241111'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -3979,25 +3979,6 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 stdout = ''
                 stderr = str(e)
 
-        # Apr, 2023 by Hysun(hysun.he@oracle.com): Added support for OCI
-        # May, 2023 by Hysun: Allow terminate INIT cluster which may have
-        # some instances provisioning in background but not completed.
-        elif (isinstance(cloud, clouds.OCI) and terminate and
-              prev_cluster_status in (status_lib.ClusterStatus.STOPPED,
-                                      status_lib.ClusterStatus.INIT)):
-            region = config['provider']['region']
-
-            # pylint: disable=import-outside-toplevel
-            from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME
-
-            from sky.skylet.providers.oci.query_helper import oci_query_helper
-
-            # 0: All terminated successfully, failed count otherwise
-            returncode = oci_query_helper.terminate_instances_by_tags(
-                {TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud}, region)
-
-            # To avoid undefined local variables error.
-            stdout = stderr = ''
         else:
             config['provider']['cache_stopped_nodes'] = not terminate
             with tempfile.NamedTemporaryFile('w',

sky/clouds/oci.py

@@ -31,6 +31,7 @@ from sky import status_lib
 from sky.adaptors import oci as oci_adaptor
 from sky.clouds import service_catalog
 from sky.clouds.utils import oci_utils
+from sky.provision.oci.query_utils import query_helper
 from sky.utils import common_utils
 from sky.utils import resources_utils
 from sky.utils import ux_utils
@@ -60,6 +61,9 @@ class OCI(clouds.Cloud):
                              {resources_utils.DiskTier.ULTRA})
     _BEST_DISK_TIER = resources_utils.DiskTier.HIGH
 
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
     @classmethod
     def _unsupported_features_for_resources(
         cls, resources: 'resources_lib.Resources'
@@ -433,7 +437,7 @@ class OCI(clouds.Cloud):
             return True, None
         except (oci_adaptor.oci.exceptions.ConfigFileNotFound,
                 oci_adaptor.oci.exceptions.InvalidConfig,
-                oci_adaptor.service_exception()) as e:
+                oci_adaptor.oci.exceptions.ServiceError) as e:
             return False, (
                 f'OCI credential is not correctly set. '
                 f'Check the credential file at {conf_file}\n'
@@ -597,25 +601,11 @@ class OCI(clouds.Cloud):
                      region: Optional[str], zone: Optional[str],
                      **kwargs) -> List[status_lib.ClusterStatus]:
         del zone, kwargs  # Unused.
-        # Check the lifecycleState definition from the page
-        # https://docs.oracle.com/en-us/iaas/api/#/en/iaas/latest/Instance/
-        status_map = {
-            'PROVISIONING': status_lib.ClusterStatus.INIT,
-            'STARTING': status_lib.ClusterStatus.INIT,
-            'RUNNING': status_lib.ClusterStatus.UP,
-            'STOPPING': status_lib.ClusterStatus.STOPPED,
-            'STOPPED': status_lib.ClusterStatus.STOPPED,
-            'TERMINATED': None,
-            'TERMINATING': None,
-        }
-
-        # pylint: disable=import-outside-toplevel
-        from sky.skylet.providers.oci.query_helper import oci_query_helper
 
         status_list = []
         try:
-            vms = oci_query_helper.query_instances_by_tags(
-                tag_filters=tag_filters, region=region)
+            vms = query_helper.query_instances_by_tags(tag_filters=tag_filters,
+                                                       region=region)
         except Exception as e:  # pylint: disable=broad-except
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.ClusterStatusFetchingError(
@@ -625,9 +615,9 @@ class OCI(clouds.Cloud):
 
         for node in vms:
             vm_status = node.lifecycle_state
-            if vm_status in status_map:
-                sky_status = status_map[vm_status]
-                if sky_status is not None:
-                    status_list.append(sky_status)
+            sky_status = oci_utils.oci_config.STATE_MAPPING_OCI_TO_SKY.get(
+                vm_status, None)
+            if sky_status is not None:
+                status_list.append(sky_status)
 
         return status_list

sky/clouds/service_catalog/oci_catalog.py

@@ -66,7 +66,7 @@ def _get_df() -> 'pd.DataFrame':
             logger.debug(f'It is OK goes here when testing: {str(e)}')
             subscribed_regions = []
 
-        except oci_adaptor.service_exception() as e:
+        except oci_adaptor.oci.exceptions.ServiceError as e:
             # Should never expect going here. However, we still catch
             # it so that if any OCI call failed, the program can still
             # proceed with try-and-error way.

sky/clouds/utils/oci_utils.py

@@ -5,13 +5,14 @@ History:
  - Hysun He (hysun.he@oracle.com) @ Oct, 2024: Add default image OS
    configuration.
 """
-import logging
 import os
 
+from sky import sky_logging
 from sky import skypilot_config
+from sky import status_lib
 from sky.utils import resources_utils
 
-logger = logging.getLogger(__name__)
+logger = sky_logging.init_logger(__name__)
 
 
 class OCIConfig:
@@ -77,6 +78,19 @@ class OCIConfig:
         resources_utils.DiskTier.HIGH: DISK_TIER_HIGH,
     }
 
+    # Oracle instance's lifecycle state to sky state mapping.
+    # For Oracle VM instance's lifecyle state, please refer to the link:
+    # https://docs.oracle.com/en-us/iaas/api/#/en/iaas/latest/Instance/
+    STATE_MAPPING_OCI_TO_SKY = {
+        'PROVISIONING': status_lib.ClusterStatus.INIT,
+        'STARTING': status_lib.ClusterStatus.INIT,
+        'RUNNING': status_lib.ClusterStatus.UP,
+        'STOPPING': status_lib.ClusterStatus.STOPPED,
+        'STOPPED': status_lib.ClusterStatus.STOPPED,
+        'TERMINATED': None,
+        'TERMINATING': None,
+    }
+
     @classmethod
     def get_compartment(cls, region):
         # Allow task(cluster)-specific compartment/VCN parameters.

sky/dag.py

@@ -56,21 +56,25 @@ class Dag:
         return self.graph
 
     def is_chain(self) -> bool:
-        # NOTE: this method assumes that the graph has no cycle.
-        is_chain = True
-        visited_zero_out_degree = False
-        for node in self.graph.nodes:
-            out_degree = self.graph.out_degree(node)
-            if out_degree > 1:
-                is_chain = False
-                break
-            elif out_degree == 0:
-                if visited_zero_out_degree:
-                    is_chain = False
-                    break
-                else:
-                    visited_zero_out_degree = True
-        return is_chain
+        """Check if the DAG is a linear chain of tasks."""
+
+        nodes = list(self.graph.nodes)
+
+        if len(nodes) == 0:
+            return True
+
+        in_degrees = [self.graph.in_degree(node) for node in nodes]
+        out_degrees = [self.graph.out_degree(node) for node in nodes]
+
+        # Check out-degrees: all <= 1 and exactly one node has out_degree == 0
+        out_degree_condition = (all(degree <= 1 for degree in out_degrees) and
+                                sum(degree == 0 for degree in out_degrees) == 1)
+
+        # Check in-degrees: all <= 1 and exactly one node has in_degree == 0
+        in_degree_condition = (all(degree <= 1 for degree in in_degrees) and
+                               sum(degree == 0 for degree in in_degrees) == 1)
+
+        return out_degree_condition and in_degree_condition
 
 
 class _DagContext(threading.local):

sky/provision/__init__.py

@@ -20,6 +20,7 @@ from sky.provision import fluidstack
 from sky.provision import gcp
 from sky.provision import kubernetes
 from sky.provision import lambda_cloud
+from sky.provision import oci
 from sky.provision import runpod
 from sky.provision import vsphere
 from sky.utils import command_runner

sky/provision/docker_utils.py

@@ -20,7 +20,7 @@ SETUP_ENV_VARS_CMD = (
     '{ if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; } && '
     'printenv | while IFS=\'=\' read -r key value; do echo "export $key=\\\"$value\\\""; done > '  # pylint: disable=line-too-long
     '~/container_env_var.sh && '
-    '$(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh'
+    '$(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh;'
 )
 
 # Docker daemon may not be ready when the machine is firstly started. The error

sky/provision/kubernetes/instance.py

@@ -333,52 +333,37 @@ def _run_function_with_retries(func: Callable,
                 raise
 
 
-def _set_env_vars_in_pods(namespace: str, context: Optional[str],
-                          new_pods: List):
-    """Setting environment variables in pods.
-
-    Once all containers are ready, we can exec into them and set env vars.
-    Kubernetes automatically populates containers with critical
-    environment variables, such as those for discovering services running
-    in the cluster and CUDA/nvidia environment variables. We need to
-    make sure these env vars are available in every task and ssh session.
-    This is needed for GPU support and service discovery.
-    See https://github.com/skypilot-org/skypilot/issues/2287 for
-    more details.
-
-    To do so, we capture env vars from the pod's runtime and write them to
-    /etc/profile.d/, making them available for all users in future
-    shell sessions.
-    """
-    set_k8s_env_var_cmd = docker_utils.SETUP_ENV_VARS_CMD
+def pre_init(namespace: str, context: Optional[str], new_nodes: List) -> None:
+    """Pre-initialization step for SkyPilot pods.
 
-    def _set_env_vars_thread(new_pod):
-        pod_name = new_pod.metadata.name
-        logger.info(f'{"-"*20}Start: Set up env vars in pod {pod_name!r} '
-                    f'{"-"*20}')
-        runner = command_runner.KubernetesCommandRunner(
-            ((namespace, context), pod_name))
+    This step is run in the pod right after it is created and before the
+    SkyPilot runtime is setup.
 
-        def _run_env_vars_cmd():
-            rc, stdout, _ = runner.run(set_k8s_env_var_cmd,
-                                       require_outputs=True,
-                                       stream_logs=False)
-            _raise_command_running_error('set env vars', set_k8s_env_var_cmd,
-                                         pod_name, rc, stdout)
+    This step includes three key steps:
 
-        _run_function_with_retries(_run_env_vars_cmd,
-                                   f'set env vars in pod {pod_name}')
-        logger.info(f'{"-"*20}End: Set up env vars in pod {pod_name!r} '
-                    f'{"-"*20}')
+    1. Privilege check: Checks if the default user has sufficient privilege
+    to set up the kubernetes instance pod.
+    2. SSH setup: Sets up SSH for the pod instance.
+    3. Environment variable setup to populate k8s env vars in the pod.
 
-    subprocess_utils.run_in_parallel(_set_env_vars_thread, new_pods,
-                                     NUM_THREADS)
+    Make sure commands used in these methods are generic and work
+    on most base images. E.g., do not use Python, since that may not
+    be installed by default.
 
+    If you run any apt commands, be sure to check if the lock is available.
+    It is possible the `apt update` run in the pod container args may still
+    be running.
+
+    Args:
+        namespace (str): Kubernetes namespace.
+        context (Optional[str]): Kubernetes context.
+        new_nodes (List): List of new pod instances.
+
+    Raises:
+        config_lib.KubernetesError: If user privileges are insufficient or
+          setup fails.
+    """
 
-def _check_user_privilege(namespace: str, context: Optional[str],
-                          new_nodes: List) -> None:
-    # Checks if the default user has sufficient privilege to set up
-    # the kubernetes instance pod.
     check_k8s_user_sudo_cmd = (
         'if [ $(id -u) -eq 0 ]; then'
         # If user is root, create an alias for sudo used in skypilot setup
@@ -386,56 +371,67 @@ def _check_user_privilege(namespace: str, context: Optional[str],
         'else '
         '  if command -v sudo >/dev/null 2>&1; then '
         '    timeout 2 sudo -l >/dev/null 2>&1 && echo succeed || '
-        f'    ( echo {exceptions.INSUFFICIENT_PRIVILEGES_CODE!r}; ); '
+        f'    ( echo {exceptions.INSUFFICIENT_PRIVILEGES_CODE!r}; '
+        f'      exit {exceptions.INSUFFICIENT_PRIVILEGES_CODE}; ); '
         '  else '
-        f'    ( echo {exceptions.INSUFFICIENT_PRIVILEGES_CODE!r}; ); '
+        f'    ( echo {exceptions.INSUFFICIENT_PRIVILEGES_CODE!r}; '
+        f'      exit {exceptions.INSUFFICIENT_PRIVILEGES_CODE}; ); '
         '  fi; '
-        'fi')
+        'fi;')
+
+    # Kubernetes automatically populates containers with critical
+    # environment variables, such as those for discovering services running
+    # in the cluster and CUDA/nvidia environment variables. We need to
+    # make sure these env vars are available in every task and ssh session.
+    # This is needed for GPU support and service discovery.
+    # See https://github.com/skypilot-org/skypilot/issues/2287 for more details.
+    # To do so, we capture env vars from the pod's runtime and write them to
+    # /etc/profile.d/, making them available for all users in future
+    # shell sessions.
+    set_k8s_env_var_cmd = docker_utils.SETUP_ENV_VARS_CMD
 
-    # This check needs to run on a per-image basis, so running the check on
-    # any one pod is sufficient.
-    new_node = new_nodes[0]
-    pod_name = new_node.metadata.name
+    check_apt_update_complete_cmd = (
+        'echo "Checking if apt update from container init is complete..."; '
+        'timeout_secs=600; '
+        'start_time=$(date +%s); '
+        'while ! grep -q "Fetched" /tmp/apt-update.log 2>/dev/null; do '
+        '  echo "apt update still running. Logs:"; '
+        '  cat /tmp/apt-update.log; '
+        '  current_time=$(date +%s); '
+        '  elapsed=$((current_time - start_time)); '
+        '  if [ $elapsed -ge $timeout_secs ]; then '
+        '    echo "Timed out waiting for apt update"; '
+        '    exit 1; '
+        '  fi; '
+        '  sleep 5; '
+        'done; '
+        'echo "apt update complete."; ')
 
-    runner = command_runner.KubernetesCommandRunner(
-        ((namespace, context), pod_name))
-    logger.info(f'{"-"*20}Start: Check user privilege in pod {pod_name!r} '
-                f'{"-"*20}')
-
-    def _run_privilege_check():
-        rc, stdout, stderr = runner.run(check_k8s_user_sudo_cmd,
-                                        require_outputs=True,
-                                        separate_stderr=True,
-                                        stream_logs=False)
-        _raise_command_running_error('check user privilege',
-                                     check_k8s_user_sudo_cmd, pod_name, rc,
-                                     stdout + stderr)
-        return stdout
-
-    stdout = _run_function_with_retries(
-        _run_privilege_check, f'check user privilege in pod {pod_name!r}')
-
-    if stdout == str(exceptions.INSUFFICIENT_PRIVILEGES_CODE):
-        raise config_lib.KubernetesError(
-            'Insufficient system privileges detected. '
-            'Ensure the default user has root access or '
-            '"sudo" is installed and the user is added to the sudoers '
-            'from the image.')
-    logger.info(f'{"-"*20}End: Check user privilege in pod {pod_name!r} '
-                f'{"-"*20}')
-
-
-def _setup_ssh_in_pods(namespace: str, context: Optional[str],
-                       new_nodes: List) -> None:
-    # Setting up ssh for the pod instance. This is already setup for
-    # the jump pod so it does not need to be run for it.
-    set_k8s_ssh_cmd = (
-        'set -ex; '
+    install_ssh_k8s_cmd = (
         'prefix_cmd() '
         '{ if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; }; '
         'export DEBIAN_FRONTEND=noninteractive;'
-        '$(prefix_cmd) apt-get update;'
-        '$(prefix_cmd) apt install openssh-server rsync -y; '
+        'echo "Installing missing packages..."; '
+        'for i in {1..5}; do '
+        '  output=$($(prefix_cmd) apt install openssh-server rsync -y 2>&1); '
+        '  rc=$?; '
+        '  if [ $rc -eq 0 ]; then '
+        '    break; '
+        '  fi; '
+        '  echo "$output" | grep -qi "could not get lock" || '
+        '  grep -qi "Unable to acquire the dpkg frontend lock"; '
+        '  if [ $? -eq 0 ]; then '
+        '    echo "apt install failed due to lock, retrying. (Attempt $i/5)"; '
+        '    sleep 5; '
+        '  else '
+        '    echo "apt install failed for a non-lock reason: $output"; '
+        '    exit $rc; '
+        '  fi; '
+        'done; '
+        'if [ $rc -ne 0 ]; then '
+        '    echo "apt install failed after 5 attempts due to lock errors."; '
+        '    exit $rc; '
+        'fi; '
         '$(prefix_cmd) mkdir -p /var/run/sshd; '
         '$(prefix_cmd) '
         'sed -i "s/PermitRootLogin prohibit-password/PermitRootLogin yes/" '
@@ -456,24 +452,35 @@ def _setup_ssh_in_pods(namespace: str, context: Optional[str],
         # See https://www.educative.io/answers/error-mesg-ttyname-failed-inappropriate-ioctl-for-device  # pylint: disable=line-too-long
         '$(prefix_cmd) sed -i "s/mesg n/tty -s \\&\\& mesg n/" ~/.profile;')
 
-    def _setup_ssh_thread(new_node):
+    pre_init_cmd = ('set -ex; ' + check_k8s_user_sudo_cmd +
+                    set_k8s_env_var_cmd + check_apt_update_complete_cmd +
+                    install_ssh_k8s_cmd)
+
+    def _pre_init_thread(new_node):
         pod_name = new_node.metadata.name
+        logger.info(f'{"-"*20}Start: Pre-init in pod {pod_name!r} {"-"*20}')
         runner = command_runner.KubernetesCommandRunner(
             ((namespace, context), pod_name))
-        logger.info(f'{"-"*20}Start: Set up SSH in pod {pod_name!r} {"-"*20}')
 
-        def _run_ssh_setup():
-            rc, stdout, _ = runner.run(set_k8s_ssh_cmd,
-                                       require_outputs=True,
-                                       stream_logs=False)
-            _raise_command_running_error('setup ssh', set_k8s_ssh_cmd, pod_name,
-                                         rc, stdout)
+        # Run the combined pre-init command
+        rc, stdout, _ = runner.run(pre_init_cmd,
+                                   require_outputs=True,
+                                   stream_logs=False)
+        if rc == exceptions.INSUFFICIENT_PRIVILEGES_CODE:
+            raise config_lib.KubernetesError(
+                'Insufficient system privileges detected. '
+                'Ensure the default user has root access or '
+                '"sudo" is installed and the user is added to the sudoers '
+                'from the image.')
+
+        op_name = 'pre-init'
+        _raise_command_running_error(op_name, pre_init_cmd, pod_name, rc,
+                                     stdout)
 
-        _run_function_with_retries(_run_ssh_setup,
-                                   f'setup ssh in pod {pod_name!r}')
-        logger.info(f'{"-"*20}End: Set up SSH in pod {pod_name!r} {"-"*20}')
+        logger.info(f'{"-"*20}End: Pre-init in pod {pod_name!r} {"-"*20}')
 
-    subprocess_utils.run_in_parallel(_setup_ssh_thread, new_nodes, NUM_THREADS)
+    # Run pre_init in parallel across all new_nodes
+    subprocess_utils.run_in_parallel(_pre_init_thread, new_nodes, NUM_THREADS)
 
 
 def _label_pod(namespace: str, context: Optional[str], pod_name: str,
@@ -724,13 +731,8 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
                      f'pods: {list(uninitialized_pods.keys())}')
         uninitialized_pods_list = list(uninitialized_pods.values())
 
-        # Setup SSH and environment variables in pods.
-        # Make sure commands used in these methods are generic and work
-        # on most base images. E.g., do not use Python, since that may not
-        # be installed by default.
-        _check_user_privilege(namespace, context, uninitialized_pods_list)
-        _setup_ssh_in_pods(namespace, context, uninitialized_pods_list)
-        _set_env_vars_in_pods(namespace, context, uninitialized_pods_list)
+        # Run pre-init steps in the pod.
+        pre_init(namespace, context, uninitialized_pods_list)
 
         for pod in uninitialized_pods.values():
             _label_pod(namespace,

sky/provision/oci/__init__.py

@@ -0,0 +1,15 @@
+"""OCI provisioner for SkyPilot.
+
+History:
+ - Hysun He (hysun.he@oracle.com) @ Oct.16, 2024: Initial implementation
+"""
+
+from sky.provision.oci.config import bootstrap_instances
+from sky.provision.oci.instance import cleanup_ports
+from sky.provision.oci.instance import get_cluster_info
+from sky.provision.oci.instance import open_ports
+from sky.provision.oci.instance import query_instances
+from sky.provision.oci.instance import run_instances
+from sky.provision.oci.instance import stop_instances
+from sky.provision.oci.instance import terminate_instances
+from sky.provision.oci.instance import wait_instances

sky/provision/oci/config.py

@@ -0,0 +1,51 @@
+"""OCI configuration bootstrapping.
+
+Creates the resource group and deploys the configuration template to OCI for
+a cluster to be launched.
+
+History:
+ - Hysun He (hysun.he@oracle.com) @ Oct.16, 2024: Initial implementation
+"""
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import oci as oci_adaptor
+from sky.clouds.utils import oci_utils
+from sky.provision import common
+from sky.provision.oci.query_utils import query_helper
+
+logger = sky_logging.init_logger(__name__)
+
+
+@common.log_function_start_end
+def bootstrap_instances(
+        region: str, cluster_name_on_cloud: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    """See sky/provision/__init__.py"""
+    # OCI module import and oci client
+    oci_adaptor.get_core_client(region, oci_utils.oci_config.get_profile())
+
+    # Find / create a compartment for creating instances.
+    compartment = query_helper.find_compartment(region)
+
+    # Find the configured VCN, or create a new one.
+    vcn = query_helper.find_create_vcn_subnet(region)
+    if vcn is None:
+        # pylint: disable=line-too-long
+        raise exceptions.ResourcesUnavailableError(
+            'Failed to create a new VCN, possibly you hit the resource limitation.'
+        )
+
+    node_config = config.node_config
+
+    # Subscribe the image if it is from Marketplace listing.
+    query_helper.subscribe_image(
+        compartment_id=compartment,
+        listing_id=node_config['AppCatalogListingId'],
+        resource_version=node_config['ResourceVersion'],
+        region=region,
+    )
+
+    logger.info(f'Using cluster name: {cluster_name_on_cloud}')
+
+    return config