skypilot-nightly 1.0.0.dev20241108__py3-none-any.whl → 1.0.0.dev20241110__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '7bea46813b47a46ce7b00501413cac637ee400a3'
+_SKYPILOT_COMMIT_SHA = 'dddd65187953a5d6b32f762bea78eed1f109ec3c'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20241108'
+__version__ = '1.0.0.dev20241110'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -3711,7 +3711,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                   handle: CloudVmRayResourceHandle,
                   job_id: Optional[int],
                   managed_job_id: Optional[int] = None,
-                  follow: bool = True) -> int:
+                  follow: bool = True,
+                  tail: int = 0) -> int:
         """Tail the logs of a job.
 
         Args:
@@ -3719,10 +3720,13 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             job_id: The job ID to tail the logs of.
             managed_job_id: The managed job ID for display purpose only.
             follow: Whether to follow the logs.
+            tail: The number of lines to display from the end of the
+                log file. If 0, print all lines.
         """
         code = job_lib.JobLibCodeGen.tail_logs(job_id,
                                                managed_job_id=managed_job_id,
-                                               follow=follow)
+                                               follow=follow,
+                                               tail=tail)
         if job_id is None and managed_job_id is None:
             logger.info(
                 'Job ID not provided. Streaming the logs of the latest job.')
@@ -3975,25 +3979,6 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 stdout = ''
                 stderr = str(e)
 
-        # Apr, 2023 by Hysun(hysun.he@oracle.com): Added support for OCI
-        # May, 2023 by Hysun: Allow terminate INIT cluster which may have
-        # some instances provisioning in background but not completed.
-        elif (isinstance(cloud, clouds.OCI) and terminate and
-              prev_cluster_status in (status_lib.ClusterStatus.STOPPED,
-                                      status_lib.ClusterStatus.INIT)):
-            region = config['provider']['region']
-
-            # pylint: disable=import-outside-toplevel
-            from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME
-
-            from sky.skylet.providers.oci.query_helper import oci_query_helper
-
-            # 0: All terminated successfully, failed count otherwise
-            returncode = oci_query_helper.terminate_instances_by_tags(
-                {TAG_RAY_CLUSTER_NAME: cluster_name_on_cloud}, region)
-
-            # To avoid undefined local variables error.
-            stdout = stderr = ''
         else:
             config['provider']['cache_stopped_nodes'] = not terminate
             with tempfile.NamedTemporaryFile('w',

sky/backends/wheel_utils.py

@@ -129,7 +129,11 @@ def _build_sky_wheel() -> pathlib.Path:
 
         wheel_dir = WHEEL_DIR / hash_of_latest_wheel
         wheel_dir.mkdir(parents=True, exist_ok=True)
-        shutil.move(str(wheel_path), wheel_dir)
+        # shutil.move will fail when the file already exists and is being
+        # moved across filesystems.
+        if not os.path.exists(
+                os.path.join(wheel_dir, os.path.basename(wheel_path))):
+            shutil.move(str(wheel_path), wheel_dir)
         return wheel_dir / wheel_path.name
 
 

sky/cli.py

@@ -46,6 +46,7 @@ from rich import progress as rich_progress
 import yaml
 
 import sky
+from sky import admin_policy
 from sky import backends
 from sky import check as sky_check
 from sky import clouds as sky_clouds
@@ -67,6 +68,7 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.skylet import log_lib
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import dag_utils
@@ -582,6 +584,15 @@ def _launch_with_confirm(
             with ux_utils.print_exception_no_traceback():
                 raise RuntimeError(f'{colorama.Fore.YELLOW}{e}'
                                    f'{colorama.Style.RESET_ALL}') from e
+        dag, _ = admin_policy_utils.apply(
+            dag,
+            request_options=admin_policy.RequestOptions(
+                cluster_name=cluster,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                down=down,
+                dryrun=dryrun,
+            ),
+        )
         dag = sky.optimize(dag)
     task = dag.tasks[0]
 
@@ -2011,6 +2022,12 @@ def queue(clusters: List[str], skip_finished: bool, all_users: bool):
     help=('Follow the logs of a job. '
           'If --no-follow is specified, print the log so far and exit. '
           '[default: --follow]'))
+@click.option(
+    '--tail',
+    default=0,
+    type=int,
+    help=('The number of lines to display from the end of the log file. '
+          'Default is 0, which means print all lines.'))
 @click.argument('cluster',
                 required=True,
                 type=str,
@@ -2024,6 +2041,7 @@ def logs(
     sync_down: bool,
     status: bool,  # pylint: disable=redefined-outer-name
     follow: bool,
+    tail: int,
 ):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail the log of a job.
@@ -2090,7 +2108,7 @@ def logs(
                 click.secho(f'Job {id_str}not found', fg='red')
             sys.exit(1)
 
-    core.tail_logs(cluster, job_id, follow)
+    core.tail_logs(cluster, job_id, follow, tail)
 
 
 @cli.command()
@@ -3667,6 +3685,8 @@ def jobs_launch(
 
     click.secho(f'Managed job {dag.name!r} will be launched on (estimated):',
                 fg='cyan')
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     dag = sky.optimize(dag)
 
     if not yes:
@@ -4145,6 +4165,8 @@ def serve_up(
                 fg='cyan')
     with sky.Dag() as dag:
         dag.add(task)
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     sky.optimize(dag)
 
     if not yes:
@@ -4261,6 +4283,8 @@ def serve_update(
                 fg='cyan')
     with sky.Dag() as dag:
         dag.add(task)
+    dag, _ = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     sky.optimize(dag)
 
     if not yes:

sky/clouds/oci.py

@@ -31,6 +31,7 @@ from sky import status_lib
 from sky.adaptors import oci as oci_adaptor
 from sky.clouds import service_catalog
 from sky.clouds.utils import oci_utils
+from sky.provision.oci.query_utils import query_helper
 from sky.utils import common_utils
 from sky.utils import resources_utils
 from sky.utils import ux_utils
@@ -60,6 +61,9 @@ class OCI(clouds.Cloud):
                              {resources_utils.DiskTier.ULTRA})
     _BEST_DISK_TIER = resources_utils.DiskTier.HIGH
 
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
     @classmethod
     def _unsupported_features_for_resources(
         cls, resources: 'resources_lib.Resources'
@@ -433,7 +437,7 @@ class OCI(clouds.Cloud):
             return True, None
         except (oci_adaptor.oci.exceptions.ConfigFileNotFound,
                 oci_adaptor.oci.exceptions.InvalidConfig,
-                oci_adaptor.service_exception()) as e:
+                oci_adaptor.oci.exceptions.ServiceError) as e:
             return False, (
                 f'OCI credential is not correctly set. '
                 f'Check the credential file at {conf_file}\n'
@@ -597,25 +601,11 @@ class OCI(clouds.Cloud):
                      region: Optional[str], zone: Optional[str],
                      **kwargs) -> List[status_lib.ClusterStatus]:
         del zone, kwargs  # Unused.
-        # Check the lifecycleState definition from the page
-        # https://docs.oracle.com/en-us/iaas/api/#/en/iaas/latest/Instance/
-        status_map = {
-            'PROVISIONING': status_lib.ClusterStatus.INIT,
-            'STARTING': status_lib.ClusterStatus.INIT,
-            'RUNNING': status_lib.ClusterStatus.UP,
-            'STOPPING': status_lib.ClusterStatus.STOPPED,
-            'STOPPED': status_lib.ClusterStatus.STOPPED,
-            'TERMINATED': None,
-            'TERMINATING': None,
-        }
-
-        # pylint: disable=import-outside-toplevel
-        from sky.skylet.providers.oci.query_helper import oci_query_helper
 
         status_list = []
         try:
-            vms = oci_query_helper.query_instances_by_tags(
-                tag_filters=tag_filters, region=region)
+            vms = query_helper.query_instances_by_tags(tag_filters=tag_filters,
+                                                       region=region)
         except Exception as e:  # pylint: disable=broad-except
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.ClusterStatusFetchingError(
@@ -625,9 +615,9 @@ class OCI(clouds.Cloud):
 
         for node in vms:
             vm_status = node.lifecycle_state
-            if vm_status in status_map:
-                sky_status = status_map[vm_status]
-                if sky_status is not None:
-                    status_list.append(sky_status)
+            sky_status = oci_utils.oci_config.STATE_MAPPING_OCI_TO_SKY.get(
+                vm_status, None)
+            if sky_status is not None:
+                status_list.append(sky_status)
 
         return status_list

sky/clouds/service_catalog/oci_catalog.py

@@ -66,7 +66,7 @@ def _get_df() -> 'pd.DataFrame':
             logger.debug(f'It is OK goes here when testing: {str(e)}')
             subscribed_regions = []
 
-        except oci_adaptor.service_exception() as e:
+        except oci_adaptor.oci.exceptions.ServiceError as e:
             # Should never expect going here. However, we still catch
             # it so that if any OCI call failed, the program can still
             # proceed with try-and-error way.

sky/clouds/utils/oci_utils.py

@@ -5,13 +5,14 @@ History:
  - Hysun He (hysun.he@oracle.com) @ Oct, 2024: Add default image OS
    configuration.
 """
-import logging
 import os
 
+from sky import sky_logging
 from sky import skypilot_config
+from sky import status_lib
 from sky.utils import resources_utils
 
-logger = logging.getLogger(__name__)
+logger = sky_logging.init_logger(__name__)
 
 
 class OCIConfig:
@@ -77,6 +78,19 @@ class OCIConfig:
         resources_utils.DiskTier.HIGH: DISK_TIER_HIGH,
     }
 
+    # Oracle instance's lifecycle state to sky state mapping.
+    # For Oracle VM instance's lifecyle state, please refer to the link:
+    # https://docs.oracle.com/en-us/iaas/api/#/en/iaas/latest/Instance/
+    STATE_MAPPING_OCI_TO_SKY = {
+        'PROVISIONING': status_lib.ClusterStatus.INIT,
+        'STARTING': status_lib.ClusterStatus.INIT,
+        'RUNNING': status_lib.ClusterStatus.UP,
+        'STOPPING': status_lib.ClusterStatus.STOPPED,
+        'STOPPED': status_lib.ClusterStatus.STOPPED,
+        'TERMINATED': None,
+        'TERMINATING': None,
+    }
+
     @classmethod
     def get_compartment(cls, region):
         # Allow task(cluster)-specific compartment/VCN parameters.

sky/core.py

@@ -742,7 +742,8 @@ def cancel(
 @usage_lib.entrypoint
 def tail_logs(cluster_name: str,
               job_id: Optional[int],
-              follow: bool = True) -> None:
+              follow: bool = True,
+              tail: int = 0) -> None:
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail the logs of a job.
 
@@ -775,7 +776,7 @@ def tail_logs(cluster_name: str,
         f'{colorama.Style.RESET_ALL}')
 
     usage_lib.record_cluster_name_for_current_operation(cluster_name)
-    backend.tail_logs(handle, job_id, follow=follow)
+    backend.tail_logs(handle, job_id, follow=follow, tail=tail)
 
 
 @usage_lib.entrypoint

sky/dag.py

@@ -23,6 +23,7 @@ class Dag:
 
         self.graph = nx.DiGraph()
         self.name: Optional[str] = None
+        self.policy_applied: bool = False
 
     def add(self, task: 'task.Task') -> None:
         self.graph.add_node(task)
@@ -55,21 +56,25 @@ class Dag:
         return self.graph
 
     def is_chain(self) -> bool:
-        # NOTE: this method assumes that the graph has no cycle.
-        is_chain = True
-        visited_zero_out_degree = False
-        for node in self.graph.nodes:
-            out_degree = self.graph.out_degree(node)
-            if out_degree > 1:
-                is_chain = False
-                break
-            elif out_degree == 0:
-                if visited_zero_out_degree:
-                    is_chain = False
-                    break
-                else:
-                    visited_zero_out_degree = True
-        return is_chain
+        """Check if the DAG is a linear chain of tasks."""
+
+        nodes = list(self.graph.nodes)
+
+        if len(nodes) == 0:
+            return True
+
+        in_degrees = [self.graph.in_degree(node) for node in nodes]
+        out_degrees = [self.graph.out_degree(node) for node in nodes]
+
+        # Check out-degrees: all <= 1 and exactly one node has out_degree == 0
+        out_degree_condition = (all(degree <= 1 for degree in out_degrees) and
+                                sum(degree == 0 for degree in out_degrees) == 1)
+
+        # Check in-degrees: all <= 1 and exactly one node has in_degree == 0
+        in_degree_condition = (all(degree <= 1 for degree in in_degrees) and
+                               sum(degree == 0 for degree in in_degrees) == 1)
+
+        return out_degree_condition and in_degree_condition
 
 
 class _DagContext(threading.local):

sky/data/mounting_utils.py

@@ -276,23 +276,11 @@ def get_mounting_command(
     script = get_mounting_script(mount_path, mount_cmd, install_cmd,
                                  version_check_cmd)
 
-    # TODO(romilb): Get direct bash script to work like so:
-    # command = f'bash <<-\EOL' \
-    #           f'{script}' \
-    #           'EOL'
-
-    # TODO(romilb): This heredoc should have EOF after script, but it
-    #  fails with sky's ssh pipeline. Instead, we don't use EOF and use )
-    #  as the end of heredoc. This raises a warning (here-document delimited
-    #  by end-of-file) that can be safely ignored.
-
     # While these commands are run sequentially for each storage object,
     # we add random int to be on the safer side and avoid collisions.
     script_path = f'~/.sky/mount_{random.randint(0, 1000000)}.sh'
-    first_line = r'(cat <<-\EOF > {}'.format(script_path)
-    command = (f'{first_line}'
-               f'{script}'
-               f') && chmod +x {script_path}'
-               f' && bash {script_path}'
-               f' && rm {script_path}')
+    command = (f'echo {shlex.quote(script)} > {script_path} && '
+               f'chmod +x {script_path} && '
+               f'bash {script_path} && '
+               f'rm {script_path}')
     return command

sky/exceptions.py

@@ -3,6 +3,8 @@ import enum
 import typing
 from typing import List, Optional, Sequence
 
+from sky.utils import env_options
+
 if typing.TYPE_CHECKING:
     from sky import status_lib
     from sky.backends import backend
@@ -104,7 +106,8 @@ class CommandError(Exception):
         if not command:
             message = error_msg
         else:
-            if len(command) > 100:
+            if (len(command) > 100 and
+                    not env_options.Options.SHOW_DEBUG_INFO.get()):
                 # Chunck the command to avoid overflow.
                 command = command[:100] + '...'
             message = (f'Command {command} failed with return code '

sky/execution.py

@@ -160,14 +160,16 @@ def _execute(
     """
 
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
-    dag, _ = admin_policy_utils.apply(
-        dag,
-        request_options=admin_policy.RequestOptions(
-            cluster_name=cluster_name,
-            idle_minutes_to_autostop=idle_minutes_to_autostop,
-            down=down,
-            dryrun=dryrun,
-        ))
+    if not dag.policy_applied:
+        dag, _ = admin_policy_utils.apply(
+            dag,
+            request_options=admin_policy.RequestOptions(
+                cluster_name=cluster_name,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                down=down,
+                dryrun=dryrun,
+            ),
+        )
     assert len(dag) == 1, f'We support 1 task for now. {dag}'
     task = dag.tasks[0]
 

sky/jobs/core.py

@@ -59,8 +59,10 @@ def launch(
     """
     entrypoint = task
     dag_uuid = str(uuid.uuid4().hex[:4])
-
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
+    # Always apply the policy again here, even though it might have been applied
+    # in the CLI. This is to ensure that we apply the policy to the final DAG
+    # and get the mutated config.
     dag, mutated_user_config = admin_policy_utils.apply(
         dag, use_mutated_config_in_current_request=False)
     if not dag.is_chain():

sky/provision/__init__.py

@@ -20,6 +20,7 @@ from sky.provision import fluidstack
 from sky.provision import gcp
 from sky.provision import kubernetes
 from sky.provision import lambda_cloud
+from sky.provision import oci
 from sky.provision import runpod
 from sky.provision import vsphere
 from sky.utils import command_runner

sky/provision/aws/config.py

@@ -42,8 +42,9 @@ def _skypilot_log_error_and_exit_for_failover(error: str) -> None:
     Mainly used for handling VPC/subnet errors before nodes are launched.
     """
     # NOTE: keep. The backend looks for this to know no nodes are launched.
-    prefix = 'SKYPILOT_ERROR_NO_NODES_LAUNCHED: '
-    raise RuntimeError(prefix + error)
+    full_error = f'SKYPILOT_ERROR_NO_NODES_LAUNCHED: {error}'
+    logger.error(full_error)
+    raise RuntimeError(full_error)
 
 
 def bootstrap_instances(
@@ -222,10 +223,27 @@ def _configure_iam_role(iam) -> Dict[str, Any]:
 
 
 @functools.lru_cache(maxsize=128)  # Keep bounded.
-def _get_route_tables(ec2, vpc_id: Optional[str], main: bool) -> List[Any]:
+def _get_route_tables(ec2, vpc_id: Optional[str], region: str,
+                      main: bool) -> List[Any]:
+    """Get route tables associated with a VPC and region
+
+    Args:
+        ec2: ec2 resource object
+        vpc_id: vpc_id is optional, if not provided, all route tables in the
+                region will be returned
+        region: region is mandatory to allow the lru cache
+                   to return the corect results
+        main: if True, only main route tables will be returned otherwise
+                only non-main route tables will be returned
+
+    Returns:
+        A list of route tables associated with the options VPC and region
+    """
     filters = [{'Name': 'association.main', 'Values': [str(main).lower()]}]
     if vpc_id is not None:
         filters.append({'Name': 'vpc-id', 'Values': [vpc_id]})
+    logger.debug(
+        f'Getting route tables with filters: {filters} in region: {region}')
     return ec2.meta.client.describe_route_tables(Filters=filters).get(
         'RouteTables', [])
 
@@ -238,7 +256,8 @@ def _is_subnet_public(ec2, subnet_id, vpc_id: Optional[str]) -> bool:
     https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html
     """
     # Get the route tables associated with the subnet
-    all_route_tables = _get_route_tables(ec2, vpc_id, main=False)
+    region = ec2.meta.client.meta.region_name
+    all_route_tables = _get_route_tables(ec2, vpc_id, region, main=False)
     route_tables = [
         rt for rt in all_route_tables
         # An RT can be associated with multiple subnets, i.e.,
@@ -267,7 +286,8 @@ def _is_subnet_public(ec2, subnet_id, vpc_id: Optional[str]) -> bool:
     # subnets. Since the associations are implicit, the filter above won't find
     # any. Check there exists a main route table with routes pointing to an IGW.
     logger.debug('Checking main route table')
-    main_route_tables = _get_route_tables(ec2, vpc_id, main=True)
+    region = ec2.meta.client.meta.region_name
+    main_route_tables = _get_route_tables(ec2, vpc_id, region, main=True)
     return _has_igw_route(main_route_tables)
 
 

sky/provision/oci/__init__.py

@@ -0,0 +1,15 @@
+"""OCI provisioner for SkyPilot.
+
+History:
+ - Hysun He (hysun.he@oracle.com) @ Oct.16, 2024: Initial implementation
+"""
+
+from sky.provision.oci.config import bootstrap_instances
+from sky.provision.oci.instance import cleanup_ports
+from sky.provision.oci.instance import get_cluster_info
+from sky.provision.oci.instance import open_ports
+from sky.provision.oci.instance import query_instances
+from sky.provision.oci.instance import run_instances
+from sky.provision.oci.instance import stop_instances
+from sky.provision.oci.instance import terminate_instances
+from sky.provision.oci.instance import wait_instances

sky/provision/oci/config.py

@@ -0,0 +1,51 @@
+"""OCI configuration bootstrapping.
+
+Creates the resource group and deploys the configuration template to OCI for
+a cluster to be launched.
+
+History:
+ - Hysun He (hysun.he@oracle.com) @ Oct.16, 2024: Initial implementation
+"""
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import oci as oci_adaptor
+from sky.clouds.utils import oci_utils
+from sky.provision import common
+from sky.provision.oci.query_utils import query_helper
+
+logger = sky_logging.init_logger(__name__)
+
+
+@common.log_function_start_end
+def bootstrap_instances(
+        region: str, cluster_name_on_cloud: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    """See sky/provision/__init__.py"""
+    # OCI module import and oci client
+    oci_adaptor.get_core_client(region, oci_utils.oci_config.get_profile())
+
+    # Find / create a compartment for creating instances.
+    compartment = query_helper.find_compartment(region)
+
+    # Find the configured VCN, or create a new one.
+    vcn = query_helper.find_create_vcn_subnet(region)
+    if vcn is None:
+        # pylint: disable=line-too-long
+        raise exceptions.ResourcesUnavailableError(
+            'Failed to create a new VCN, possibly you hit the resource limitation.'
+        )
+
+    node_config = config.node_config
+
+    # Subscribe the image if it is from Marketplace listing.
+    query_helper.subscribe_image(
+        compartment_id=compartment,
+        listing_id=node_config['AppCatalogListingId'],
+        resource_version=node_config['ResourceVersion'],
+        region=region,
+    )
+
+    logger.info(f'Using cluster name: {cluster_name_on_cloud}')
+
+    return config