oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/jobs/builders/infrastructure/dsc_job.py

@@ -0,0 +1,1894 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from __future__ import annotations
+
+import datetime
+import inspect
+import logging
+import os
+import re
+import time
+import traceback
+import uuid
+from io import DEFAULT_BUFFER_SIZE
+from string import Template
+from typing import Any, Dict, List, Optional, Union
+
+import fsspec
+import oci
+import oci.data_science
+import oci.util as oci_util
+from oci.data_science.models import JobInfrastructureConfigurationDetails
+from oci.exceptions import ServiceError
+import yaml
+from ads.common import utils
+from ads.common.oci_datascience import DSCNotebookSession, OCIDataScienceMixin
+from ads.common.oci_logging import OCILog
+from ads.common.oci_resource import ResourceNotFoundError
+from ads.jobs.builders.infrastructure.base import Infrastructure, RunInstance
+from ads.jobs.builders.infrastructure.dsc_job_runtime import (
+    ContainerRuntimeHandler,
+    DataScienceJobRuntimeManager,
+)
+from ads.jobs.builders.infrastructure.utils import get_value
+from ads.jobs.builders.runtimes.artifact import Artifact
+from ads.jobs.builders.runtimes.container_runtime import ContainerRuntime
+from ads.jobs.builders.runtimes.python_runtime import GitPythonRuntime
+
+from ads.common.dsc_file_system import (
+    OCIFileStorage,
+    DSCFileSystemManager,
+    OCIObjectStorage,
+)
+from ads.common.decorator.utils import class_or_instance_method
+
+logger = logging.getLogger(__name__)
+
+SLEEP_INTERVAL = 3
+WAIT_SECONDS_AFTER_FINISHED = 90
+MAXIMUM_MOUNT_COUNT = 5
+FILE_STORAGE_TYPE = "FILE_STORAGE"
+OBJECT_STORAGE_TYPE = "OBJECT_STORAGE"
+
+
+class DSCJob(OCIDataScienceMixin, oci.data_science.models.Job):
+    """Represents an OCI Data Science Job
+    This class contains all attributes of the oci.data_science.models.Job.
+    The main purpose of this class is to link the oci.data_science.models.Job model and the related client methods.
+    Mainly, linking the Job model (payload) to Create/Update/Get/List/Delete methods.
+
+    A DSCJob can be initialized by unpacking a the properties stored in a dictionary (payload):
+
+    .. code-block:: python
+
+        job_properties = {
+            "display_name": "my_job",
+            "job_infrastructure_configuration_details": {"shape_name": "VM.MY_SHAPE"}
+        }
+        job = DSCJob(**job_properties)
+
+    The properties can also be OCI REST API payload, in which the keys are in camel format.
+
+    .. code-block:: python
+
+        job_payload = {
+            "projectId": "<project_ocid>",
+            "compartmentId": "<compartment_ocid>",
+            "displayName": "<job_name>",
+            "jobConfigurationDetails": {
+                "jobType": "DEFAULT",
+                "commandLineArguments": "pos_arg1 pos_arg2 --key1 val1 --key2 val2",
+                "environmentVariables": {
+                    "KEY1": "VALUE1",
+                    "KEY2": "VALUE2",
+                    # User specifies conda env via env var
+                    "CONDA_ENV_TYPE" : "service",
+                    "CONDA_ENV_SLUG" : "mlcpuv1"
+                }
+            },
+            "jobInfrastructureConfigurationDetails": {
+                "jobInfrastructureType": "STANDALONE",
+                "shapeName": "VM.Standard.E3.Flex",
+                "jobShapeConfigDetails": {
+                    "memoryInGBs": 16,
+                    "ocpus": 1
+                },
+                "blockStorageSizeInGBs": "100",
+                "subnetId": "<subnet_ocid>"
+            }
+        }
+        job = DSCJob(**job_payload)
+    """
+
+    DEFAULT_INFRA_TYPE = (
+        JobInfrastructureConfigurationDetails.JOB_INFRASTRUCTURE_TYPE_ME_STANDALONE
+    )
+
+    CONST_DEFAULT_BLOCK_STORAGE_SIZE = 50
+
+    def __init__(self, artifact: Union[str, Artifact] = None, **kwargs) -> None:
+        """Initialize a DSCJob object.
+
+        Parameters
+        ----------
+        artifact: str or Artifact
+            Job artifact, which can be a path or an Artifact object. Defaults to None.
+        kwargs:
+            Same as kwargs in oci.data_science.models.Job.
+            Keyword arguments are passed into OCI Job model to initialize the properties.
+
+        """
+        self._artifact = artifact
+
+        super().__init__(**kwargs)
+        if not self.job_configuration_details:
+            self.job_configuration_details = {
+                "jobType": "DEFAULT",
+            }
+        if not self.job_infrastructure_configuration_details:
+            self.job_infrastructure_configuration_details = {}
+
+    @property
+    def artifact(self) -> Union[str, Artifact]:
+        """Job artifact.
+
+        Returns
+        -------
+        str or Artifact
+            When creating a job, this be a path or an Artifact object.
+            When loading the job from OCI, this will be the filename of the job artifact.
+        """
+        if self.id and self._artifact is None:
+            try:
+                res = self.client.head_job_artifact(self.id)
+                content = res.headers.get("content-disposition")
+                if content and "filename=" in content:
+                    self._artifact = content.split("filename=", 1)[-1]
+            except ServiceError:
+                self._artifact = ""
+        return self._artifact
+
+    @artifact.setter
+    def artifact(self, artifact: Union[str, Artifact]):
+        """Sets the job artifact."""
+        self._artifact = artifact
+
+    def _load_infra_from_notebook(self, nb_config):
+        """Loads the infrastructure configuration from notebook configuration."""
+        infra = self.job_infrastructure_configuration_details
+        nb_shape_config_details = oci_util.to_dict(
+            getattr(nb_config, "notebook_session_shape_config_details", None) or {}
+        )
+        if isinstance(infra, dict):
+            shape_name = infra.get("shapeName", nb_config.shape)
+
+            # Ignore notebook shape config details if shape names do not match.
+            if shape_name != nb_config.shape:
+                nb_shape_config_details = {}
+
+            infra_type = infra.get("jobInfrastructureType")
+            block_storage = infra.get(
+                "blockStorageSizeInGBs", nb_config.block_storage_size_in_gbs
+            )
+            subnet_id = infra.get(
+                "subnetId",
+                (
+                    nb_config.subnet_id
+                    if infra_type
+                    != JobInfrastructureConfigurationDetails.JOB_INFRASTRUCTURE_TYPE_ME_STANDALONE
+                    else None
+                ),
+            )
+            job_shape_config_details = infra.get("jobShapeConfigDetails", {})
+            memory_in_gbs = job_shape_config_details.get(
+                "memoryInGBs", nb_shape_config_details.get("memory_in_gbs")
+            )
+            ocpus = job_shape_config_details.get(
+                "ocpus", nb_shape_config_details.get("ocpus")
+            )
+        else:
+            shape_name = (
+                infra.shape_name
+                if getattr(infra, "shape_name", None)
+                else nb_config.shape
+            )
+            # Ignore notebook shape config details if shape names do not match.
+            if shape_name != nb_config.shape:
+                nb_shape_config_details = {}
+
+            infra_type = getattr(infra, "job_infrastructure_type", None)
+
+            block_storage = (
+                infra.block_storage_size_in_gbs
+                if getattr(infra, "block_storage_size_in_gbs", None)
+                else nb_config.block_storage_size_in_gbs
+            )
+            subnet_id = (
+                infra.subnet_id
+                if getattr(infra, "subnet_id", None)
+                else (
+                    nb_config.subnet_id
+                    if infra_type
+                    != JobInfrastructureConfigurationDetails.JOB_INFRASTRUCTURE_TYPE_ME_STANDALONE
+                    else None
+                )
+            )
+            job_shape_config_details = oci_util.to_dict(
+                getattr(infra, "job_shape_config_details", {}) or {}
+            )
+            memory_in_gbs = job_shape_config_details.get(
+                "memory_in_gbs", nb_shape_config_details.get("memory_in_gbs")
+            )
+            ocpus = job_shape_config_details.get(
+                "ocpus", nb_shape_config_details.get("ocpus")
+            )
+
+        self.job_infrastructure_configuration_details = {
+            "jobInfrastructureType": infra_type,
+            "shapeName": shape_name,
+            "blockStorageSizeInGBs": block_storage,
+        }
+        # ADS does not provide explicit API for setting infrastructure type.
+        # If subnet is configured, the type will be set to STANDALONE,
+        # otherwise ME_STANDALONE
+        if subnet_id:
+            self.job_infrastructure_configuration_details.update(
+                {
+                    "subnetId": subnet_id,
+                    "jobInfrastructureType": JobInfrastructureConfigurationDetails.JOB_INFRASTRUCTURE_TYPE_STANDALONE,
+                }
+            )
+        else:
+            self.job_infrastructure_configuration_details.update(
+                {
+                    "jobInfrastructureType": self.DEFAULT_INFRA_TYPE,
+                }
+            )
+
+        # Specify shape config details
+        if memory_in_gbs or ocpus:
+            self.job_infrastructure_configuration_details.update(
+                {
+                    "jobShapeConfigDetails": {
+                        "memoryInGBs": memory_in_gbs,
+                        "ocpus": ocpus,
+                    }
+                }
+            )
+
+    def load_properties_from_env(self) -> None:
+        """Loads default properties from the environment"""
+        if "NB_SESSION_OCID" in os.environ:
+            try:
+                nb_session = DSCNotebookSession.from_ocid(os.environ["NB_SESSION_OCID"])
+            except Exception:
+                logger.debug("Failed to load config from notebook.")
+                logger.debug(traceback.format_exc())
+                # If there is an error loading the notebook infra configurations.
+                # Ignore it by setting nb_session to None
+                # This will skip loading the default configure.
+                nb_session = None
+            if nb_session:
+                nb_config = getattr(
+                    nb_session, "notebook_session_config_details", None
+                ) or getattr(nb_session, "notebook_session_configuration_details", None)
+
+                if nb_config:
+                    self._load_infra_from_notebook(nb_config)
+                if self.project_id is None:
+                    self.project_id = nb_session.project_id
+        super().load_properties_from_env()
+
+    def load_defaults(self) -> DSCJob:
+        self.load_properties_from_env()
+        if not self.job_infrastructure_configuration_details:
+            self.job_infrastructure_configuration_details = {}
+        # Convert the dict to JobInfrastructureConfigurationDetails object
+        if isinstance(self.job_infrastructure_configuration_details, dict):
+            # Default networking
+            if not self.job_infrastructure_configuration_details.get(
+                "jobInfrastructureType"
+            ):
+                self.job_infrastructure_configuration_details[
+                    "jobInfrastructureType"
+                ] = self.DEFAULT_INFRA_TYPE
+            self.job_infrastructure_configuration_details = self.deserialize(
+                self.job_infrastructure_configuration_details,
+                JobInfrastructureConfigurationDetails.__name__,
+            )
+
+        # Default block storage size
+        if not self.job_infrastructure_configuration_details.block_storage_size_in_gbs:
+            self.job_infrastructure_configuration_details.block_storage_size_in_gbs = (
+                self.CONST_DEFAULT_BLOCK_STORAGE_SIZE
+            )
+        return self
+
+    def _create_with_oci_api(self) -> None:
+        oci_model = self.to_oci_model(oci.data_science.models.CreateJobDetails)
+        logger.debug(oci_model)
+        res = self.client.create_job(oci_model)
+        self.update_from_oci_model(res.data)
+        if not self.artifact:
+            return
+        try:
+            if issubclass(self.artifact.__class__, Artifact):
+                with self.artifact as artifact:
+                    self.upload_artifact(artifact.path)
+            else:
+                self.upload_artifact()
+        except Exception as ex:
+            # Delete the job if upload artifact is failed.
+            self.delete()
+            raise ex
+
+    def create(self) -> DSCJob:
+        """Create the job on OCI Data Science platform
+
+        Returns
+        -------
+        DSCJob
+            The DSCJob instance (self), which allows chaining additional method.
+
+        """
+        if not self.display_name:
+            if self.artifact:
+                timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H:%M.%S")
+                self.display_name = (
+                    os.path.basename(str(self.artifact)).split(".")[0] + f"-{timestamp}"
+                )
+            else:
+                # Set default display_name if not specified - randomly generated easy to remember name generated
+                self.display_name = utils.get_random_name_for_resource()
+        try:
+            self.load_defaults()
+        except Exception:
+            logger.exception("Failed to load default properties.")
+        # Check compartment ID and project ID before calling the OCI API
+        if not self.compartment_id:
+            raise ValueError("Specify compartment ID for data science job.")
+        if not self.project_id:
+            raise ValueError("Specify project ID for data science job.")
+        self._create_with_oci_api()
+        return self
+
+    def update(self) -> DSCJob:
+        """Updates the Data Science Job."""
+        raise NotImplementedError("Updating Job is not supported at the moment.")
+
+    def delete(self, force_delete: bool = False) -> DSCJob:
+        """Deletes the job and the corresponding job runs.
+
+        Parameters
+        ----------
+        force_delete : bool, optional, defaults to False
+            the deletion fails when associated job runs are in progress, but if force_delete to true, then
+            the job run will be canceled, then it will be deleted. In this case, delete job has to wait till
+            job has been canceled.
+
+        Returns
+        -------
+        DSCJob
+            The DSCJob instance (self), which allows chaining additional method.
+
+        """
+        runs = self.run_list()
+        for run in runs:
+            if force_delete and run.lifecycle_state in [
+                DataScienceJobRun.LIFECYCLE_STATE_ACCEPTED,
+                DataScienceJobRun.LIFECYCLE_STATE_IN_PROGRESS,
+                DataScienceJobRun.LIFECYCLE_STATE_NEEDS_ATTENTION,
+            ]:
+                run.cancel(wait_for_completion=True)
+            run.delete()
+        self.client.delete_job(self.id)
+        return self
+
+    def upload_artifact(self, artifact_path: str = None) -> DSCJob:
+        """Uploads the job artifact to OCI
+
+        Parameters
+        ----------
+        artifact_path : str, optional
+            Local path to the job artifact file to be uploaded, by default None.
+            If artifact_path is None, the path in self.artifact will be used.
+
+        Returns
+        -------
+        DSCJob
+            The DSCJob instance (self), which allows chaining additional method.
+
+        """
+        if not artifact_path:
+            artifact_path = self.artifact
+        with fsspec.open(artifact_path, "rb") as f:
+            self.client.create_job_artifact(
+                self.id,
+                f,
+                content_disposition=f"attachment; filename={os.path.basename(artifact_path)}",
+            )
+        return self
+
+    def download_artifact(self, artifact_path: str) -> DSCJob:
+        """Downloads the artifact from OCI
+
+        Parameters
+        ----------
+        artifact_path : str
+            Local path to store the job artifact.
+
+        Returns
+        -------
+        DSCJob
+            The DSCJob instance (self), which allows chaining additional method.
+
+        """
+        res = self.client.get_job_artifact_content(self.id)
+        with open(artifact_path, "wb") as f:
+            for chunk in res.data.iter_content(chunk_size=DEFAULT_BUFFER_SIZE * 16):
+                f.write(chunk)
+        return self
+
+    def run_list(self, **kwargs) -> list[DataScienceJobRun]:
+        """Lists the runs of this job.
+
+        Parameters
+        ----------
+        **kwargs :
+            Keyword arguments to te passed into the OCI list_job_runs() for filtering the job runs.
+
+        Returns
+        -------
+        list
+            A list of DSCJobRun objects
+
+        """
+        items = oci.pagination.list_call_get_all_results(
+            self.client.list_job_runs, self.compartment_id, job_id=self.id, **kwargs
+        ).data
+        return [DataScienceJobRun(**self.auth).from_oci_model(item) for item in items]
+
+    def run(self, **kwargs) -> DataScienceJobRun:
+        """Runs the job
+
+        Parameters
+        ----------
+        **kwargs :
+            Keyword arguments for initializing a Data Science Job Run.
+            The keys can be any keys in supported by OCI JobConfigurationDetails, OcirContainerJobEnvironmentConfigurationDetails and JobRun, including:
+            * hyperparameter_values: dict(str, str)
+            * environment_variables: dict(str, str)
+            * command_line_arguments: str
+            * maximum_runtime_in_minutes: int
+            * display_name: str
+            * freeform_tags: dict(str, str)
+            * defined_tags: dict(str, dict(str, object))
+            * image: str
+            * cmd: list[str]
+            * entrypoint: list[str]
+            * image_digest: str
+            * image_signature_id: str
+
+        If display_name is not specified, it will be generated as "<JOB_NAME>-run-<TIMESTAMP>".
+
+        Returns
+        -------
+        DSCJobRun
+            An instance of DSCJobRun, which can be used to monitor the job run.
+
+        """
+        if not self.id:
+            self.create()
+
+        config_swagger_types = (
+            oci.data_science.models.DefaultJobConfigurationDetails().swagger_types.keys()
+        )
+        env_config_swagger_types = {}
+        if hasattr(
+            oci.data_science.models, "OcirContainerJobEnvironmentConfigurationDetails"
+        ):
+            env_config_swagger_types = (
+                oci.data_science.models.OcirContainerJobEnvironmentConfigurationDetails().swagger_types.keys()
+            )
+        config_kwargs = {}
+        env_config_kwargs = {}
+        keys = list(kwargs.keys())
+        for key in keys:
+            if key in config_swagger_types:
+                config_kwargs[key] = kwargs.pop(key)
+            elif key in env_config_swagger_types:
+                value = kwargs.pop(key)
+                if key in [
+                    ContainerRuntime.CONST_CMD,
+                    ContainerRuntime.CONST_ENTRYPOINT,
+                ] and isinstance(value, str):
+                    value = ContainerRuntimeHandler.split_args(value)
+                env_config_kwargs[key] = value
+
+        # remove timestamp from the job name (added in default names, when display_name not specified by user)
+        if self.display_name:
+            try:
+                datetime.datetime.strptime(self.display_name[-19:], "%Y-%m-%d-%H:%M.%S")
+                self.display_name = self.display_name[:-20]
+            except ValueError:
+                pass
+
+        job_attrs = dict(
+            project_id=self.project_id,
+            display_name=self.display_name
+            + "-run-"
+            + datetime.datetime.now().strftime("%Y-%m-%d-%H:%M.%S"),
+            job_id=self.id,
+            compartment_id=self.compartment_id,
+        )
+
+        for key, val in job_attrs.items():
+            if not kwargs.get(key):
+                kwargs[key] = val
+
+        if config_kwargs:
+            config_kwargs["jobType"] = "DEFAULT"
+            config_override = kwargs.get("job_configuration_override_details", {})
+            config_override.update(config_kwargs)
+            kwargs["job_configuration_override_details"] = config_override
+
+        if env_config_kwargs:
+            env_config_kwargs["jobEnvironmentType"] = "OCIR_CONTAINER"
+            env_config_override = kwargs.get(
+                "job_environment_configuration_override_details", {}
+            )
+            env_config_override.update(env_config_kwargs)
+            kwargs["job_environment_configuration_override_details"] = (
+                env_config_override
+            )
+
+        wait = kwargs.pop("wait", False)
+        run = DataScienceJobRun(**kwargs, **self.auth).create()
+        if wait:
+            return run.watch()
+        return run
+
+
+class DataScienceJobRun(
+    OCIDataScienceMixin, oci.data_science.models.JobRun, RunInstance
+):
+    """Represents a Data Science Job run"""
+
+    _DETAILS_LINK = (
+        "https://console.{region}.oraclecloud.com/data-science/job-runs/{id}"
+    )
+
+    TERMINAL_STATES = [
+        oci.data_science.models.JobRun.LIFECYCLE_STATE_SUCCEEDED,
+        oci.data_science.models.JobRun.LIFECYCLE_STATE_FAILED,
+        oci.data_science.models.JobRun.LIFECYCLE_STATE_CANCELED,
+        oci.data_science.models.JobRun.LIFECYCLE_STATE_DELETED,
+    ]
+
+    def create(self) -> DataScienceJobRun:
+        """Creates a job run"""
+        self.load_properties_from_env()
+        res = self.client.create_job_run(
+            self.to_oci_model(oci.data_science.models.CreateJobRunDetails)
+        )
+        self.update_from_oci_model(res.data)
+        return self
+
+    @property
+    def status(self) -> str:
+        """Lifecycle status
+
+        Returns
+        -------
+        str
+            Status in a string.
+        """
+        return self.lifecycle_state
+
+    @property
+    def log_id(self) -> str:
+        """The log ID from OCI logging service containing the logs from the job run."""
+        if not self.log_details:
+            return None
+        return self.log_details.log_id
+
+    @property
+    def log_group_id(self) -> str:
+        """The log group ID from OCI logging service containing the logs from the job run."""
+        if not self.log_details:
+            return None
+        return self.log_details.log_group_id
+
+    @property
+    def logging(self) -> OCILog:
+        """The OCILog object containing the logs from the job run"""
+        if not self.log_id:
+            raise ValueError("Log OCID is not specified for this job run.")
+        # Specifying log group ID when initializing OCILog can reduce the number of API calls.
+        auth = self.auth
+        if "client_kwargs" in auth and isinstance(auth["client_kwargs"], dict):
+            auth["client_kwargs"].pop("service_endpoint", None)
+        return OCILog(
+            id=self.log_id, log_group_id=self.log_details.log_group_id, **auth
+        )
+
+    @property
+    def exit_code(self):
+        """The exit code of the job run from the lifecycle details.
+        Note that,
+        None will be returned if the job run is not finished or failed without exit code.
+        0 will be returned if job run succeeded.
+        """
+        if self.lifecycle_state == self.LIFECYCLE_STATE_SUCCEEDED:
+            return 0
+        if not self.lifecycle_details:
+            return None
+        match = re.search(r"exit code (\d+)", self.lifecycle_details)
+        if not match:
+            return None
+        try:
+            return int(match.group(1))
+        except Exception:
+            return None
+
+    @staticmethod
+    def _format_log(message: str, date_time: datetime.datetime) -> dict:
+        """Formats a message as log record with datetime.
+        This is used to add additional logs to show job run status change.
+
+        Parameters
+        ----------
+        message : str
+            Log message.
+        date_time : datetime or str
+            Timestamp for the message
+
+        Returns
+        -------
+        dict
+            log record as a dictionary, including id, time and message as keys.
+        """
+        if isinstance(date_time, datetime.datetime):
+            date_time = date_time.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        return {
+            "id": str(uuid.uuid4()),
+            "message": message,
+            "time": date_time,
+        }
+
+    def logs(self, limit: int = None) -> list:
+        """Gets the logs of the job run.
+
+        Parameters
+        ----------
+        limit : int, optional
+            Limit the number of logs to be returned.
+            Defaults to None. All logs will be returned.
+
+        Returns
+        -------
+        list
+            A list of log records. Each log record is a dictionary with the following keys: id, time, message.
+        """
+        if self.time_accepted:
+            log_messages = self.logging.tail(
+                source=self.id, limit=limit, time_start=self.time_accepted
+            )
+        else:
+            log_messages = []
+        if self.time_started:
+            log_messages.insert(
+                0, self._format_log("Job Run STARTED", self.time_started)
+            )
+        if self.time_accepted:
+            log_messages.insert(
+                0, self._format_log("Job Run ACCEPTED", self.time_accepted)
+            )
+        if self.time_finished:
+            log_messages.append(
+                self._format_log("Job Run FINISHED", self.time_finished)
+            )
+        return log_messages
+
+    def _job_run_status_text(self) -> str:
+        details = f", {self.lifecycle_details}" if self.lifecycle_details else ""
+        return f"Job Run {self.lifecycle_state}" + details
+
+    def _check_and_print_status(self, prev_status) -> str:
+        status = self._job_run_status_text()
+        if status != prev_status:
+            if self.lifecycle_state in self.TERMINAL_STATES and self.time_finished:
+                timestamp = self.time_finished.strftime("%Y-%m-%d %H:%M:%S")
+            else:
+                timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+            print(f"{timestamp} - {status}")
+        return status
+
+    def wait(self, interval: float = SLEEP_INTERVAL):
+        """Waits for the job run until if finishes.
+
+        Parameters
+        ----------
+        interval : float
+            Time interval in seconds between each request to update the logs.
+            Defaults to 3 (seconds).
+
+        """
+        self.sync()
+        while self.status not in self.TERMINAL_STATES:
+            time.sleep(interval)
+            self.sync()
+        return self
+
+    def watch(
+        self,
+        interval: float = SLEEP_INTERVAL,
+        wait: float = WAIT_SECONDS_AFTER_FINISHED,
+    ) -> DataScienceJobRun:
+        """Watches the job run until it finishes.
+        Before the job start running, this method will output the job run status.
+        Once the job start running,
+        the logs will be streamed until the job is success, failed or cancelled.
+
+        Parameters
+        ----------
+        interval : float
+            Time interval in seconds between each request to update the logs.
+            Defaults to 3 (seconds).
+        wait : float
+            Time in seconds to keep updating the logs after the job run finished.
+            It may take some time for logs to appear in OCI logging service
+            after the job run is finished.
+            Defaults to 90 (seconds).
+
+        """
+
+        def stop_condition():
+            """Stops the log streaming once the job is in a terminal state."""
+            self.sync()
+            if self.lifecycle_state not in self.TERMINAL_STATES:
+                return False
+            # Stop if time_finished is not available.
+            if not self.time_finished:
+                return True
+            # Stop only if time_finished is over 2 minute ago.
+            # This is for the time delay between job run stopped and the logs appear in oci logging.
+            if (
+                datetime.datetime.now(self.time_finished.tzinfo)
+                - datetime.timedelta(seconds=wait)
+                > self.time_finished
+            ):
+                return True
+            return False
+
+        if not self.log_id and not self.log_group_id:
+            print(
+                "Logging is not configured for the job. Watch() will only show job status."
+            )
+
+        print(f"Job OCID: {self.job.id}")
+        print(f"Job Run OCID: {self.id}")
+
+        status = ""
+        while not stop_condition():
+            status = self._check_and_print_status(status)
+            # Break and stream logs if job has log ID and started.
+            # Otherwise, keep watching the status until job terminates.
+            if self.time_started and self.log_id:
+                break
+            time.sleep(interval)
+
+        if self.log_id and self.time_accepted:
+            count = self.logging.stream(
+                source=self.id,
+                interval=interval,
+                stop_condition=stop_condition,
+                time_start=self.time_accepted,
+            )
+            if not count:
+                print(
+                    "No logs in the last 14 days. Please set time_start to see older logs."
+                )
+
+        self._check_and_print_status(status)
+
+        return self
+
+    def cancel(self, wait_for_completion: bool = True) -> DataScienceJobRun:
+        """Cancels a job run
+
+        Parameters
+        ----------
+        wait_for_completion: bool
+            Whether to wait for job run to be cancelled before proceeding.
+            Defaults to True.
+
+        Returns
+        -------
+        self
+            The job run instance.
+        """
+        self.client.cancel_job_run(self.id)
+        if wait_for_completion:
+            while (
+                self.lifecycle_state
+                != oci.data_science.models.JobRun.LIFECYCLE_STATE_CANCELED
+            ):
+                self.sync()
+                time.sleep(SLEEP_INTERVAL)
+        return self
+
+    def __repr__(self) -> str:
+        """Displays the object as YAML."""
+        return self.to_yaml()
+
+    def to_yaml(self) -> str:
+        """Serializes the object into YAML string.
+
+        Returns
+        -------
+        str
+            YAML stored in a string.
+        """
+        # Here the job YAML is used as the base for the job run
+        job_dict = self.job.to_dict()
+
+        # Update infrastructure from job run
+        run_dict = self.to_dict()
+        infra_specs = [
+            run_dict,
+            run_dict.get("jobInfrastructureConfigurationDetails", {}),
+            run_dict.get("logDetails", {}),
+        ]
+        for infra_spec in infra_specs:
+            for key in infra_spec:
+                if key in job_dict["spec"]["infrastructure"]["spec"]:
+                    job_dict["spec"]["infrastructure"]["spec"][key] = infra_spec[key]
+
+        # Update runtime from job run
+        from ads.jobs import Job
+
+        job = Job(**self.auth).from_dict(job_dict)
+        envs = job.runtime.envs
+        run_config_override = run_dict.get("jobConfigurationOverrideDetails", {})
+        envs.update(run_config_override.get("environmentVariables", {}))
+        job.runtime.with_environment_variable(**envs)
+        if run_config_override.get("commandLineArguments"):
+            job.runtime.set_spec(
+                "args",
+                run_config_override.get("commandLineArguments"),
+            )
+
+        # Update kind, id and name
+        run_dict = job.to_dict()
+        run_dict["kind"] = "jobRun"
+        run_dict["spec"]["id"] = self.id
+        run_dict["spec"]["name"] = self.display_name
+        return yaml.safe_dump(run_dict)
+
+    @property
+    def job(self):
+        """The job instance of this run.
+
+        Returns
+        -------
+        Job
+            An ADS Job instance
+        """
+        from ads.jobs import Job
+
+        return Job(**self.auth).from_datascience_job(self.job_id)
+
+    def download(self, to_dir):
+        """Downloads files from job run output URI to local.
+
+        Parameters
+        ----------
+        to_dir : str
+            Local directory to which the files will be downloaded to.
+
+        Returns
+        -------
+        DataScienceJobRun
+            The job run instance (self)
+        """
+        self.job.download(to_dir)
+        return self
+
+    def delete(self, force_delete: bool = False):
+        if force_delete and self.status in [
+            DataScienceJobRun.LIFECYCLE_STATE_ACCEPTED,
+            DataScienceJobRun.LIFECYCLE_STATE_IN_PROGRESS,
+            DataScienceJobRun.LIFECYCLE_STATE_NEEDS_ATTENTION,
+        ]:
+            self.cancel(wait_for_completion=True)
+        super().delete()
+        return self
+
+
+# This is for backward compatibility
+DSCJobRun = DataScienceJobRun
+
+
+class DataScienceJob(Infrastructure):
+    """Represents the OCI Data Science Job infrastructure.
+
+    To configure the infrastructure for a Data Science Job::
+
+        infrastructure = (
+            DataScienceJob()
+            # Configure logging for getting the job run outputs.
+            .with_log_group_id("<log_group_ocid>")
+            # Log resource will be auto-generated if log ID is not specified.
+            .with_log_id("<log_ocid>")
+            # If you are in an OCI data science notebook session,
+            # the following configurations are not required.
+            # Configurations from the notebook session will be used as defaults.
+            .with_compartment_id("<compartment_ocid>")
+            .with_project_id("<project_ocid>")
+            .with_subnet_id("<subnet_ocid>")
+            .with_shape_name("VM.Standard.E3.Flex")
+            # Shape config details are applicable only for the flexible shapes.
+            .with_shape_config_details(memory_in_gbs=16, ocpus=1)
+            # Minimum/Default block storage size is 50 (GB).
+            .with_block_storage_size(50)
+            # A list of file systems to be mounted
+            .with_storage_mount(
+                {
+                    "src" : "<mount_target_ip_address>:<export_path>",
+                    "dest" : "<destination_directory_name>"
+                }
+            )
+            # Tags
+            .with_freeform_tag(my_tag="my_value")
+            .with_defined_tag(**{"Operations": {"CostCenter": "42"}})
+        )
+
+    """
+
+    CONST_PROJECT_ID = "projectId"
+    CONST_COMPARTMENT_ID = "compartmentId"
+    CONST_DISPLAY_NAME = "displayName"
+    CONST_JOB_TYPE = "jobType"
+    CONST_JOB_INFRA = "jobInfrastructureType"
+    CONST_SHAPE_NAME = "shapeName"
+    CONST_BLOCK_STORAGE = "blockStorageSize"
+    CONST_SUBNET_ID = "subnetId"
+    CONST_SHAPE_CONFIG_DETAILS = "shapeConfigDetails"
+    CONST_MEMORY_IN_GBS = "memoryInGBs"
+    CONST_OCPUS = "ocpus"
+    CONST_LOG_ID = "logId"
+    CONST_LOG_GROUP_ID = "logGroupId"
+    CONST_STORAGE_MOUNT = "storageMount"
+    CONST_FREEFORM_TAGS = "freeformTags"
+    CONST_DEFINED_TAGS = "definedTags"
+
+    attribute_map = {
+        CONST_PROJECT_ID: "project_id",
+        CONST_COMPARTMENT_ID: "compartment_id",
+        CONST_DISPLAY_NAME: "display_name",
+        CONST_JOB_TYPE: "job_type",
+        CONST_JOB_INFRA: "job_infrastructure_type",
+        CONST_SHAPE_NAME: "shape_name",
+        CONST_BLOCK_STORAGE: "block_storage_size",
+        CONST_SUBNET_ID: "subnet_id",
+        CONST_SHAPE_CONFIG_DETAILS: "shape_config_details",
+        CONST_LOG_ID: "log_id",
+        CONST_LOG_GROUP_ID: "log_group_id",
+        CONST_STORAGE_MOUNT: "storage_mount",
+        CONST_FREEFORM_TAGS: "freeform_tags",
+        CONST_DEFINED_TAGS: "defined_tags",
+    }
+
+    shape_config_details_attribute_map = {
+        CONST_MEMORY_IN_GBS: "memory_in_gbs",
+        CONST_OCPUS: "ocpus",
+    }
+
+    payload_attribute_map = {
+        CONST_PROJECT_ID: "project_id",
+        CONST_COMPARTMENT_ID: "compartment_id",
+        CONST_DISPLAY_NAME: "display_name",
+        CONST_JOB_TYPE: "job_configuration_details.job_type",
+        CONST_JOB_INFRA: "job_infrastructure_configuration_details.job_infrastructure_type",
+        CONST_SHAPE_NAME: "job_infrastructure_configuration_details.shape_name",
+        CONST_BLOCK_STORAGE: "job_infrastructure_configuration_details.block_storage_size_in_gbs",
+        CONST_SUBNET_ID: "job_infrastructure_configuration_details.subnet_id",
+        CONST_SHAPE_CONFIG_DETAILS: "job_infrastructure_configuration_details.job_shape_config_details",
+        CONST_LOG_ID: "job_log_configuration_details.log_id",
+        CONST_LOG_GROUP_ID: "job_log_configuration_details.log_group_id",
+    }
+
+    snake_to_camel_map = {
+        v.split(".", maxsplit=1)[-1]: k for k, v in payload_attribute_map.items()
+    }
+
+    storage_mount_type_dict = {
+        FILE_STORAGE_TYPE: OCIFileStorage,
+        OBJECT_STORAGE_TYPE: OCIObjectStorage,
+    }
+
+    auth = {}
+
+    @staticmethod
+    def standardize_spec(spec):
+        if not spec:
+            return {}
+
+        attribute_map = {
+            **DataScienceJob.attribute_map,
+            **DataScienceJob.shape_config_details_attribute_map,
+        }
+        snake_to_camel_map = {v: k for k, v in attribute_map.items()}
+        snake_to_camel_map = {
+            **{v: k for k, v in attribute_map.items()},
+            **DataScienceJob.snake_to_camel_map,
+        }
+
+        for key in list(spec.keys()):
+            if key not in attribute_map and key.lower() in snake_to_camel_map:
+                value = spec.pop(key)
+                if isinstance(value, dict):
+                    spec[snake_to_camel_map[key.lower()]] = (
+                        DataScienceJob.standardize_spec(value)
+                    )
+                else:
+                    spec[snake_to_camel_map[key.lower()]] = value
+        return spec
+
+    def __init__(self, spec: Dict = None, **kwargs) -> None:
+        """Initializes a data science job infrastructure
+
+        Parameters
+        ----------
+        spec : dict, optional
+            Object specification, by default None
+        kwargs: dict
+            Specification as keyword arguments.
+            If spec contains the same key as the one in kwargs, the value from kwargs will be used.
+        """
+        # Saves a copy of the auth object from the class to the instance.
+        # Future changes to the class level Job.auth will not affect the auth of existing instances.
+        self.auth = self.auth.copy()
+        for key in ["config", "signer", "client_kwargs"]:
+            if kwargs.get(key):
+                self.auth[key] = kwargs.pop(key)
+
+        self.standardize_spec(spec)
+        self.standardize_spec(kwargs)
+        super().__init__(spec=spec, **kwargs)
+        if not self.job_type:
+            self.with_job_type("DEFAULT")
+        self.dsc_job = DSCJob(**self.auth)
+        self.runtime = None
+        self._name = None
+
+    @property
+    def name(self) -> str:
+        """Display name of the job"""
+        if self.dsc_job:
+            self._name = self.dsc_job.display_name
+        return self._name
+
+    @name.setter
+    def name(self, value: str):
+        """Sets the display name of the job
+
+        Parameters
+        ----------
+        value : str
+            The display name of the job
+        """
+        self._name = value
+        if self.dsc_job:
+            self.dsc_job.display_name = value
+
+    @property
+    def job_id(self) -> Optional[str]:
+        """The OCID of the job"""
+        if self.dsc_job:
+            return self.dsc_job.id
+        return None
+
+    @property
+    def status(self) -> Optional[str]:
+        """Status of the job.
+
+        Returns
+        -------
+        str
+            Status of the job.
+        """
+        if self.dsc_job:
+            return self.dsc_job.lifecycle_state
+        return None
+
+    def with_project_id(self, project_id: str) -> DataScienceJob:
+        """Sets the project OCID
+
+        Parameters
+        ----------
+        project_id : str
+            The project OCID
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_PROJECT_ID, project_id)
+
+    @property
+    def project_id(self) -> Optional[str]:
+        """Project OCID"""
+        return self.get_spec(self.CONST_PROJECT_ID)
+
+    def with_compartment_id(self, compartment_id: str) -> DataScienceJob:
+        """Sets the compartment OCID
+
+        Parameters
+        ----------
+        compartment_id : str
+            The compartment OCID
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_COMPARTMENT_ID, compartment_id)
+
+    @property
+    def compartment_id(self) -> Optional[str]:
+        """The compartment OCID"""
+        return self.get_spec(self.CONST_COMPARTMENT_ID)
+
+    def with_job_type(self, job_type: str) -> DataScienceJob:
+        """Sets the job type
+
+        Parameters
+        ----------
+        job_type : str
+            Job type as string
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_JOB_TYPE, job_type)
+
+    @property
+    def job_type(self) -> Optional[str]:
+        """Job type"""
+        return self.get_spec(self.CONST_JOB_TYPE)
+
+    def with_job_infrastructure_type(self, infrastructure_type: str) -> DataScienceJob:
+        """Sets the job infrastructure type
+
+        Parameters
+        ----------
+        infrastructure_type : str
+            Job infrastructure type as string
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_JOB_INFRA, infrastructure_type)
+
+    @property
+    def job_infrastructure_type(self) -> Optional[str]:
+        """Job infrastructure type"""
+        return self.get_spec(self.CONST_JOB_INFRA)
+
+    def with_shape_name(self, shape_name: str) -> DataScienceJob:
+        """Sets the shape name for running the job
+
+        Parameters
+        ----------
+        shape_name : str
+            Shape name
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_SHAPE_NAME, shape_name)
+
+    @property
+    def shape_name(self) -> Optional[str]:
+        """Shape name"""
+        return self.get_spec(self.CONST_SHAPE_NAME)
+
+    def with_block_storage_size(self, size_in_gb: int) -> DataScienceJob:
+        """Sets the block storage size in GB
+
+        Parameters
+        ----------
+        size_in_gb : int
+            Block storage size in GB
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_BLOCK_STORAGE, size_in_gb)
+
+    @property
+    def block_storage_size(self) -> int:
+        """Block storage size for the job"""
+        return self.get_spec(self.CONST_BLOCK_STORAGE)
+
+    def with_subnet_id(self, subnet_id: str) -> DataScienceJob:
+        """Sets the subnet ID
+
+        Parameters
+        ----------
+        subnet_id : str
+            Subnet ID
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        return self.set_spec(self.CONST_SUBNET_ID, subnet_id)
+
+    @property
+    def subnet_id(self) -> str:
+        """Subnet ID"""
+        return self.get_spec(self.CONST_SUBNET_ID)
+
+    def with_shape_config_details(
+        self, memory_in_gbs: float, ocpus: float, **kwargs: Dict[str, Any]
+    ) -> DataScienceJob:
+        """Sets the details for the job run shape configuration.
+        Specify only when a flex shape is selected.
+        For example `VM.Standard.E3.Flex` allows the memory_in_gbs and cpu count to be specified.
+
+        Parameters
+        ----------
+        memory_in_gbs: float
+            The size of the memory in GBs.
+        ocpus: float
+            The OCPUs count.
+        kwargs
+            Additional keyword arguments.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(
+            self.CONST_SHAPE_CONFIG_DETAILS,
+            {
+                self.CONST_OCPUS: ocpus,
+                self.CONST_MEMORY_IN_GBS: memory_in_gbs,
+                **kwargs,
+            },
+        )
+
+    @property
+    def shape_config_details(self) -> Dict:
+        """The details for the job run shape configuration."""
+        return self.get_spec(self.CONST_SHAPE_CONFIG_DETAILS)
+
+    def with_log_id(self, log_id: str) -> DataScienceJob:
+        """Sets the log OCID for the data science job.
+        If log ID is specified, setting the log group ID (with_log_group_id()) is not strictly needed.
+        ADS will look up the log group ID automatically.
+        However, this may require additional permission,
+        and the look up may not be available for newly created log group.
+        Specifying both log ID (with_log_id()) and log group ID (with_log_group_id())
+        can avoid such lookup and speed up the job creation.
+
+        Parameters
+        ----------
+        log_id : str
+            Log resource OCID.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_LOG_ID, log_id)
+
+    @property
+    def log_id(self) -> str:
+        """Log OCID for the data science job.
+
+        Returns
+        -------
+        str
+            Log OCID
+        """
+        return self.get_spec(self.CONST_LOG_ID)
+
+    def with_log_group_id(self, log_group_id: str) -> DataScienceJob:
+        """Sets the log group OCID for the data science job.
+        If log group ID is specified but log ID is not,
+        a new log resource will be created automatically for each job run to store the logs.
+
+        Parameters
+        ----------
+        log_group_id : str
+            Log Group OCID
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_LOG_GROUP_ID, log_group_id)
+
+    @property
+    def log_group_id(self) -> str:
+        """Log group OCID of the data science job
+
+        Returns
+        -------
+        str
+            Log group OCID
+        """
+        return self.get_spec(self.CONST_LOG_GROUP_ID)
+
+    def with_storage_mount(self, *storage_mount: List[dict]) -> DataScienceJob:
+        """Sets the file systems to be mounted for the data science job.
+        A maximum number of 5 file systems are allowed to be mounted for a single data science job.
+
+        Parameters
+        ----------
+        storage_mount : List[dict]
+            A list of file systems to be mounted.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        storage_mount_list = []
+        for item in storage_mount:
+            if not isinstance(item, dict):
+                raise ValueError(
+                    "Parameter `storage_mount` should be a list of dictionaries."
+                )
+            storage_mount_list.append(item)
+        if len(storage_mount_list) > MAXIMUM_MOUNT_COUNT:
+            raise ValueError(
+                f"A maximum number of {MAXIMUM_MOUNT_COUNT} file systems are allowed to be mounted at this time for a job."
+            )
+        return self.set_spec(self.CONST_STORAGE_MOUNT, storage_mount_list)
+
+    @property
+    def storage_mount(self) -> List[dict]:
+        """Files systems that have been mounted for the data science job
+
+        Returns
+        -------
+        list
+            A list of file systems that have been mounted
+        """
+        return self.get_spec(self.CONST_STORAGE_MOUNT, [])
+
+    def with_freeform_tag(self, **kwargs) -> DataScienceJob:
+        """Sets freeform tags
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_FREEFORM_TAGS, kwargs)
+
+    def with_defined_tag(self, **kwargs) -> DataScienceJob:
+        """Sets defined tags
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_DEFINED_TAGS, kwargs)
+
+    @property
+    def freeform_tags(self) -> dict:
+        """Freeform tags"""
+        return self.get_spec(self.CONST_FREEFORM_TAGS, {})
+
+    @property
+    def defined_tags(self) -> dict:
+        """Defined tags"""
+        return self.get_spec(self.CONST_DEFINED_TAGS, {})
+
+    def _prepare_log_config(self) -> dict:
+        if not self.log_group_id and not self.log_id:
+            return None
+        # Look up log group ID if only the log ID is specified
+        if self.log_id and not self.log_group_id:
+            try:
+                log_obj = OCILog.from_ocid(self.log_id)
+            except ResourceNotFoundError as exc:
+                raise ResourceNotFoundError(
+                    f"Unable to determine log group ID for Log ({self.log_id})."
+                    " The log resource may not exist or You may not have the required permission."
+                    " Try to avoid this by specifying the log group ID."
+                ) from exc
+            self.with_log_group_id(log_obj.log_group_id)
+
+        if self.log_group_id and not self.log_id:
+            enable_auto_log_creation = True
+        else:
+            enable_auto_log_creation = False
+
+        log_config = {
+            "enable_logging": True,
+            "enable_auto_log_creation": enable_auto_log_creation,
+        }
+        if self.log_id:
+            log_config["log_id"] = self.log_id
+
+        if self.log_group_id:
+            log_config["log_group_id"] = self.log_group_id
+        return log_config
+
+    def _update_from_dsc_model(
+        self, dsc_job: oci.data_science.models.Job, overwrite: bool = True
+    ) -> DataScienceJob:
+        """Update the properties from an OCI data science job model.
+
+        Parameters
+        ----------
+        dsc_job: oci.data_science.models.Job
+            An OCI data science job model.
+
+        overwrite: bool
+            Whether to overwrite the existing values.
+            If this is set to False, only the empty/None properties will be updated.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        sub_level = {
+            self.CONST_SHAPE_CONFIG_DETAILS: self.shape_config_details_attribute_map
+        }
+        self.dsc_job = dsc_job
+
+        for infra_attr, dsc_attr in self.payload_attribute_map.items():
+            value = get_value(dsc_job, dsc_attr)
+            if not value:
+                continue
+            if infra_attr not in sub_level:
+                if overwrite or not self._spec.get(infra_attr):
+                    self._spec[infra_attr] = value
+            else:
+                sub_spec = self._spec.get(infra_attr, {})
+                self._spec[infra_attr] = {}
+                for sub_infra_attr, sub_dsc_attr in sub_level[infra_attr].items():
+                    sub_value = get_value(value, sub_dsc_attr)
+                    if not sub_value:
+                        continue
+                    if overwrite or not sub_spec.get(sub_infra_attr):
+                        sub_spec[sub_infra_attr] = sub_value
+                if sub_spec:
+                    self._spec[infra_attr] = sub_spec
+
+        self._update_storage_mount_from_dsc_model(dsc_job, overwrite)
+        return self
+
+    def _update_storage_mount_from_dsc_model(
+        self, dsc_job: oci.data_science.models.Job, overwrite: bool = True
+    ) -> DataScienceJob:
+        """Update the mount storage properties from an OCI data science job model.
+
+        Parameters
+        ----------
+        dsc_job: oci.data_science.models.Job
+            An OCI data science job model.
+
+        overwrite: bool
+            Whether to overwrite the existing values.
+            If this is set to False, only the empty/None properties will be updated.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        storage_mount_list = get_value(
+            dsc_job, "job_storage_mount_configuration_details_list"
+        )
+        if storage_mount_list:
+            storage_mount = [
+                self.storage_mount_type_dict[
+                    file_system.storage_type
+                ].update_from_dsc_model(file_system)
+                for file_system in storage_mount_list
+                if file_system.storage_type in self.storage_mount_type_dict
+            ]
+            if overwrite or not self.get_spec(self.CONST_STORAGE_MOUNT):
+                self.set_spec(self.CONST_STORAGE_MOUNT, storage_mount)
+        return self
+
+    def _update_job_infra(self, dsc_job: DSCJob) -> DataScienceJob:
+        """Updates the job infrastructure from a DSCJob object.
+
+        Parameters
+        ----------
+        dsc_job : DSCJob
+            A DSCJob instance.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        attr_map = {
+            self.CONST_JOB_INFRA: "jobInfrastructureType",
+            self.CONST_SHAPE_NAME: "shapeName",
+            self.CONST_SUBNET_ID: "subnetId",
+            self.CONST_BLOCK_STORAGE: "blockStorageSizeInGBs",
+            self.CONST_SHAPE_CONFIG_DETAILS: "jobShapeConfigDetails",
+        }
+
+        if not dsc_job.job_infrastructure_configuration_details:
+            dsc_job.job_infrastructure_configuration_details = {}
+
+        for snake_attr, camel_attr in attr_map.items():
+            value = self.get_spec(snake_attr)
+            if value:
+                dsc_job.job_infrastructure_configuration_details[camel_attr] = value
+
+        if not dsc_job.job_infrastructure_configuration_details.get(
+            "shapeName", ""
+        ).endswith("Flex") and dsc_job.job_infrastructure_configuration_details.get(
+            "jobShapeConfigDetails"
+        ):
+            raise ValueError(
+                "Shape config is not required for non flex shape from user end."
+            )
+
+        if dsc_job.job_infrastructure_configuration_details.get("subnetId"):
+            dsc_job.job_infrastructure_configuration_details[
+                "jobInfrastructureType"
+            ] = JobInfrastructureConfigurationDetails.JOB_INFRASTRUCTURE_TYPE_STANDALONE
+
+        if self.storage_mount:
+            if not hasattr(oci.data_science.models, "StorageMountConfigurationDetails"):
+                raise EnvironmentError(
+                    "Storage mount hasn't been supported in the current OCI SDK installed."
+                )
+            dsc_job.job_storage_mount_configuration_details_list = [
+                DSCFileSystemManager.initialize(file_system)
+                for file_system in self.storage_mount
+            ]
+        return self
+
+    def build(self) -> DataScienceJob:
+        self.dsc_job.load_defaults()
+
+        try:
+            self.dsc_job.load_defaults()
+        except Exception:
+            logger.exception("Failed to load default properties.")
+
+        self._update_from_dsc_model(self.dsc_job, overwrite=False)
+        return self
+
+    def init(self, **kwargs) -> DataScienceJob:
+        """Initializes a starter specification for the DataScienceJob.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return (
+            self.build()
+            .with_compartment_id(self.compartment_id or "{Provide a compartment OCID}")
+            .with_project_id(self.project_id or "{Provide a project OCID}")
+            .with_subnet_id(
+                self.subnet_id
+                or "{Provide a subnet OCID or remove this field if you use a default networking}"
+            )
+        )
+
+    def create(self, runtime, **kwargs) -> DataScienceJob:
+        """Creates a job with runtime.
+
+        Parameters
+        ----------
+        runtime : Runtime
+            An ADS job runtime.
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+
+        """
+        if not runtime:
+            raise ValueError("Set a valid runtime.")
+        payload = DataScienceJobRuntimeManager(self).translate(runtime)
+        # Add infra properties to payload
+        for attr in ["project_id", "compartment_id"]:
+            if getattr(self, attr):
+                payload[attr] = getattr(self, attr)
+
+        if self.name:
+            display_name = Template(self.name).safe_substitute(runtime.envs)
+        elif isinstance(runtime, GitPythonRuntime) or isinstance(
+            runtime, ContainerRuntime
+        ):
+            display_name = utils.get_random_name_for_resource()
+        else:
+            display_name = None
+
+        payload["display_name"] = display_name
+        payload["job_log_configuration_details"] = self._prepare_log_config()
+        if not payload.get("freeform_tags"):
+            payload["freeform_tags"] = self.freeform_tags
+        if not payload.get("defined_tags"):
+            payload["defined_tags"] = self.defined_tags
+
+        self.dsc_job = DSCJob(**payload, **self.auth)
+        # Set Job infra to user values after DSCJob initialized the defaults
+        self._update_job_infra(self.dsc_job)
+        self.dsc_job.create()
+        # Update the model from infra after job creation.
+        self._update_from_dsc_model(self.dsc_job)
+        return self
+
+    def run(
+        self,
+        name=None,
+        args=None,
+        env_var=None,
+        freeform_tags=None,
+        defined_tags=None,
+        wait=False,
+        **kwargs,
+    ) -> DataScienceJobRun:
+        """Runs a job on OCI Data Science job
+
+        Parameters
+        ----------
+        name : str, optional
+            The name of the job run, by default None.
+        args : str, optional
+            Command line arguments for the job run, by default None.
+        env_var : dict, optional
+            Environment variable for the job run, by default None
+        freeform_tags : dict, optional
+            Freeform tags for the job run, by default None
+        defined_tags : dict, optional
+            Defined tags for the job run, by default None
+        wait : bool, optional
+            Indicate if this method should wait for the run to finish before it returns, by default False.
+        kwargs
+            additional keyword arguments
+
+        Returns
+        -------
+        DataScienceJobRun
+            A Data Science Job Run instance.
+
+        """
+        # Runtime in the infrastructure will be None if the job is not created.
+        if not self.runtime:
+            raise RuntimeError(
+                "Job is not created. Call create() to create the job first."
+            )
+
+        if not freeform_tags:
+            freeform_tags = {}
+        runtime_freeform_tags = self.runtime.freeform_tags
+        if runtime_freeform_tags:
+            freeform_tags.update(runtime_freeform_tags)
+
+        if not defined_tags:
+            defined_tags = {}
+        runtime_defined_tags = self.runtime.defined_tags
+        if runtime_defined_tags:
+            defined_tags.update(runtime_defined_tags)
+
+        if name:
+            envs = self.runtime.envs
+            if env_var:
+                envs.update(env_var)
+            name = Template(name).safe_substitute(envs)
+
+        kwargs = dict(
+            display_name=name,
+            command_line_arguments=args,
+            environment_variables=env_var,
+            freeform_tags=freeform_tags,
+            defined_tags=defined_tags,
+            wait=wait,
+            **kwargs,
+        )
+        # A Runtime class may define customized run() method.
+        # Use the customized method if the run() method is defined by the runtime.
+        # Otherwise, use the default run() method defined in this class.
+        if hasattr(self.runtime, "run"):
+            return self.runtime.run(self.dsc_job, **kwargs)
+        return self.dsc_job.run(**kwargs)
+
+    def delete(self) -> None:
+        """Deletes a job"""
+        self.dsc_job.delete()
+
+    def run_list(self, **kwargs) -> List[DataScienceJobRun]:
+        """Gets a list of job runs.
+
+        Parameters
+        ----------
+        **kwargs :
+            Keyword arguments for filtering the job runs.
+            These arguments will be passed to OCI API.
+
+
+        Returns
+        -------
+        List[DSCJobRun]:
+            A list of job runs.
+
+        """
+        return self.dsc_job.run_list(**kwargs)
+
+    @classmethod
+    def from_dsc_job(cls, dsc_job: DSCJob) -> DataScienceJob:
+        """Initialize a DataScienceJob instance from a DSCJob
+
+        Parameters
+        ----------
+        dsc_job : DSCJob
+            An instance of DSCJob
+
+        Returns
+        -------
+        DataScienceJob
+            An instance of DataScienceJob
+
+        """
+        instance = cls()
+        instance._update_from_dsc_model(dsc_job)
+        instance.runtime = DataScienceJobRuntimeManager(instance).extract(dsc_job)
+        return instance
+
+    @class_or_instance_method
+    def from_id(cls, job_id: str) -> DataScienceJob:
+        """Gets an existing job using Job OCID
+
+        Parameters
+        ----------
+        job_id : str
+            Job OCID
+
+
+        Returns
+        -------
+        DataScienceJob
+            An instance of DataScienceJob
+
+        """
+        return cls.from_dsc_job(DSCJob(**cls.auth).from_ocid(job_id))
+
+    @class_or_instance_method
+    def from_dict(cls, obj_dict: dict):
+        """Initialize the object from a Python dictionary"""
+        if inspect.isclass(cls):
+            job_cls = cls
+        else:
+            job_cls = cls.__class__
+        return job_cls(spec=obj_dict.get("spec"), **cls.auth)
+
+    @class_or_instance_method
+    def list_jobs(cls, compartment_id: str = None, **kwargs) -> List[DataScienceJob]:
+        """Lists all jobs in a compartment.
+
+        Parameters
+        ----------
+        compartment_id : str, optional
+            The compartment ID for running the jobs, by default None.
+            This is optional in a OCI Data Science notebook session.
+            If this is not specified, the compartment ID of the notebook session will be used.
+        **kwargs :
+            Keyword arguments to be passed into OCI list_jobs API for filtering the jobs.
+
+        Returns
+        -------
+        List[DataScienceJob]
+            A list of DataScienceJob object.
+
+        """
+        return [
+            cls.from_dsc_job(job)
+            for job in DSCJob(**cls.auth).list_resource(compartment_id, **kwargs)
+        ]
+
+    @class_or_instance_method
+    def instance_shapes(cls, compartment_id: str = None, **kwargs) -> list:
+        """Lists the supported shapes for running jobs in a compartment.
+
+        Parameters
+        ----------
+        compartment_id : str, optional
+            The compartment ID for running the jobs, by default None.
+            This is optional in a OCI Data Science notebook session.
+            If this is not specified, the compartment ID of the notebook session will be used.
+
+        Returns
+        -------
+        list
+            A list of oci.data_science.models.JobShapeSummary objects
+            containing the information of the supported shapes.
+
+        Examples
+        --------
+        To get a list of shape names::
+
+            shapes = DataScienceJob.fast_launch_shapes(
+                compartment_id=os.environ["PROJECT_COMPARTMENT_OCID"]
+            )
+            shape_names = [shape.name for shape in shapes]
+
+        """
+        shapes = oci.pagination.list_call_get_all_results(
+            DSCJob(**cls.auth).init_client().list_job_shapes,
+            DSCJob.check_compartment_id(compartment_id),
+            **kwargs,
+        ).data
+        return shapes
+
+    @class_or_instance_method
+    def fast_launch_shapes(cls, compartment_id: str = None, **kwargs) -> list:
+        """Lists the supported fast launch shapes for running jobs in a compartment.
+
+        Parameters
+        ----------
+        compartment_id : str, optional
+            The compartment ID for running the jobs, by default None.
+            This is optional in a OCI Data Science notebook session.
+            If this is not specified, the compartment ID of the notebook session will be used.
+
+        Returns
+        -------
+        list
+            A list of oci.data_science.models.FastLaunchJobConfigSummary objects
+            containing the information of the supported shapes.
+
+        Examples
+        --------
+        To get a list of shape names::
+
+            shapes = DataScienceJob.fast_launch_shapes(
+                compartment_id=os.environ["PROJECT_COMPARTMENT_OCID"]
+            )
+            shape_names = [shape.shape_name for shape in shapes]
+
+        """
+        shapes = oci.pagination.list_call_get_all_results(
+            DSCJob(**cls.auth).init_client().list_fast_launch_job_configs,
+            DSCJob.check_compartment_id(compartment_id),
+            **kwargs,
+        ).data
+        return shapes