oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/jobs/builders/infrastructure/dataflow.py

@@ -0,0 +1,1252 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import copy
+import datetime
+import io
+import json
+import logging
+import os
+import time
+import urllib.parse
+from typing import Any, Dict, List, Optional
+
+import fsspec
+import oci.data_flow
+import oci.util as oci_util
+import yaml
+from ads.common import utils
+from ads.common.auth import default_signer
+from ads.common.decorator.utils import class_or_instance_method
+from ads.common.oci_client import OCIClientFactory
+from ads.common.oci_mixin import OCIModelMixin
+from ads.common.utils import batch_convert_case, camel_to_snake
+from ads.jobs.builders.infrastructure.base import Infrastructure, RunInstance
+from ads.jobs.builders.infrastructure.utils import normalize_config
+from ads.jobs.builders.runtimes.python_runtime import DataFlowRuntime
+from ads.model.runtime.env_info import InferenceEnvInfo
+from oci.data_flow.models import CreateApplicationDetails, CreateRunDetails
+from tqdm import tqdm
+
+from ads.config import NB_SESSION_COMPARTMENT_OCID, NB_SESSION_OCID
+
+logger = logging.getLogger(__name__)
+
+CONDA_PACK_SUFFIX = "#conda"
+SLEEP_INTERVAL = 3
+
+DEFAULT_LANGUAGE = "PYTHON"
+DEFAULT_SPARK_VERSION = "3.2.1"
+DEFAULT_NUM_EXECUTORS = 1
+DEFAULT_SHAPE = "VM.Standard.E3.Flex"
+DATAFLOW_SHAPE_FAMILY = [
+    "Standard.E3",
+    "Standard.E4",
+    "Standard3",
+    "Standard.A1",
+    "Standard2",
+]
+
+
+def conda_pack_name_to_dataflow_config(conda_uri):
+    return {
+        "spark.archives": conda_uri + CONDA_PACK_SUFFIX,
+        "dataflow.auth": "resource_principal",
+    }
+
+
+def _env_variables_to_dataflow_config(
+    env_vars: Dict[str, str] = None
+) -> Dict[str, str]:
+    """Prepares environment variables for the application.
+    Similar environment variables will be setup for the driver and executor.
+
+    Parameters
+    ----------
+    env_vars: (Dict[str, str], optional). Defaults to `None`
+        The dictionary with SRC env variables.
+        Example: {"env1": "value1"}
+
+    Returns
+    -------
+    Dict[str, str]
+        Dictionary with pre populated environment variables.
+        Example: {"spark.executorEnv.env1": "value1", "spark.driverEnv.env1": "value1"}
+    """
+    if not env_vars:
+        return {}
+
+    result = {}
+    for level in ("spark.executorEnv", "spark.driverEnv"):
+        for env_name, env_value in env_vars.items():
+            result[f"{level}.{env_name}"] = env_value
+
+    return result
+
+
+class DataFlowApp(OCIModelMixin, oci.data_flow.models.Application):
+    @classmethod
+    def init_client(cls, **kwargs) -> oci.data_flow.data_flow_client.DataFlowClient:
+        return cls._init_client(
+            client=oci.data_flow.data_flow_client.DataFlowClient, **kwargs
+        )
+
+    @property
+    def client(self) -> oci.data_flow.data_flow_client.DataFlowClient:
+        return super().client
+
+    def create(self) -> "DataFlowApp":
+        """
+        Create a Data Flow application.
+
+        Returns
+        -------
+        DataFlowApp
+            a DataFlowApp instance
+
+        """
+        resp = self.client.create_application(
+            self.to_oci_model(CreateApplicationDetails)
+        )
+        logger.debug(f"Created a DataFlow Application {resp.data}")
+        for k, v in json.loads(repr(resp.data)).items():
+            setattr(self, k, v)
+        return self
+
+    def delete(self) -> None:
+        """
+        Delete a Data Flow application.
+
+        Returns
+        -------
+        None
+        """
+        self.application = self.client.delete_application(self.id)
+        self.lifecycle_state = oci.data_flow.models.Application.LIFECYCLE_STATE_DELETED
+
+    def __repr__(self) -> str:
+        """Displays the object as YAML."""
+        return self.to_yaml()
+
+    def to_yaml(self) -> str:
+        """Serializes the object into YAML string.
+
+        Returns
+        -------
+        str
+            YAML stored in a string.
+        """
+        return yaml.safe_dump(self.to_dict())
+
+
+class DataFlowRun(OCIModelMixin, oci.data_flow.models.Run, RunInstance):
+    _DETAILS_LINK = (
+        "https://console.{region}.oraclecloud.com/data-flow/runs/details/{id}"
+    )
+
+    TERMINATED_STATES = [
+        oci.data_flow.models.Run.LIFECYCLE_STATE_CANCELED,
+        oci.data_flow.models.Run.LIFECYCLE_STATE_FAILED,
+        oci.data_flow.models.Run.LIFECYCLE_STATE_SUCCEEDED,
+    ]
+
+    @classmethod
+    def init_client(cls, **kwargs) -> oci.data_flow.data_flow_client.DataFlowClient:
+        return cls._init_client(
+            client=oci.data_flow.data_flow_client.DataFlowClient, **kwargs
+        )
+
+    @property
+    def client(self) -> oci.data_flow.data_flow_client.DataFlowClient:
+        return super().client
+
+    def create(self) -> "DataFlowRun":
+        """
+        Create a Data Flow run.
+
+        Returns
+        -------
+        DataFlowRun
+            a DataFlowRun instance
+        """
+        resp = self.client.create_run(self.to_oci_model(CreateRunDetails))
+        logger.debug(f"Created a DataFlow Run {resp.data}")
+        for k, v in json.loads(repr(resp.data)).items():
+            setattr(self, k, v)
+        return self
+
+    @property
+    def status(self) -> str:
+        """
+        Show status (lifecycle state) of a run.
+
+        Returns
+        -------
+        str
+            status of the run
+        """
+        if self.id:
+            resp = self.client.get_run(self.id)
+            self.lifecycle_state = resp.data.lifecycle_state
+        return self.lifecycle_state
+
+    @property
+    def logs(self) -> "DataFlowLogs":
+        """
+        Show logs from a run.
+        There are three types of logs: application log, driver log and executor log,
+        each with stdout and stderr separately.
+        To access each type of logs,
+        >>> dfr.logs.application.stdout
+        >>> dfr.logs.driver.stderr
+
+        Returns
+        -------
+        DataFlowLogs
+            an instance of DataFlowLogs
+        """
+        return DataFlowLogs(run_id=self.id)
+
+    def wait(self, interval: int = SLEEP_INTERVAL) -> "DataFlowRun":
+        """
+        Wait for a run to terminate.
+
+        Parameters
+        ----------
+        interval: int, optional
+            interval to wait before probing again
+
+        Returns
+        -------
+        DataFlowRun
+            a DataFlowRun instance
+        """
+        current = self.status
+        timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+        logger.info(f"{timestamp} - {self.status}")
+        print(f"{timestamp} - {self.status}")
+        while current not in self.TERMINATED_STATES:
+            time.sleep(interval)
+            if self.status != current:
+                timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+                logger.info(f"{timestamp} - {self.status}")
+                print(f"{timestamp} - {self.status}")
+                current = self.status
+        return self
+
+    def watch(self, interval: int = SLEEP_INTERVAL) -> "DataFlowRun":
+        """This is an alias of `wait()` method. It waits for a run to terminate.
+
+        Parameters
+        ----------
+        interval: int, optional
+            interval to wait before probing again
+
+        Returns
+        -------
+        DataFlowRun
+            a DataFlowRun instance
+        """
+        logger.info(
+            "This is an alias of `wait`. Logs are available after run completes."
+        )
+        return self.wait(interval=interval)
+
+    def to_yaml(self) -> str:
+        """Serializes the object into YAML string.
+
+        Returns
+        -------
+        str
+            YAML stored in a string.
+        """
+        run = self.to_dict()
+        run["lifecycleState"] = self.status
+        return yaml.safe_dump(run)
+
+    def delete(self) -> "DataFlowRun":
+        """
+        Cancel and delete a Data Flow run if it is not yet terminated.
+        Will be executed asynchronously.
+
+        Returns
+        -------
+        self
+            The dataflow run instance.
+        """
+        if self.status not in self.TERMINATED_STATES:
+            self.client.delete_run(self.id)
+            self.lifecycle_state = oci.data_flow.models.Run.LIFECYCLE_STATE_CANCELING
+
+        return self
+
+    def cancel(self) -> "DataFlowRun":
+        """Cancel a Data Flow run if it is not yet terminated.
+        Will be executed synchronously.
+
+        Returns
+        -------
+        self
+            The dataflow run instance.
+        """
+        self.delete()
+        self.wait()
+        return self
+
+
+class _Log:
+    def __init__(self, run_id, src_type, output_type):
+        self.run_id = run_id
+        self.src_type = src_type
+        self.output_type = output_type
+        self._cache = None
+
+    @property
+    def client(self):
+        return OCIClientFactory(**default_signer()).dataflow
+
+    def _get_logs(self, n=None, reverse=False):
+        if not self._cache:
+            logs = self.client.list_run_logs(self.run_id).data
+            lines = []
+            for log in logs:
+                if log.type == self.output_type and self.src_type == log.source:
+                    lines.extend(
+                        self.client.get_run_log(self.run_id, log.name)
+                        .data.text.lstrip("\x00")
+                        .rstrip("\n")
+                        .splitlines()
+                    )
+            self._cache = lines
+        if n and reverse:
+            return "\n".join(self._cache[-n:])
+        elif n and not reverse:
+            return "\n".join(self._cache[:n])
+        else:
+            return "\n".join(self._cache)
+
+    def __repr__(self):
+        return self._get_logs()
+
+    def head(self, n=None):
+        return self._get_logs(n=n)
+
+    def tail(self, n=None):
+        return self._get_logs(n=n, reverse=True)
+
+    def download(self, path):
+        with open(path, mode="w") as f:
+            f.write(self._get_logs())
+
+
+class _DataFlowLog:
+    def __init__(self, run_id, src_type):
+        self.run_id = run_id
+        self.src_type = src_type
+
+    @property
+    def stdout(self):
+        return _Log(self.run_id, self.src_type, "STDOUT")
+
+    @property
+    def stderr(self):
+        return _Log(self.run_id, self.src_type, "STDERR")
+
+
+class DataFlowLogs:
+    def __init__(self, run_id):
+        self.run_id = run_id
+
+    @property
+    def application(self):
+        return _DataFlowLog(run_id=self.run_id, src_type="APPLICATION")
+
+    @property
+    def driver(self):
+        return _DataFlowLog(run_id=self.run_id, src_type="DRIVER")
+
+    @property
+    def executor(self):
+        return _DataFlowLog(run_id=self.run_id, src_type="EXECUTOR")
+
+
+class DataFlow(Infrastructure):
+    CONST_COMPARTMENT_ID = "compartment_id"
+    CONST_CONFIG = "configuration"
+    CONST_EXECUTE = "execute"
+    CONST_DRIVER_SHAPE = "driver_shape"
+    CONST_EXECUTOR_SHAPE = "executor_shape"
+    CONST_LANGUAGE = "language"
+    CONST_METASTORE_ID = "metastore_id"
+    CONST_BUCKET_URI = "logs_bucket_uri"
+    CONST_NUM_EXECUTORS = "num_executors"
+    CONST_SPARK_VERSION = "spark_version"
+    CONST_WAREHOUSE_BUCKET_URI = "warehouse_bucket_uri"
+    CONST_DRIVER_SHAPE_CONFIG = "driver_shape_config"
+    CONST_EXECUTOR_SHAPE_CONFIG = "executor_shape_config"
+    CONST_MEMORY_IN_GBS = "memory_in_gbs"
+    CONST_OCPUS = "ocpus"
+    CONST_ID = "id"
+    CONST_PRIVATE_ENDPOINT_ID = "private_endpoint_id"
+    CONST_POOL_ID = "pool_id"
+    CONST_FREEFORM_TAGS = "freeform_tags"
+    CONST_DEFINED_TAGS = "defined_tags"
+
+    attribute_map = {
+        CONST_COMPARTMENT_ID: "compartmentId",
+        CONST_CONFIG: CONST_CONFIG,
+        CONST_EXECUTE: CONST_EXECUTE,
+        CONST_DRIVER_SHAPE: "driverShape",
+        CONST_EXECUTOR_SHAPE: "executorShape",
+        CONST_METASTORE_ID: "metastoreId",
+        CONST_BUCKET_URI: "logsBucketUri",
+        CONST_NUM_EXECUTORS: "numExecutors",
+        CONST_SPARK_VERSION: "sparkVersion",
+        CONST_WAREHOUSE_BUCKET_URI: "warehouseBucketUri",
+        CONST_DRIVER_SHAPE_CONFIG: "driverShapeConfig",
+        CONST_EXECUTOR_SHAPE_CONFIG: "executorShapeConfig",
+        CONST_MEMORY_IN_GBS: "memoryInGBs",
+        CONST_OCPUS: CONST_OCPUS,
+        CONST_ID: CONST_ID,
+        CONST_PRIVATE_ENDPOINT_ID: "privateEndpointId",
+        CONST_POOL_ID: "poolId",
+        CONST_FREEFORM_TAGS: "freeformTags",
+        CONST_DEFINED_TAGS: "definedTags",
+    }
+
+    def __init__(self, spec: dict = None, **kwargs):
+        defaults = self._load_default_properties()
+        spec = self._standardize_spec(spec)
+        kwargs = self._standardize_spec(kwargs)
+        if spec is None:
+            super(DataFlow, self).__init__(defaults, **kwargs)
+        else:
+            spec = {
+                k: v
+                for k, v in spec.items()
+                if (
+                    f"with_{camel_to_snake(k)}" in self.__dir__()
+                    or (k == "defined_tags" or "freeform_tags")
+                )
+                and v is not None
+            }
+            defaults.update(spec)
+            super().__init__(defaults, **kwargs)
+
+        self.df_app = DataFlowApp(**self._spec)
+        self.runtime = None
+        self._name = None
+
+    def _load_default_properties(self) -> Dict:
+        """
+        Load default properties from environment variables, notebook session, etc.
+
+        Returns
+        -------
+        dict
+            a dictionary of default properties
+        """
+        defaults = {}
+        if NB_SESSION_COMPARTMENT_OCID:
+            defaults[self.CONST_COMPARTMENT_ID] = NB_SESSION_COMPARTMENT_OCID
+        if NB_SESSION_OCID:
+            dsc_client = OCIClientFactory(**default_signer()).data_science
+            try:
+                nb_session = dsc_client.get_notebook_session(NB_SESSION_OCID).data
+                nb_config = nb_session.notebook_session_configuration_details
+
+                defaults[self.CONST_DRIVER_SHAPE] = nb_config.shape
+                logger.debug(f"Set driver shape to {nb_config.shape}")
+
+                defaults[self.CONST_EXECUTOR_SHAPE] = nb_config.shape
+                logger.debug(f"Set executor shape to {nb_config.shape}")
+
+                if nb_config.notebook_session_shape_config_details:
+                    notebook_shape_config_details = oci_util.to_dict(
+                        nb_config.notebook_session_shape_config_details
+                    )
+
+                    defaults[self.CONST_DRIVER_SHAPE_CONFIG] = copy.deepcopy(
+                        notebook_shape_config_details
+                    )
+                    logger.debug(
+                        f"Set driver shape config to {nb_config.notebook_session_shape_config_details}"
+                    )
+
+                    defaults[self.CONST_EXECUTOR_SHAPE_CONFIG] = copy.deepcopy(
+                        notebook_shape_config_details
+                    )
+                    logger.debug(
+                        f"Set executor shape config to {nb_config.notebook_session_shape_config_details}"
+                    )
+
+            except Exception as e:
+                logger.warning(
+                    f"Error fetching details about Notebook session: {os.environ['NB_SESSION_OCID']}. {e}"
+                )
+
+        defaults["language"] = DEFAULT_LANGUAGE
+        defaults["spark_version"] = DEFAULT_SPARK_VERSION
+        defaults["num_executors"] = DEFAULT_NUM_EXECUTORS
+
+        logger.debug(f"Set spark version to be {defaults['spark_version']}")
+        logger.debug(f"Set number of executors to be {defaults['num_executors']}")
+
+        return defaults
+
+    @property
+    def name(self) -> str:
+        """Display name of the job"""
+        return self._name
+
+    @name.setter
+    def name(self, value: str):
+        """Sets the display name of the job
+
+        Parameters
+        ----------
+        value : str
+            The display name of the job
+        """
+        if self.df_app:
+            self.df_app.display_name = value
+        self._name = value
+
+    @property
+    def job_id(self) -> Optional[str]:
+        """The OCID of the job"""
+        return self.get_spec("id")
+
+    def with_compartment_id(self, id: str) -> "DataFlow":
+        """
+        Set compartment id for a Data Flow job.
+
+        Parameters
+        ----------
+        id: str
+            compartment id
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_COMPARTMENT_ID, id)
+
+    def with_configuration(self, configs: dict) -> "DataFlow":
+        """
+        Set configuration for a Data Flow job.
+
+        Parameters
+        ----------
+        configs: dict
+            dictionary of configurations
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_CONFIG, configs)
+
+    def with_execute(self, exec: str) -> "DataFlow":
+        """
+        Set command for spark-submit.
+
+        Parameters
+        ----------
+        exec: str
+            str of commands
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_EXECUTE, exec)
+
+    def with_driver_shape(self, shape: str) -> "DataFlow":
+        """
+        Set driver shape for a Data Flow job.
+
+        Parameters
+        ----------
+        shape: str
+            driver shape
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_DRIVER_SHAPE, shape)
+
+    def with_executor_shape(self, shape: str) -> "DataFlow":
+        """
+        Set executor shape for a Data Flow job.
+
+        Parameters
+        ----------
+        shape: str
+            executor shape
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_EXECUTOR_SHAPE, shape)
+
+    def with_language(self, lang: str) -> "DataFlow":
+        """
+        Set language for a Data Flow job.
+
+        Parameters
+        ----------
+        lang: str
+            language for the job
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_LANGUAGE, lang)
+
+    def with_metastore_id(self, id: str) -> "DataFlow":
+        """
+        Set Hive metastore id for a Data Flow job.
+
+        Parameters
+        ----------
+        id: str
+            metastore id
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+
+        return self.set_spec(self.CONST_METASTORE_ID, id)
+
+    def with_logs_bucket_uri(self, uri: str) -> "DataFlow":
+        """
+        Set logs bucket uri for a Data Flow job.
+
+        Parameters
+        ----------
+        uri: str
+            uri to logs bucket
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_BUCKET_URI, uri)
+
+    def with_num_executors(self, n: int) -> "DataFlow":
+        """
+        Set number of executors for a Data Flow job.
+
+        Parameters
+        ----------
+        n: int
+            number of executors
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_NUM_EXECUTORS, n)
+
+    def with_spark_version(self, ver: str) -> "DataFlow":
+        """
+        Set spark version for a Data Flow job.
+        Currently supported versions are 2.4.4, 3.0.2 and 3.2.1
+        Documentation: https://docs.oracle.com/en-us/iaas/data-flow/using/dfs_getting_started.htm#before_you_begin
+
+        Parameters
+        ----------
+        ver: str
+            spark version
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_SPARK_VERSION, ver)
+
+    def with_warehouse_bucket_uri(self, uri: str) -> "DataFlow":
+        """
+        Set warehouse bucket uri for a Data Flow job.
+
+        Parameters
+        ----------
+        uri: str
+            uri to warehouse bucket
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_WAREHOUSE_BUCKET_URI, uri)
+
+    def with_id(self, id: str) -> "DataFlow":
+        """
+        Set id for a Data Flow job.
+
+        Parameters
+        ----------
+        id: str
+            id of a job
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_ID, id)
+
+    def with_driver_shape_config(
+        self, memory_in_gbs: float, ocpus: float, **kwargs: Dict[str, Any]
+    ) -> "DataFlow":
+        """
+        Sets the driver shape config details of Data Flow job infrastructure.
+        Specify only when a flex shape is selected.
+        For example `VM.Standard.E3.Flex` allows the memory_in_gbs and cpu count to be specified.
+
+        Parameters
+        ----------
+        memory_in_gbs: float
+            The size of the memory in GBs.
+        ocpus: float
+            The OCPUs count.
+        kwargs
+            Additional keyword arguments.
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself.
+        """
+        return self.set_spec(
+            self.CONST_DRIVER_SHAPE_CONFIG,
+            {
+                self.CONST_OCPUS: ocpus,
+                self.CONST_MEMORY_IN_GBS: memory_in_gbs,
+                **kwargs,
+            },
+        )
+
+    def with_executor_shape_config(
+        self, memory_in_gbs: float, ocpus: float, **kwargs: Dict[str, Any]
+    ) -> "DataFlow":
+        """
+        Sets the executor shape config details of Data Flow job infrastructure.
+        Specify only when a flex shape is selected.
+        For example `VM.Standard.E3.Flex` allows the memory_in_gbs and cpu count to be specified.
+
+        Parameters
+        ----------
+        memory_in_gbs: float
+            The size of the memory in GBs.
+        ocpus: float
+            The OCPUs count.
+        kwargs
+            Additional keyword arguments.
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself.
+        """
+        return self.set_spec(
+            self.CONST_EXECUTOR_SHAPE_CONFIG,
+            {
+                self.CONST_OCPUS: ocpus,
+                self.CONST_MEMORY_IN_GBS: memory_in_gbs,
+                **kwargs,
+            },
+        )
+
+    def with_private_endpoint_id(self, private_endpoint_id: str) -> "DataFlow":
+        """
+        Set the private endpoint ID for a Data Flow job infrastructure.
+
+        Parameters
+        ----------
+        private_endpoint_id: str
+            The OCID of a private endpoint.
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        return self.set_spec(self.CONST_PRIVATE_ENDPOINT_ID, private_endpoint_id)
+
+    def with_freeform_tag(self, **kwargs) -> "DataFlow":
+        """Sets freeform tags
+
+        Returns
+        -------
+        DataFlow
+            The DataFlow instance (self)
+        """
+        return self.set_spec(self.CONST_FREEFORM_TAGS, kwargs)
+
+    def with_defined_tag(self, **kwargs) -> "DataFlow":
+        """Sets defined tags
+
+        Returns
+        -------
+        DataFlow
+            The DataFlow instance (self)
+        """
+        return self.set_spec(self.CONST_DEFINED_TAGS, kwargs)
+
+    def with_pool_id(self, pool_id: str) -> "DataFlow":
+        """
+        Set the Data Flow Pool Id for a Data Flow job.
+
+        Parameters
+        ----------
+        pool_id: str
+            The OCID of a Data Flow Pool.
+
+        Returns
+        -------
+        DataFlow
+            the Data Flow instance itself
+        """
+        if not hasattr(CreateApplicationDetails, "pool_id"):
+            raise EnvironmentError(
+                "Data Flow Pool has not been supported in the current OCI SDK installed."
+            )
+        return self.set_spec(self.CONST_POOL_ID, pool_id)
+
+    def __getattr__(self, item):
+        if item == self.CONST_DEFINED_TAGS or item == self.CONST_FREEFORM_TAGS:
+            return self.get_spec(item)
+        elif (
+            f"with_{item}" in self.__dir__()
+            and item != "defined_tag"
+            and item != "freeform_tag"
+        ):
+            return self.get_spec(item)
+        raise AttributeError(f"Attribute {item} not found.")
+
+    def create(self, runtime: DataFlowRuntime, **kwargs) -> "DataFlow":
+        """
+        Create a Data Flow job given a runtime.
+
+        Parameters
+        ----------
+        runtime
+            runtime to bind to the Data Flow job
+        kwargs
+            additional keyword arguments
+
+        Returns
+        -------
+        DataFlow
+            a Data Flow job instance
+        """
+        if self.pool_id:
+            if not hasattr(CreateApplicationDetails, "pool_id"):
+                raise EnvironmentError(
+                    "Data Flow Pool has not been supported in the current OCI SDK installed."
+                )
+        # Set default display_name if not specified - randomly generated easy to remember name
+        if not self.name:
+            self.name = utils.get_random_name_for_resource()
+        payload = copy.deepcopy(self._spec)
+        overwrite = kwargs.pop("overwrite", runtime.overwrite) or False
+        runtime.convert(overwrite=overwrite)
+        if not runtime.script_uri:
+            raise ValueError("script uri must be specified in runtime.")
+        if runtime.script_uri.split(":")[0] != "oci":
+            if runtime.script_bucket:
+                runtime.with_script_uri(
+                    self._upload_file(
+                        runtime.script_uri, runtime.script_bucket, overwrite
+                    )
+                )
+            else:
+                raise ValueError(
+                    "script bucket must be specified if script uri given is local."
+                )
+        if runtime.archive_uri and runtime.archive_uri.split(":")[0] != "oci":
+            if runtime.archive_bucket:
+                runtime.with_archive_uri(
+                    self._upload_file(
+                        runtime.archive_uri, runtime.archive_bucket, overwrite
+                    )
+                )
+            else:
+                raise ValueError(
+                    "archive bucket must be specified if archive uri given is local."
+                )
+        if runtime.conda:
+            conda_type = runtime.conda.get(runtime.CONST_CONDA_TYPE)
+            if conda_type == runtime.CONST_CONDA_TYPE_CUSTOM:
+                conda_uri = runtime.conda.get(runtime.CONST_CONDA_URI)
+            elif conda_type == runtime.CONST_CONDA_TYPE_SERVICE:
+                conda_slug = runtime.conda.get(runtime.CONST_CONDA_SLUG)
+                env_info = InferenceEnvInfo.from_slug(env_slug=conda_slug)
+                conda_uri = env_info.inference_env_path
+                # TODO: Re-visit if we can enable a global read policy
+                raise NotImplementedError(
+                    "Service Conda Packs not yet supported. Please use a published pack."
+                )
+            else:
+                raise ValueError(f"Conda built type not understood: {conda_type}.")
+            runtime_config = runtime.configuration or dict()
+            runtime_config.update(conda_pack_name_to_dataflow_config(conda_uri))
+            runtime.with_configuration(runtime_config)
+
+        payload.update(
+            {
+                "display_name": self.name,
+                "file_uri": runtime.script_uri,
+                "freeform_tags": runtime.freeform_tags or self.freeform_tags,
+                "defined_tags": runtime.defined_tags or self.defined_tags,
+                "archive_uri": runtime.archive_uri,
+                "configuration": runtime.configuration,
+            }
+        )
+        if len(runtime.args) > 0:
+            payload["arguments"] = runtime.args
+        if not payload.get("compartment_id", None):
+            raise ValueError(
+                "Compartment id is required. Specify compartment id via 'with_compartment_id()'."
+            )
+        self._validate_shapes(payload)
+        payload.pop("id", None)
+        logger.debug(f"Creating a DataFlow Application with payload {payload}")
+        self.df_app = DataFlowApp(**payload).create()
+        self.with_id(self.df_app.id)
+        return self
+
+    @staticmethod
+    def _validate_shapes(payload: Dict):
+        if "executor_shape" not in payload:
+            payload["executor_shape"] = DEFAULT_SHAPE
+        if "driver_shape" not in payload:
+            payload["driver_shape"] = DEFAULT_SHAPE
+        executor_shape = payload["executor_shape"]
+        executor_shape_config = payload.get("executor_shape_config", {})
+        driver_shape = payload["driver_shape"]
+        driver_shape_config = payload.get("driver_shape_config", {})
+        same_shape_family = False
+        for shape in DATAFLOW_SHAPE_FAMILY:
+            if shape in executor_shape and shape in driver_shape:
+                same_shape_family = True
+                break
+        if not same_shape_family:
+            raise ValueError(
+                "`executor_shape` and `driver_shape` must be from the same shape family."
+            )
+        if (not executor_shape.endswith("Flex") and executor_shape_config) or (
+            not driver_shape.endswith("Flex") and driver_shape_config
+        ):
+            raise ValueError(
+                "Shape config is not required for non flex shape from user end."
+            )
+
+    @staticmethod
+    def _upload_file(local_path, bucket, overwrite=False):
+        signer = default_signer()
+        os_client = OCIClientFactory(**signer).object_storage
+        namespace = os_client.get_namespace().data
+        parsed = urllib.parse.urlparse(bucket)
+        chunk_size = io.DEFAULT_BUFFER_SIZE
+        if parsed.scheme == "oci":
+            dst_path = os.path.join(bucket, os.path.basename(local_path))
+        else:
+            dst_path = f"oci://{bucket}@{namespace}/{os.path.basename(local_path)}"
+        logger.debug(f"Uploading {local_path} to {dst_path}")
+        with fsspec.open(dst_path, mode="wb", **signer) as fto:
+            file_system_clz = fsspec.get_filesystem_class(
+                urllib.parse.urlparse(local_path).scheme or "file"
+            )
+            file_size = file_system_clz().info(local_path)["size"]
+
+            if not overwrite:
+                remote_file_system_clz = fsspec.get_filesystem_class(
+                    urllib.parse.urlparse(dst_path).scheme or "file"
+                )
+                remote_file_system = remote_file_system_clz(**default_signer())
+                if remote_file_system.exists(dst_path):
+                    raise FileExistsError(
+                        f"{dst_path} exists. Please use a new file name."
+                    )
+
+            with fsspec.open(local_path, mode="rb", encoding=None) as fread:
+                with tqdm.wrapattr(
+                    fread,
+                    "read",
+                    desc=f"Uploading file: {local_path} to {dst_path}",
+                    total=file_size,
+                ) as ffrom:
+                    while True:
+                        chunk = ffrom.read(chunk_size)
+                        if chunk:
+                            fto.write(chunk)
+                        else:
+                            break
+        return dst_path
+
+    def run(
+        self,
+        name: str = None,
+        args: List[str] = None,
+        env_vars: Dict[str, str] = None,
+        freeform_tags: Dict[str, str] = None,
+        defined_tags: Dict[str, Dict[str, object]] = None,
+        wait: bool = False,
+        **kwargs,
+    ) -> DataFlowRun:
+        """
+        Run a Data Flow job.
+
+        Parameters
+        ----------
+        name: str, optional
+            name of the run. If a name is not provided, a randomly generated easy to remember name
+            with timestamp will be generated, like 'strange-spider-2022-08-17-23:55.02'.
+        args: List[str], optional
+            list of command line arguments
+        env_vars: Dict[str, str], optional
+            dictionary of environment variables (not used for data flow)
+        freeform_tags: Dict[str, str], optional
+            freeform tags
+        defined_tags: Dict[str, Dict[str, object]], optional
+            defined tags
+        wait: bool, optional
+            whether to wait for a run to terminate
+        kwargs
+            additional keyword arguments
+
+        Returns
+        -------
+        DataFlowRun
+            a DataFlowRun instance
+        """
+        payload = kwargs
+        if "application_id" not in payload:
+            payload["application_id"] = self.id
+        if "compartment_id" not in payload:
+            payload["compartment_id"] = self.get_spec("compartment_id")
+        # Set default display_name if not specified - randomly generated easy to remember name generated
+        payload["display_name"] = name if name else utils.get_random_name_for_resource()
+        payload["arguments"] = args if args and len(args) > 0 else None
+        payload["freeform_tags"] = freeform_tags or self.freeform_tags
+        payload["defined_tags"] = defined_tags or self.defined_tags
+        payload.pop("spark_version", None)
+        logger.debug(f"Creating a DataFlow Run with payload {payload}")
+        run = DataFlowRun(**payload).create()
+        if wait:
+            run.wait(kwargs.pop("interval", SLEEP_INTERVAL))
+        return run
+
+    def run_list(self, **kwargs) -> List[DataFlowRun]:
+        """
+        List runs associated with a Data Flow job.
+
+        Parameters
+        ----------
+        kwargs
+         additional arguments for filtering runs.
+
+        Returns
+        -------
+        List[DataFlowRun]
+            list of DataFlowRun instances
+        """
+        items = oci.pagination.list_call_get_all_results(
+            self.df_app.client.list_runs,
+            self.compartment_id,
+            application_id=self.id,
+            **kwargs,
+        ).data
+        return [DataFlowRun.from_oci_model(item) for item in items]
+
+    def delete(self):
+        """
+        Delete a Data Flow job and canceling associated runs.
+
+        Returns
+        -------
+        None
+        """
+        runs = self.run_list()
+        for run in runs:
+            run.delete()
+        self.df_app.delete()
+
+    @classmethod
+    def from_id(cls, id: str) -> "DataFlow":
+        """
+        Load a Data Flow job given an id.
+
+        Parameters
+        ----------
+        id: str
+            id of the Data Flow job to load
+
+        Returns
+        -------
+        DataFlow
+            a Data Flow job instance
+        """
+        payload = batch_convert_case(
+            yaml.safe_load(repr(DataFlowApp.from_ocid(id))), "snake"
+        )
+        config = normalize_config(payload)
+        name = config.pop("display_name")
+        ins = cls(config)
+        rt = DataFlowRuntime()
+        for k, v in config.items():
+            if hasattr(DataFlowRuntime, f"with_{k}"):
+                getattr(rt, f"with_{k}")(v)
+        ins.runtime = rt
+        ins.name = name
+        return ins
+
+    @classmethod
+    def list_jobs(cls, compartment_id: str = None, **kwargs) -> List["DataFlow"]:
+        """
+        List Data Flow jobs in a given compartment.
+
+        Parameters
+        ----------
+        compartment_id: str
+            id of that compartment
+        kwargs
+            additional keyword arguments for filtering jobs
+
+        Returns
+        -------
+        List[DataFlow]
+            list of Data Flow jobs
+        """
+        return [
+            cls.from_id(job.id)
+            for job in DataFlowApp.list_resource(compartment_id, **kwargs)
+        ]
+
+    def to_dict(self, **kwargs) -> dict:
+        """
+        Serialize job to a dictionary.
+
+        Returns
+        -------
+        dict
+            serialized job as a dictionary
+        """
+        spec = self._convert_shape_config(copy.deepcopy(self._spec), "camel")
+        return {
+            "kind": self.kind,
+            "type": self.type,
+            "spec": batch_convert_case(spec, "camel"),
+        }
+
+    @classmethod
+    def from_dict(cls, config: dict) -> "DataFlow":
+        """
+        Load a Data Flow job instance from a dictionary of configurations.
+
+        Parameters
+        ----------
+        config: dict
+            dictionary of configurations
+
+        Returns
+        -------
+        DataFlow
+            a Data Flow job instance
+        """
+        spec = cls._convert_shape_config(copy.deepcopy(config["spec"]), "snake")
+        return cls(spec=batch_convert_case(spec, "snake"))
+
+    @class_or_instance_method
+    def _convert_shape_config(cls, spec: Dict, to_format: str) -> Dict:
+        """Converts the format of shape config details from camel to snake, or vice versa.
+
+        Parameters
+        ----------
+        spec: dict
+            dictionary of specs
+        to_format: str
+            the format that's converted to
+
+        Returns
+        -------
+        Dict
+            dictionary with converted shape config details
+        """
+        shape_config_map = [
+            cls.CONST_DRIVER_SHAPE_CONFIG,
+            cls.CONST_EXECUTOR_SHAPE_CONFIG,
+            cls.attribute_map[cls.CONST_DRIVER_SHAPE_CONFIG],
+            cls.attribute_map[cls.CONST_EXECUTOR_SHAPE_CONFIG],
+        ]
+        converted_map = {
+            "camel": {
+                cls.CONST_MEMORY_IN_GBS: cls.attribute_map[cls.CONST_MEMORY_IN_GBS],
+                cls.CONST_OCPUS: cls.CONST_OCPUS,
+            },
+            "snake": {
+                cls.attribute_map[cls.CONST_MEMORY_IN_GBS]: cls.CONST_MEMORY_IN_GBS,
+                cls.CONST_OCPUS: cls.CONST_OCPUS,
+            },
+        }
+        for shape_config in shape_config_map:
+            shape_config_value = spec.pop(shape_config, {})
+            if shape_config_value:
+                temp_maps = {}
+                for key, value in shape_config_value.items():
+                    converted_key = converted_map[to_format].get(key, None)
+                    if converted_key:
+                        temp_maps[converted_key] = value
+                    else:
+                        temp_maps[key] = value
+                spec[shape_config] = copy.deepcopy(temp_maps)
+        return spec
+
+    def to_yaml(self, **kwargs) -> str:
+        """Serializes the object into YAML string.
+
+        Returns
+        -------
+        str
+            YAML stored in a string.
+        """
+        return yaml.safe_dump(self.to_dict(**kwargs))
+
+    def init(self, **kwargs) -> "DataFlow":
+        """Initializes a starter specification for the DataFlow.
+
+        Returns
+        -------
+        DataFlow
+            The DataFlow instance (self)
+        """
+        return (
+            self.build()
+            .with_compartment_id(self.compartment_id or "{Provide a compartment OCID}")
+            .with_language(self.language or DEFAULT_LANGUAGE)
+            .with_spark_version(self.spark_version or DEFAULT_SPARK_VERSION)
+            .with_num_executors(self.num_executors or DEFAULT_NUM_EXECUTORS)
+            .with_driver_shape(self.driver_shape or DEFAULT_SHAPE)
+            .with_executor_shape(self.executor_shape or DEFAULT_SHAPE)
+        )