oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/opctl/backend/local.py

@@ -0,0 +1,1053 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import copy
+import json
+import os
+import runpy
+import sys
+import tempfile
+from concurrent.futures import Future, ThreadPoolExecutor
+from time import sleep
+from typing import Dict, List, Optional, Union
+
+from oci.data_science.models import PipelineStepRun
+
+from ads.common.auth import AuthContext, create_signer
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.common.oci_client import OCIClientFactory
+from ads.config import NO_CONTAINER
+from ads.model.model_metadata import ModelCustomMetadata
+from ads.model.runtime.runtime_info import RuntimeInfo
+from ads.opctl import logger
+from ads.opctl.backend.base import Backend
+from ads.opctl.conda.cmds import _install
+from ads.opctl.config.resolver import ConfigResolver
+from ads.opctl.constants import (
+    DEFAULT_IMAGE_CONDA_DIR,
+    DEFAULT_IMAGE_HOME_DIR,
+    DEFAULT_IMAGE_SCRIPT_DIR,
+    DEFAULT_MODEL_DEPLOYMENT_FOLDER,
+    DEFAULT_MODEL_FOLDER,
+    DEFAULT_NOTEBOOK_SESSION_CONDA_DIR,
+    DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
+    ML_JOB_GPU_IMAGE,
+    ML_JOB_IMAGE,
+)
+from ads.opctl.distributed.cmds import load_ini, local_run
+from ads.opctl.model.cmds import _download_model
+from ads.opctl.operator import __operators__
+from ads.opctl.operator.common.const import ENV_OPERATOR_ARGS
+from ads.opctl.operator.common.operator_loader import OperatorInfo, OperatorLoader
+from ads.opctl.operator.runtime import const as operator_runtime_const
+from ads.opctl.operator.runtime import runtime as operator_runtime
+from ads.opctl.spark.cmds import (
+    generate_core_site_properties,
+    generate_core_site_properties_str,
+)
+from ads.opctl.utils import (
+    build_image,
+    get_docker_client,
+    is_in_notebook_session,
+    run_command,
+    run_container,
+)
+from ads.pipeline.ads_pipeline import Pipeline, PipelineStep
+
+
+class CondaPackNotFound(Exception):  # pragma: no cover
+    pass
+
+
+class LocalBackend(Backend):
+    def __init__(self, config: Dict) -> None:
+        """
+        Initialize a LocalBackend object with given config.
+
+        Parameters
+        ----------
+        config: dict
+            dictionary of configurations
+        """
+        super().__init__(config=config)
+
+    def run(self):
+        if self.config.get("version") == "v1.0":
+            docker_image = self.config["spec"]["Infrastructure"]["spec"]["dockerImage"]
+            # TODO: don't hard code api keys
+            bind_volumes = {
+                os.path.expanduser("~/.oci"): {
+                    "bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, ".oci"),
+                    "mode": "ro",
+                }
+            }
+            self._run_with_image_v1(bind_volumes)
+
+        else:
+            bind_volumes = {}
+            if not is_in_notebook_session():
+                bind_volumes = {
+                    os.path.expanduser(
+                        os.path.dirname(self.config["execution"]["oci_config"])
+                    ): {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, ".oci")}
+                }
+            if self.config["execution"].get("conda_slug", None):
+                exit_code = self._run_with_conda_pack(bind_volumes)
+            elif self.config["execution"].get("image"):
+                exit_code = self._run_with_image(bind_volumes)
+            else:
+                raise ValueError("Either conda pack info or image should be specified.")
+
+            if exit_code != 0:
+                raise RuntimeError(
+                    f"Job did not complete successfully. Exit code: {exit_code}. "
+                    f"Run with the --debug argument to view container logs."
+                )
+
+    @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
+    def init_vscode_container(self) -> None:
+        """
+        Create a .devcontainer.json file for development with VSCode.
+
+        Returns
+        -------
+        None
+        """
+        source_folder = self.config["execution"].get("source_folder", None)
+        if source_folder:
+            source_folder = os.path.abspath(source_folder)
+            if not os.path.exists(source_folder):
+                raise FileNotFoundError(
+                    f"source folder {source_folder} does not exist."
+                )
+            if self.config["execution"].get("gpu", False):
+                image = self.config["execution"].get("image", ML_JOB_GPU_IMAGE)
+            else:
+                image = self.config["execution"].get("image", ML_JOB_IMAGE)
+            oci_config_folder = os.path.expanduser(
+                os.path.dirname(self.config["execution"]["oci_config"])
+            )
+            dev_container = {
+                "image": image,
+                "extensions": ["ms-python.python"],
+                "mounts": [
+                    f"source={oci_config_folder},target={os.path.join(DEFAULT_IMAGE_HOME_DIR, '.oci')},type=bind",
+                ],
+                "workspaceMount": f"source={source_folder},target={os.path.join(DEFAULT_IMAGE_HOME_DIR, os.path.basename(source_folder))},type=bind",
+                "workspaceFolder": DEFAULT_IMAGE_HOME_DIR,
+                "name": f"{image}-dev-env",
+            }
+            if image == ML_JOB_IMAGE or image == ML_JOB_GPU_IMAGE:
+                conda_folder = os.path.expanduser(
+                    self.config["execution"]["conda_pack_folder"]
+                )
+                dev_container["mounts"].append(
+                    f"source={conda_folder},target={DEFAULT_IMAGE_CONDA_DIR},type=bind"
+                )
+                dev_container[
+                    "postCreateCommand"
+                ] = "conda init bash && source ~/.bashrc"
+            else:
+                raise ValueError(
+                    "`--source-folder` option works with image `ml-job`, `ml-job-gpu` only. Those can be build with `ads opctl build-image`. Please check `ads opctl build-image -h`."
+                )
+        else:
+            image = self.config["execution"].get("image", None)
+            if not image:
+                raise ValueError("Image must be specified.")
+            else:
+                dev_container = {
+                    "image": image,
+                    "mounts": [],
+                    "extensions": ["ms-python.python"],
+                }
+
+        dev_container["containerEnv"] = self.config["execution"].get("env_vars", {})
+        for k, v in self.config["execution"]["volumes"].items():
+            dev_container["mounts"].append(
+                f"source={os.path.abspath(k)},target={v['bind']},type=bind"
+            )
+
+        try:
+            client = get_docker_client()
+            client.api.inspect_image(image)
+        except docker.errors.ImageNotFound:
+            cmd = None
+            if image == ML_JOB_IMAGE:
+                cmd = "ads opctl build-image job-local"
+            elif image == ML_JOB_GPU_IMAGE:
+                cmd = "ads opctl build-image job-local -g"
+            if cmd:
+                raise RuntimeError(
+                    f"Image {image} not found. Please run `{cmd}` to build the image."
+                )
+            else:
+                raise RuntimeError(f"Image {image} not found.")
+        if source_folder:
+            with open(
+                os.path.join(os.path.abspath(source_folder), ".devcontainer.json"), "w"
+            ) as f:
+                f.write(json.dumps(dev_container, indent=2))
+            print(f"File {os.path.join(source_folder, '.devcontainer.json')} created.")
+        else:
+            with open(os.path.abspath(".devcontainer.json"), "w") as f:
+                f.write(json.dumps(dev_container, indent=2))
+            print(f"File {os.path.abspath('.devcontainer.json')} created.")
+
+    def _run_with_conda_pack(
+        self,
+        bind_volumes: Dict,
+        extra_cmd: str = "",
+        install: bool = False,
+        conda_uri: str = "",
+    ) -> int:
+        env_vars = self.config["execution"].get("env_vars", {})
+        slug = self.config["execution"]["conda_slug"]
+        image = self.config["execution"].get("image", None)
+
+        # bind_volumes is modified in-place and does not need to be returned
+        # it is returned just to be explicit that it is changed during this function call
+        bind_volumes, env_vars = self._check_conda_pack_and_install_if_applicable(
+            slug, bind_volumes, env_vars, install=install, conda_uri=conda_uri
+        )
+        bind_volumes = self._mount_source_folder_if_exists(bind_volumes)
+        command = self._build_command_for_conda_run(extra_cmd)
+        if is_in_notebook_session():
+            run_command(command, shell=True)
+        else:
+            conda_pack_path = os.path.join(
+                os.path.expanduser(self.config["execution"]["conda_pack_folder"]), slug
+            )
+            if os.path.exists(os.path.join(conda_pack_path, "spark-defaults.conf")):
+                env_vars["SPARK_CONF_DIR"] = os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug)
+            logger.info(
+                f"Running with conda pack in a container with command {command}"
+            )
+            return self._activate_conda_env_and_run(
+                image, slug, command, bind_volumes, env_vars
+            )
+
+    def _build_command_for_conda_run(self, extra_cmd: str = "") -> str:
+        if ConfigResolver(self.config)._is_ads_operator():
+            if is_in_notebook_session():
+                curr_dir = os.path.dirname(os.path.abspath(__file__))
+                script = os.path.abspath(
+                    os.path.join(curr_dir, "..", "operators", "run.py")
+                )
+            else:
+                script = os.path.join(DEFAULT_IMAGE_SCRIPT_DIR, "operators/run.py")
+            command = f"python {script} "
+            if self.config["execution"]["auth"] == "resource_principal":
+                command += "-r "
+        else:
+            entry_script = self.config["execution"].get("entrypoint")
+            if not entry_script:
+                raise ValueError(
+                    "An entrypoint script must be specified when running with conda pack. "
+                    "Use `--entrypoint`."
+                )
+            if not os.path.exists(
+                os.path.join(self.config["execution"]["source_folder"], entry_script)
+            ):
+                raise FileNotFoundError(
+                    f"{os.path.join(self.config['execution']['source_folder'], entry_script)} is not found."
+                )
+            if is_in_notebook_session():
+                source_folder = os.path.join(self.config["execution"]["source_folder"])
+            else:
+                source_folder = os.path.join(
+                    DEFAULT_IMAGE_SCRIPT_DIR,
+                    "operators",
+                    os.path.basename(self.config["execution"]["source_folder"]),
+                )
+            if os.path.splitext(entry_script)[-1] == ".py":
+                command = f"python {os.path.join(source_folder, entry_script)} "
+                if is_in_notebook_session():
+                    command = (
+                        f"source activate {os.path.join(DEFAULT_NOTEBOOK_SESSION_CONDA_DIR, self.config['execution']['conda_slug'])} && "
+                        + command
+                    )
+            elif os.path.splitext(entry_script)[-1] == ".sh":
+                command = f"cd {source_folder} && /bin/bash {entry_script} "
+            else:
+                logger.warn(
+                    "ML Job only support .py and .sh files."
+                    "If you intend to submit to ML Job later, please update file extension."
+                )
+                command = f"cd {source_folder} && {entry_script} "
+        if self.config["execution"].get("command"):
+            command += f"{self.config['execution']['command']}"
+        command += extra_cmd
+        return command
+
+    def _run_with_image(self, bind_volumes: Dict) -> int:
+        env_vars = self.config["execution"]["env_vars"]
+        image = self.config["execution"]["image"]
+        if ConfigResolver(self.config)._is_ads_operator():
+            # running operators
+            command = (
+                f"python {os.path.join(DEFAULT_IMAGE_SCRIPT_DIR, 'operators/run.py')} "
+            )
+            entrypoint = None
+            if self.config["execution"].get("command", None):
+                command += f"{self.config['execution']['command']}"
+        else:
+            # in case of running a user image, entrypoint is not required
+            entrypoint = self.config["execution"].get("entrypoint", None)
+            command = self.config["execution"].get("command", None)
+        if self.config["execution"].get("source_folder", None):
+            bind_volumes.update(self._mount_source_folder_if_exists(bind_volumes))
+        bind_volumes.update(self.config["execution"]["volumes"])
+
+        return run_container(image, bind_volumes, env_vars, command, entrypoint)
+
+    def _run_with_image_v1(self, bind_volumes: Dict) -> int:
+        env_vars = [
+            str(d["name"]) + "=" + str(d["value"])
+            for d in self.config["spec"]["Runtime"]["spec"]["environmentVariables"]
+        ]
+        image = self.config["spec"]["Infrastructure"]["spec"]["dockerImage"]
+        command = self.config["spec"]["Runtime"]["spec"]["entrypoint"]
+        entrypoint = "python /etc/datascience/operator/cluster_helper.py"
+
+        print("looking to bind volume")
+        bind_volumes.update(self.config["spec"]["Framework"]["spec"]["bindVolumes"])
+        return run_container(
+            image=image,
+            bind_volumes=bind_volumes,
+            env_vars=env_vars,
+            command=command,
+            entrypoint=entrypoint,
+        )
+
+    def _check_conda_pack_and_install_if_applicable(
+        self,
+        slug: str,
+        bind_volumes: Dict,
+        env_vars: Dict,
+        install: bool = False,
+        conda_uri: str = None,
+    ) -> Dict:
+        conda_pack_folder = os.path.abspath(
+            os.path.expanduser(self.config["execution"]["conda_pack_folder"])
+        )
+        conda_pack_path = os.path.join(conda_pack_folder, slug)
+        if not os.path.exists(conda_pack_path):
+            if install:
+                logger.info(
+                    f"Downloading a `{slug}` to the `{conda_pack_folder}`. If this conda pack is already installed locally in a different location, pass in `conda_pack_folder` to avoid downloading it again."
+                )
+                _install(
+                    conda_uri=conda_uri,
+                    conda_pack_folder=conda_pack_folder,
+                    oci_config=self.oci_config,
+                    oci_profile=self.profile,
+                    auth_type=self.auth_type,
+                )
+            else:
+                raise CondaPackNotFound(
+                    f"Conda pack {conda_pack_path} not found. Please run `ads opctl conda create` or `ads opctl conda install`."
+                )
+        if os.path.exists(os.path.join(conda_pack_path, "spark-defaults.conf")):
+            if not is_in_notebook_session():
+                env_vars["SPARK_CONF_DIR"] = os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug)
+            # write core_site.xml
+            if self.config["execution"]["auth"] == "api_key":
+                properties = generate_core_site_properties(
+                    "api_key",
+                    self.config["execution"]["oci_config"],
+                    self.config["execution"]["oci_profile"],
+                )
+                # key path cannot have "~/"
+                oci_config_folder = os.path.expanduser(
+                    os.path.dirname(self.config["execution"]["oci_config"])
+                )
+                properties[-1] = (
+                    properties[-1][0],
+                    os.path.join(
+                        DEFAULT_IMAGE_HOME_DIR,
+                        ".oci",
+                        os.path.relpath(
+                            os.path.expanduser(properties[-1][1]), oci_config_folder
+                        ),
+                    ),
+                )
+            else:
+                properties = generate_core_site_properties("resource_principal")
+
+            core_site_str = generate_core_site_properties_str(properties)
+            if is_in_notebook_session():
+                with open(
+                    os.path.join(
+                        DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR, "core-site.xml"
+                    ),
+                    "w",
+                ) as f:
+                    f.write(core_site_str)
+            else:
+                with open(os.path.join(conda_pack_path, "core-site.xml"), "w") as f:
+                    f.write(core_site_str)
+        bind_volumes[
+            os.path.abspath(
+                os.path.expanduser(
+                    os.path.join(self.config["execution"]["conda_pack_folder"], slug)
+                )
+            )
+        ] = {"bind": os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug)}
+        return bind_volumes, env_vars
+
+    def _mount_source_folder_if_exists(self, bind_volumes: Dict) -> Dict:
+        source_folder = os.path.abspath(self.config["execution"]["source_folder"])
+        if not os.path.exists(source_folder):
+            raise FileNotFoundError(f"source folder {source_folder} does not exist.")
+        mount_path = os.path.join(
+            DEFAULT_IMAGE_SCRIPT_DIR,
+            "operators",
+            os.path.basename(self.config["execution"]["source_folder"]),
+        )
+        bind_volumes[source_folder] = {"bind": mount_path}
+        return bind_volumes
+
+    @staticmethod
+    @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
+    def _activate_conda_env_and_run(
+        image: str, slug: str, command: List[str], bind_volumes: Dict, env_vars: Dict
+    ) -> int:
+        import docker
+
+        try:
+            client = get_docker_client()
+            client.api.inspect_image(image)
+        except docker.errors.ImageNotFound:
+            logger.info(f"Image {image} not found. Attempt building it now....")
+            if image == ML_JOB_IMAGE:
+                build_image("job-local", gpu=False)
+            else:
+                build_image("job-local", gpu=True)
+        with tempfile.TemporaryDirectory(dir=os.path.expanduser("~")) as td:
+            with open(os.path.join(td, "entryscript.sh"), "w") as f:
+                f.write(
+                    f"""
+#!/bin/bash
+source {os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug, 'bin/activate')}
+{command}
+                        """
+                )
+            bind_volumes[os.path.join(td, "entryscript.sh")] = {
+                "bind": os.path.join(DEFAULT_IMAGE_SCRIPT_DIR, "entryscript.sh")
+            }
+            env_vars["conda_slug"] = slug
+            return run_container(
+                image,
+                bind_volumes,
+                env_vars,
+                command=f"bash {os.path.join(DEFAULT_IMAGE_SCRIPT_DIR, 'entryscript.sh')}",
+            )
+
+
+class LocalBackendDistributed(LocalBackend):
+    def __init__(self, config: Dict) -> None:
+        """
+        Initialize a LocalBackendDistributed object with given config. This serves local single node(docker) testing
+        for Distributed Tranining
+
+
+        Parameters
+        ----------
+        config: dict
+            dictionary of configurations
+        """
+        self.config = config
+
+    def run(self):
+        local_run(self.config, load_ini())
+
+
+class LocalPipelineBackend(Backend):
+    LOG_PREFIX = "Local Pipeline:"
+    DEFAULT_PARALLEL_CONTAINER_MAXIMUM = 4
+    DEFAULT_STATUS_POLL_INTERVAL_SECONDS = 5
+
+    def __init__(self, config: Dict) -> None:
+        """
+        Initialize a LocalPipelineBackend object with given config.
+
+        Parameters
+        ----------
+        config: dict
+            dictionary of configurations
+        """
+        self.config = config
+
+    def run(self) -> None:
+        pipeline = Pipeline.from_dict(self.config)
+
+        self._log_orchestration_message(f"Starting pipeline {pipeline.name} locally.")
+        if pipeline.dag:
+            self._log_orchestration_message(f"Pipeline DAG:")
+            for d in pipeline.dag:
+                self._log_orchestration_message(f"  {d}")
+
+        completed_status = {}
+        waiting_steps = {}
+        for s in pipeline.step_details:
+            waiting_steps[s.name] = s
+        futures = {}
+        pipeline_failure = False
+        done = False
+
+        if "max_parallel_containers" in self.config["infrastructure"]:
+            max_parallel_containers = int(
+                self.config["infrastructure"]["max_parallel_containers"]
+            )
+        else:
+            max_parallel_containers = min(
+                self.DEFAULT_PARALLEL_CONTAINER_MAXIMUM, os.cpu_count()
+            )
+            logger.warn(
+                f"max_parallel_containers not specified in the config. Defaulting to {max_parallel_containers}."
+                " Run `ads opctl configure` to define your local backend config."
+            )
+
+        poll_interval_seconds = int(
+            self.config["infrastructure"].get(
+                "pipeline_status_poll_interval_seconds",
+                self.DEFAULT_STATUS_POLL_INTERVAL_SECONDS,
+            )
+        )
+
+        with ThreadPoolExecutor(max_workers=max_parallel_containers) as executor:
+            while not done:
+                # Check if any running steps have completed
+                for s in list(futures):
+                    if futures[s].done():
+                        if futures[s].exception() is None:
+                            self._log_orchestration_message(
+                                f"Step {s} completed successfully."
+                            )
+                            completed_status[
+                                s
+                            ] = PipelineStepRun.LIFECYCLE_STATE_SUCCEEDED
+                        else:
+                            pipeline_failure = True
+                            self._log_orchestration_message(f"Step {s} failed:")
+                            logger.error(futures[s].exception())
+                            completed_status[s] = PipelineStepRun.LIFECYCLE_STATE_FAILED
+                        del futures[s]
+
+                for s in list(waiting_steps):
+                    # Cancel all waiting steps if a failure is encountered
+                    if pipeline_failure:
+                        self._log_orchestration_message(
+                            f"Skipping step {s} - pipeline failure encountered."
+                        )
+                        completed_status[s] = PipelineStepRun.LIFECYCLE_STATE_SKIPPED
+                        del waiting_steps[s]
+                        continue
+
+                    # Start a waiting step if all of its dependencies have completed successfully
+                    completed_deps = [
+                        dep
+                        for dep in waiting_steps[s].depends_on
+                        if dep in completed_status
+                    ]
+                    if len(waiting_steps[s].depends_on) == len(completed_deps):
+                        self._log_orchestration_message(f"Starting step {s}")
+                        futures[s] = self._start_pipeline_step(
+                            waiting_steps[s], executor
+                        )
+                        del waiting_steps[s]
+
+                if len(completed_status) == len(pipeline.step_details):
+                    done = True
+                else:
+                    sleep(poll_interval_seconds)
+
+        self._log_orchestration_message("Pipeline run complete!")
+        self._log_orchestration_message("Summary:")
+        for step in pipeline.step_details:
+            self._log_orchestration_message(
+                f"  {step.name} - {completed_status[step.name]}"
+            )
+
+    def _start_pipeline_step(
+        self, step: PipelineStep, executor: ThreadPoolExecutor
+    ) -> Future:
+        """
+        Starts a single pipeline step.
+
+        Parameters
+        ----------
+        step: PipelineStep
+            The pipeline step to start
+        executor: ThreadPoolExecutor
+            The executor used to run the pipeline step.
+
+        Returns
+        -------
+        future: Future
+            The Future that can be used to query the status of the pipeline step.
+        """
+        step_config = self._create_step_config(step)
+
+        # Have a local job backend execute the step using the updated step config
+        local_job = LocalBackend(step_config)
+        return executor.submit(local_job.run)
+
+    def _create_step_config(self, pipeline_step: PipelineStep) -> Dict:
+        """
+        Creates the config for local execution of an individual pipeline step.
+
+        Parameters
+        ----------
+        pipeline_step: PipelineStep
+            The pipeline step whose config should be generated
+
+        Returns
+        -------
+        step_config: Dict
+            The config for the individual pipeline step.
+        """
+        if pipeline_step.kind.upper() != "CUSTOM_SCRIPT":
+            raise ValueError(
+                f"Step {pipeline_step.name} has unsupported kind. "
+                f"Local pipeline execution only supports pipeline steps with kind customScript."
+            )
+
+        step_config = copy.deepcopy(self.config)
+        step_config["kind"] = pipeline_step.kind
+        step_config["type"] = pipeline_step.type
+        del step_config["spec"]
+        step_execution_config = step_config["execution"]
+        step_execution_config["conda_slug"] = pipeline_step.runtime.conda["slug"]
+        step_execution_config["env_vars"] = pipeline_step.runtime.envs
+
+        if pipeline_step.runtime.type == "script":
+            step_execution_config["entrypoint"] = pipeline_step.runtime.script_uri
+        elif pipeline_step.runtime.type == "python":
+            step_execution_config["entrypoint"] = pipeline_step.runtime.script_uri
+            step_execution_config["source_folder"] = pipeline_step.runtime.working_dir
+        elif pipeline_step.runtime.type == "notebook":
+            step_execution_config["entrypoint"] = pipeline_step.runtime.notebook_uri
+        else:
+            raise ValueError(
+                f"Step {pipeline_step.name} has unsupported runtime. "
+                f"Supported values are: script, python, notebook"
+            )
+
+        if not step_execution_config.get("source_folder"):
+            logger.warn(
+                "No source_folder provided; defaulting to the current working directory. To specify a source"
+                "folder for all pipeline steps, use the --source-folder parameter. To specify a source folder"
+                "for individual steps, use a runtime with type python and specify the workingDir property."
+            )
+            step_execution_config["source_folder"] = os.getcwd()
+
+        ConfigResolver(step_config).process()
+        return step_config
+
+    def _log_orchestration_message(self, str: str) -> None:
+        """
+        Logs a message related to pipeline run orchestration
+
+        Parameters
+        ----------
+        str: str
+            The message to log
+        """
+        logger.info(f"{self.LOG_PREFIX}: {str}")
+
+
+class LocalModelDeploymentBackend(LocalBackend):
+    def __init__(self, config: Dict) -> None:
+        """
+        Initialize a LocalModelDeploymentBackend object with given config.
+
+        Parameters
+        ----------
+        config: dict
+            dictionary of configurations
+        """
+        super().__init__(config)
+        self.oci_auth = create_signer(
+            self.auth_type,
+            self.oci_config,
+            self.profile,
+        )
+        self.client = OCIClientFactory(**self.oci_auth).data_science
+
+    def predict(self) -> None:
+        """
+        Conducts local verify.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+
+        # model artifact in artifact directory
+        artifact_directory = self.config["execution"].get("artifact_directory")
+        ocid = self.config["execution"].get("ocid")
+
+        model_folder = os.path.expanduser(
+            self.config["execution"].get("model_save_folder", DEFAULT_MODEL_FOLDER)
+        )
+        artifact_directory = artifact_directory or os.path.join(model_folder, str(ocid))
+        if ocid and (
+            not os.path.exists(artifact_directory)
+            or len(os.listdir(artifact_directory)) == 0
+        ):
+            region = self.config["execution"].get("region", None)
+            bucket_uri = self.config["execution"].get("bucket_uri", None)
+            timeout = self.config["execution"].get("timeout", None)
+            logger.info(
+                f"No cached model found. Downloading the model {ocid} to {artifact_directory}. If you already have a copy of the model, specify `artifact_directory` instead of `ocid`. You can specify `model_save_folder` to decide where to store the model artifacts."
+            )
+
+            _download_model(
+                auth=self.auth_type,
+                profile=self.profile,
+                ocid=ocid,
+                artifact_directory=artifact_directory,
+                region=region,
+                bucket_uri=bucket_uri,
+                timeout=timeout,
+                force_overwrite=True,
+            )
+
+        # conda
+        conda_slug = self.config["execution"].get("conda_slug")
+        conda_path = self.config["execution"].get("conda_path")
+        if not conda_slug and not conda_path and ocid:
+            conda_slug, conda_path = self._get_conda_info_from_custom_metadata(ocid)
+        if not conda_slug and not conda_path:
+            conda_slug, conda_path = self._get_conda_info_from_runtime(
+                artifact_dir=artifact_directory
+            )
+        if "conda_slug" not in self.config["execution"]:
+            self.config["execution"]["conda_slug"] = (
+                conda_path.split("/")[-1] if conda_path else conda_slug
+            )
+
+        self.config["execution"]["image"] = ML_JOB_IMAGE
+
+        # bind_volumnes
+        bind_volumes = {}
+        SCRIPT = "script.py"
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        if not is_in_notebook_session():
+            bind_volumes = {
+                os.path.expanduser(
+                    os.path.dirname(self.config["execution"]["oci_config"])
+                ): {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, ".oci")}
+            }
+
+            self.config["execution"]["source_folder"] = os.path.abspath(
+                os.path.join(dir_path, "..")
+            )
+            self.config["execution"]["entrypoint"] = SCRIPT
+            bind_volumes[artifact_directory] = {"bind": DEFAULT_MODEL_DEPLOYMENT_FOLDER}
+
+        # extra cmd
+        data = self.config["execution"].get("payload")
+        extra_cmd = f"--payload '{data}' " + f"--auth {self.auth_type} "
+        if self.auth_type != "resource_principal":
+            extra_cmd += f"--profile {self.profile}"
+
+        if is_in_notebook_session() or NO_CONTAINER:
+            # _run_with_conda_pack has code to handle notebook session case,
+            # however, it activate the conda pack and then run the script.
+            # For the deployment, we just take the current conda env and run it.
+            # Hence we just handle the notebook case directly here.
+            script_path = os.path.join(os.path.join(dir_path, ".."), SCRIPT)
+            cmd = (
+                f"python {script_path} "
+                + f"--artifact-directory {artifact_directory} "
+                + extra_cmd
+            )
+            logger.info(f"Running in a notebook or NO_CONTAINER with command {cmd}")
+            run_command(cmd=cmd, shell=True)
+        else:
+            extra_cmd = (
+                f"--artifact-directory {DEFAULT_MODEL_DEPLOYMENT_FOLDER} " + extra_cmd
+            )
+            exit_code = self._run_with_conda_pack(
+                bind_volumes, extra_cmd, install=True, conda_uri=conda_path
+            )
+            if exit_code != 0:
+                raise RuntimeError(
+                    f"`predict` did not complete successfully. Exit code: {exit_code}. "
+                    f"Run with the --debug argument to view container logs."
+                )
+
+    def _get_conda_info_from_custom_metadata(self, ocid):
+        """
+        Get conda env info from custom metadata from model catalog.
+
+        Returns
+        -------
+        (str, str)
+            conda slug and conda path.
+        """
+        response = self.client.get_model(ocid)
+        custom_metadata = ModelCustomMetadata._from_oci_metadata(
+            response.data.custom_metadata_list
+        )
+        conda_slug, conda_path = None, None
+        if "CondaEnvironmentPath" in custom_metadata.keys:
+            conda_path = custom_metadata["CondaEnvironmentPath"].value
+        if "SlugName" in custom_metadata.keys:
+            conda_slug = custom_metadata["SlugName"].value
+        return conda_slug, conda_path
+
+    @staticmethod
+    def _get_conda_info_from_runtime(artifact_dir):
+        """
+        Get conda env info from runtime yaml file.
+
+        Returns
+        -------
+        (str, str)
+            conda slug and conda path.
+        """
+        runtime_yaml_file = os.path.join(artifact_dir, "runtime.yaml")
+        runtime_info = RuntimeInfo.from_yaml(uri=runtime_yaml_file)
+        conda_slug = (
+            runtime_info.model_deployment.inference_conda_env.inference_env_slug
+        )
+        conda_path = (
+            runtime_info.model_deployment.inference_conda_env.inference_env_path
+        )
+        return conda_slug, conda_path
+
+
+class LocalOperatorBackend(Backend):
+    """
+    The local operator backend to execute operator in the local environment.
+    Currently supported two scenarios:
+        * Running operator within local conda environment.
+        * Running operator within local container.
+
+    Attributes
+    ----------
+    runtime_config: (Dict)
+        The runtime config for the operator.
+    operator_config: (Dict)
+        The operator specification config.
+    operator_type: str
+        The type of the operator.
+    operator_info: OperatorInfo
+        The detailed information about the operator.
+    """
+
+    def __init__(
+        self, config: Optional[Dict], operator_info: OperatorInfo = None
+    ) -> None:
+        """
+        Instantiates the operator backend.
+
+        Parameters
+        ----------
+        config: (Dict)
+            The configuration file containing operator's specification details and execution section.
+        operator_info: (OperatorInfo, optional)
+            The operator's detailed information extracted from the operator.__init__ file.
+            Will be extracted from the operator type in case if not provided.
+        """
+        super().__init__(config=config or {})
+
+        self.runtime_config = self.config.get("runtime", {})
+        self.operator_config = {
+            **{
+                key: value
+                for key, value in self.config.items()
+                if key not in ("runtime", "infrastructure", "execution")
+            }
+        }
+        self.operator_type = self.operator_config.get("type")
+
+        self._RUNTIME_MAP = {
+            operator_runtime.ContainerRuntime.type: self._run_with_container,
+            operator_runtime.PythonRuntime.type: self._run_with_python,
+        }
+
+        self.operator_info = operator_info
+
+    def _run_with_python(self, **kwargs: Dict) -> int:
+        """Runs the operator within a local python environment.
+
+        Returns
+        -------
+        int
+            The operator's run exit code.
+        """
+
+        # build runtime object
+        runtime = operator_runtime.PythonRuntime.from_dict(
+            self.runtime_config, ignore_unknown=True
+        )
+
+        # run operator
+        operator_spec = json.dumps(self.operator_config)
+        sys.argv = [self.operator_info.type, "--spec", operator_spec]
+
+        logger.info(f"{'*' * 50} Runtime Config {'*' * 50}")
+        logger.info(runtime.to_yaml())
+
+        try:
+            runpy.run_module(self.operator_info.type, run_name="__main__")
+        except SystemExit as exception:
+            return exception.code
+        else:
+            return 0
+
+    def _run_with_container(self, **kwargs: Dict) -> int:
+        """Runs the operator within a container.
+
+        Returns
+        -------
+        int
+            The operator's run exit code.
+        """
+
+        # build runtime object
+        runtime: operator_runtime.ContainerRuntime = (
+            operator_runtime.ContainerRuntime.from_dict(
+                self.runtime_config, ignore_unknown=True
+            )
+        )
+        # prepare environment variables
+        env_vars = {
+            **{env["name"]: env["value"] for env in runtime.spec.env},
+            ENV_OPERATOR_ARGS: json.dumps(self.operator_config),
+        }
+
+        # prepare container volumes
+        bind_volumes = {}
+        for volume in runtime.spec.volume:
+            host_path, container_path = volume.split(":")
+            bind_volumes[host_path.lstrip().rstrip()] = {
+                "bind": container_path.lstrip().rstrip()
+            }
+
+        logger.info(f"{'*' * 50} Runtime Config {'*' * 50}")
+        logger.info(runtime.to_yaml())
+
+        return run_container(
+            image=runtime.spec.image,
+            bind_volumes=bind_volumes,
+            env_vars=env_vars,
+            command=f"'python3 -m {self.operator_info.type}'",
+        )
+
+    def run(self, **kwargs: Dict) -> Dict:
+        """Runs the operator."""
+
+        # extract runtime
+        runtime_type = self.runtime_config.get(
+            "type", operator_runtime.OPERATOR_LOCAL_RUNTIME_TYPE.PYTHON
+        )
+
+        if runtime_type not in self._RUNTIME_MAP:
+            raise RuntimeError(
+                f"Not supported runtime - {runtime_type} for local backend. "
+                f"Supported values: {self._RUNTIME_MAP.keys()}"
+            )
+
+        if not self.operator_info:
+            self.operator_info = OperatorLoader.from_uri(self.operator_type).load()
+
+        if self.config.get("dry_run"):
+            logger.info(
+                "The dry run option is not supported for "
+                "the local backends and will be ignored."
+            )
+
+        # run operator with provided runtime
+        exit_code = self._RUNTIME_MAP.get(runtime_type, lambda: None)(**kwargs)
+
+        if exit_code != 0:
+            raise RuntimeError(
+                f"Operation did not complete successfully. Exit code: {exit_code}. "
+                f"Run with the --debug argument to view logs."
+            )
+
+    def init(
+        self,
+        uri: Union[str, None] = None,
+        overwrite: bool = False,
+        runtime_type: Union[str, None] = None,
+        **kwargs: Dict,
+    ) -> Union[str, None]:
+        """Generates a starter YAML specification for the operator local runtime.
+
+        Parameters
+        ----------
+        overwrite: (bool, optional). Defaults to False.
+            Overwrites the result specification YAML if exists.
+        uri: (str, optional), Defaults to None.
+            The filename to save the resulting specification template YAML.
+        runtime_type: (str, optional). Defaults to None.
+                The resource runtime type.
+        **kwargs: Dict
+            The optional arguments.
+
+        Returns
+        -------
+        Union[str, None]
+            The YAML specification for the given resource if `uri` was not provided.
+            `None` otherwise.
+        """
+        runtime_type = runtime_type or operator_runtime.ContainerRuntime.type
+        if runtime_type not in operator_runtime_const.RUNTIME_TYPE_MAP:
+            raise ValueError(
+                f"Not supported runtime type {runtime_type}. "
+                f"Supported values: {operator_runtime_const.RUNTIME_TYPE_MAP.keys()}"
+            )
+
+        RUNTIME_KWARGS_MAP = {
+            operator_runtime.ContainerRuntime.type: {
+                "image": f"{self.operator_config['type']}:{self.operator_config['version']}",
+                "volume": [
+                    os.path.expanduser(
+                        os.path.dirname(self.config["execution"]["oci_config"])
+                    )
+                    + ":"
+                    + "/root/.oci"
+                ],
+                "env": [
+                    {
+                        "name": "operator",
+                        "value": f"{self.operator_config['type']}:{self.operator_config['version']}",
+                    }
+                ],
+            },
+            operator_runtime.PythonRuntime.type: {},
+        }
+
+        with AuthContext(auth=self.auth_type, profile=self.profile):
+            note = (
+                "# This YAML specification was auto generated by the "
+                "`ads operator init` command.\n"
+                "# The more details about the operator's runtime YAML "
+                "specification can be found in the ADS documentation:\n"
+                "# https://accelerated-data-science.readthedocs.io/en/latest \n\n"
+            )
+
+            return (
+                operator_runtime_const.RUNTIME_TYPE_MAP[runtime_type]
+                .init(**RUNTIME_KWARGS_MAP[runtime_type])
+                .to_yaml(
+                    uri=uri,
+                    overwrite=overwrite,
+                    note=note,
+                    **kwargs,
+                )
+            )