oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/opctl/distributed/common/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/distributed/common/abstract_cluster_provider.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import ipaddress
+import json
+import os
+import sys
+from time import sleep, time, time_ns
+from urllib.parse import urlparse
+import subprocess
+
+
+import ads
+import fsspec
+import oci
+import pandas as pd  # Have to find a better way for timedelta
+import psutil
+from ads.jobs import Job
+from ads.opctl.distributed.common import cluster_config_helper
+from ads.jobs.templates.driver_oci import GitSSHKey, GitManager
+from ads.jobs.templates.driver_utils import JobRunner
+from ads.jobs.builders.runtimes.artifact import Artifact
+
+
+class ClusterProvider:
+    """
+    Provides contract for implementing Framework specific Cluster Life Cycle Manager.
+
+    The main node is identified by the environment variable - `MAIN`
+    The worker node is identified by the environment variable - `WORKER`
+
+    The worker and main coordinate cluster configuration using the directory provided via `WORK_DIR`. The main node creates config.json in the `WORK_DIR`.
+    The worker nodes polls the config.json and exports the configuration as environment variables
+    """
+
+    SYNC_SCRIPT_PATH = "/etc/datascience/sync.sh"
+
+    DEFAULT_CODE_DIR = "/code"
+
+    def __init__(self, mode, ephemeral=True, life_span="0h", work_dir=""):
+        self.mode = mode
+        self.start_time = time()
+        self.ephemeral = ephemeral
+        self.authinfo = self.get_oci_auth()
+        self.ip = self.find_self_ip(self.authinfo)
+        self.end_time = (
+            pd.Timedelta(int(life_span[:-1]), life_span[-1]).total_seconds()
+            + self.start_time
+        )
+        self.work_dir = work_dir
+        self._get_my_work_dir()
+
+        self.main_config_file = os.path.join(
+            self.my_work_dir, f"MAIN_config.json"
+        )  # In case of worker, we have two config file, worker generated and main config
+        self.time_out = int(
+            os.environ.get("OCI__TIMEOUT", "600")
+        )  # Time out to wait for the config file
+
+        self.fetch_code()
+
+        self.setup_configuration()  # Write cluster configuration to `work_dir`. Eg. IP address of scheduler, etc.
+
+        # self.tmpdir = os.environ["tmpdir"]
+
+        self.stop_file = os.path.join(
+            self.my_work_dir, "stop"
+        )  # Control file to instruct cluster to stop
+
+        scheme = urlparse(
+            self.stop_file,
+        ).scheme
+        self.scheme = scheme
+
+        self.execution_failure = False
+        self.code_execution_complete = False
+        self.sync()
+
+    def sync(self, loop=True):
+        sync_artifacts = os.environ.get("SYNC_ARTIFACTS", 0)
+        print(f"sync_artifacts - {sync_artifacts}")
+        if sync_artifacts == "1":
+            bkt_name, prefix = self.get_sync_loc()
+            if bkt_name is None:
+                print(
+                    "WARNING: Sync 'WORKSPACE', 'WORKSPACE_PREFIX' or 'work_dir' not configured. Skipping Sync"
+                )
+                return
+            sync_script_fn = self.SYNC_SCRIPT_PATH
+            if not os.path.exists(sync_script_fn):
+                sync_script = self.get_sync_script()
+                self.create_sync_script(sync_script_fn, sync_script)
+            subprocess.Popen(
+                [sync_script_fn, bkt_name, prefix, "-l"]
+            ) if loop else subprocess.Popen([sync_script_fn, bkt_name, prefix])
+        if not loop:
+            sleep_duration = int(os.environ.get("POST_PROCESSING_WAIT", 60))
+            print(f"post processing wait..{sleep_duration} seconds")
+            sleep(sleep_duration)
+            print("..")
+
+    def get_sync_loc(self):
+        bckt_name = os.environ.get("WORKSPACE")
+        pfx = os.environ.get("WORKSPACE_PREFIX")
+        if bckt_name is None:
+            scheme = urlparse(self.work_dir)
+            if scheme.scheme == "oci":
+                bckt_name = scheme.netloc.split("@")[0]
+                pfx = scheme.path
+                pfx = pfx.strip("//")
+        return bckt_name, pfx
+
+    def profile_cmd(self):
+        profile = os.environ.get("PROFILE", "0")
+        cmd = []
+        if profile == "1":
+            print("Profiler ON")
+            cmd = os.environ.get("PROFILE_CMD").split(" ")
+        return cmd
+
+    @staticmethod
+    def create_sync_script(sync_script_fn, sync_script):
+        sync_script_fn_obj = open(sync_script_fn, "w")
+        sync_script_fn_obj.write(sync_script)
+        sync_script_fn_obj.close()
+        os.chmod(sync_script_fn, 755)
+
+    def get_sync_script(self):
+        return sync_script_str
+
+    def fetch_code(self):
+        runtime_type = os.environ.get(cluster_config_helper.OCI__RUNTIME_TYPE)
+        if not runtime_type:
+            return
+        delegates = {"git": self._fetch_git, "remote": self._fetch_remote}
+        if runtime_type not in delegates:
+            raise ValueError(f"Runtime type {runtime_type} is not supported.")
+        if cluster_config_helper.OCI__CODE_DIR not in os.environ:
+            os.environ[cluster_config_helper.OCI__CODE_DIR] = self.DEFAULT_CODE_DIR
+        delegates[runtime_type](
+            code_dir=os.environ.get(cluster_config_helper.OCI__CODE_DIR)
+        )
+
+    def _fetch_git(self, code_dir):
+        uri = os.environ.get(cluster_config_helper.OCI__RUNTIME_URI)
+        branch = os.environ.get(cluster_config_helper.OCI__RUNTIME_GIT_BRANCH)
+        commit = os.environ.get(cluster_config_helper.OCI__RUNTIME_GIT_COMMIT)
+        secret_ocid = os.environ.get(cluster_config_helper.OCI__RUNTIME_GIT_SECRET_ID)
+        # with GitSSHKey does nothing if secret_ocid is None or empty
+        with GitSSHKey(secret_ocid):
+            GitManager(uri, code_dir=code_dir).fetch_repo().checkout_code(
+                branch=branch, commit=commit
+            )
+        JobRunner(code_dir=code_dir).setup_python_path(
+            python_path=os.environ.get(cluster_config_helper.OCI__RUNTIME_PYTHON_PATH),
+        )
+
+    def _fetch_remote(self, code_dir):
+        uri = os.environ.get(cluster_config_helper.OCI__RUNTIME_URI)
+        Artifact.copy_from_uri(uri, code_dir)
+
+    def run_code(self):
+        # Sub-class should implement this method to run code and,
+        # set code_execution_complete to True after running code.
+        self.code_execution_complete = True
+
+    @property
+    def stop_filesystem(self):
+        authinfo = {}
+        if self.scheme == "oci":
+            authinfo = self.get_oci_auth()
+        return fsspec.filesystem(
+            self.scheme, **authinfo
+        )  # FileSystem class corresponding to the URI scheme.
+
+    def _get_my_work_dir(self):
+        """
+        Get the work dir subfolder for the current running job
+        """
+        # Use "Undefined" as job identifier when not running on OCI jobs.
+        self.jobDefID = os.environ.get("JOB_OCID", "Undefined")
+        # Use time as job run identifier when not running on OCI jobs
+        self.jobRunID = os.environ.get("JOB_RUN_OCID", str(time_ns()))
+
+        self.my_work_dir = os.path.join(self.work_dir, self.jobDefID)
+        self.config_file = os.path.join(
+            self.my_work_dir,
+            "MAIN_config.json"
+            if self.mode == "MAIN"
+            else f"{self.mode}_{self.jobRunID}_config.json",
+        )  # Config file location
+
+    def export_config_files(self):
+        """
+        By default only exports configuration generated by main. This behavior can be overridden.
+        """
+        return [self.main_config_file if self.mode == "WORKER" else self.config_file]
+
+    def reached_endoflife(self):
+        # TODO We can get rid of this method as JobRun takes parameter to set
+        # the max run time. We dont have to handle this here.
+        return (self.end_time - self.start_time) <= 0
+
+    def get_oci_auth(self):
+        profile = os.environ.get("OCI_CONFIG_PROFILE") or os.environ.get(
+            "OCIFS_CONFIG_PROFILE"
+        )
+        ads.set_auth(
+            os.environ.get("OCI_IAM_TYPE", "resource_principal"),
+            profile=profile or "DEFAULT",
+        )
+        authinfo = ads.common.auth.default_signer()
+        return authinfo
+
+    @classmethod
+    def find_self_ip(cls, authinfo):
+        """
+        Identify IP address by finding which of the host IP intersects with the CIDR block of the subnet associated with the JOB_OCID
+        """
+        if os.environ.get("JOB_OCID"):
+            job_ocid = os.environ["JOB_OCID"]
+            jobDef = Job.from_datascience_job(job_ocid)
+            subnet_id = jobDef.infrastructure.subnet_id
+            core_client = oci.core.VirtualNetworkClient(**authinfo)
+            cidr = core_client.get_subnet(subnet_id).data.cidr_block
+
+            for interface, snics in psutil.net_if_addrs().items():
+                ip = snics[0].address
+                if ipaddress.ip_address(ip) in ipaddress.ip_network(cidr):
+                    print(f"IP address: {ip}")
+                    os.environ["GLOO_SOCKET_IFNAME"] = interface
+                    os.environ["NCCL_SOCKET_IFNAME"] = interface
+                    return ip
+            print("IP ADDRESS NOT FOUND!!")
+            return None
+        else:
+            import socket
+
+            hostname = socket.gethostname()
+            ip = socket.gethostbyname(hostname)
+            print(f"IP address: {ip}")
+            return ip
+
+    def basic_configuration(
+        self,
+    ) -> dict:
+        """
+        Prepares basic set of configuration which is framework independent.
+        This configuration is decorated later by `configuration` method implemented by framework specific implementations
+        """
+        config = {}
+        config["OCI__MAIN_IP" if self.mode == "MAIN" else "OCI__WORKER_IP"] = self.ip
+        config["tmpdir"] = self.my_work_dir
+        return config
+
+    def configuration(self, conf={}) -> dict:
+        """
+        Provides the configuration information of the cluster.
+
+        conf:
+            Contains generic information about the cluster, generated using `basic_configuration`. Eg. IP Address of the main process
+        """
+        return None
+
+    def setup_extra_configs(self, conf: dict):
+        return None
+
+    def setup_configuration(self, config: dict = None):
+        """
+        Writes the configuration information into location provided by `work_dir`
+
+        config:
+            dictionary containing configuration information that needs to be shared with the workers
+            if config is None, then this method calls `self.configuration` and saves the configuration
+        work_dir:
+            Could be any valid URI supported by fsspec
+
+        """
+        # tmpdir = self.my_work_dir
+        # self.tmpdir = tmpdir
+        config = config or self.configuration()
+        print(
+            f"Writing configuration: {config} to file: {self.config_file}", flush=True
+        )
+        with fsspec.open(self.config_file, "w", **self.authinfo) as conf:
+            conf.write(json.dumps(config))
+
+    def export_configuration(self, files):
+        """
+        Read the configuration in the files array  and export to environment variable
+        """
+        print(f"{self.mode}. Config File: {files}", flush=True)
+        for file in files:
+            with fsspec.open(file, **self.authinfo) as conf:
+                config = json.loads(conf.read())
+                print(f"Loading config: {config}", flush=True)
+            for k in config:
+                os.environ[k] = str(config[k])
+
+    def expected_worker_count(
+        self,
+    ):
+        return int(os.environ.get("OCI__WORKER_COUNT", 1))
+
+    def fetch_all_worker_info(self):
+        """
+        Fetchs all the worker configs
+        In some cluster the main requires information about all worker IPs apriori. This method maybe useful in such situation.
+        """
+        files = self.stop_filesystem.ls(
+            f"{self.my_work_dir}/WORKER*config.json", refresh=True
+        )
+        worker_details = {}
+        for file in files:
+            with open(file) as wcf:
+                worker_details[file] = json.loads(wcf.read())
+        return worker_details
+
+    def start_main(self):
+        """
+        Implement this for starting the main process. Eg. `scheduler` for Dask
+        """
+        pass
+
+    def start_worker(self):
+        """
+        Implement this for starting the worker process. Eg. `dask-worker` for Dask
+        """
+        pass
+
+    def start_ps(self):
+        """
+        Implement this for starting the ps process. Eg. `tf-parameter-server` for tensorflow
+        """
+        pass
+
+    def start(self):
+        """
+        Starts the cluster processes
+        """
+        if self.mode == "MAIN":  # Check if the docker is started in scheduler mode
+            print(f"Starting main process", flush=True)
+            self.start_main()
+        elif self.mode == "WORKER":  # Check if the docker is staretd in worker mode
+            print(f"Starting worker process", flush=True)
+            self.when_ready(self.start_worker)
+        elif self.mode == "PS":  # Check if the docker is staretd in worker mode
+            print(f"Starting ps process", flush=True)
+            self.when_ready(self.start_ps)
+        else:
+            print(f"Not a valid mode: {self.mode}", flush=True)
+            raise Exception("Not a valid mode")
+
+    def check_cluster_status(self):
+        pass
+
+    def execution_failed(self):
+        """
+        Invoked when code submitted to epheramal cluster fails. Calling this method sets the cluster tearable state
+        """
+        self.execution_failure = True
+
+    def tearable(self):
+        """
+        Checks if cluster is ready for tear down.
+        If there is a `stop` file in the tmp directory then stop.
+        If cluster is ephemeral type, check if code execution is complete
+        If TTL is reached then stop
+        """
+
+        if self.stop_filesystem.exists(self.stop_file) or self.execution_failure:
+            print(
+                f"Stopping the process. Reason: {'Code Execution Failure' if self.execution_failure else 'Stop file found'}",
+                flush=True,
+            )
+            return True
+
+        if self.ephemeral:
+            return self.code_execution_complete
+        else:
+            return self.reached_endoflife()
+
+    def stop(self):
+        """
+        Writes stop file and exits.
+        """
+        if not self.stop_filesystem.exists(self.stop_file):
+            print("Stop file not found. Writing stop file....", flush=True)
+            authinfo = {}
+            if self.scheme == "oci":
+                authinfo = self.get_oci_auth()
+            with fsspec.open(self.stop_file, "w", **authinfo) as sf:
+                sf.write("stop")
+
+    def ready(self):
+        if self.stop_filesystem.exists(self.main_config_file):
+            return True
+        else:
+            return False
+
+    def when_ready(self, func, *args, **kwargs):
+        start_time = time()
+        while not self.ready():
+            sleep(10)
+            if time() - start_time > self.time_out:
+                raise Exception(
+                    "Timed out waiting to be in ready status. Likely cause configuration is missing in the `WORK_DIR`"
+                )
+
+        func(*args, **kwargs)
+
+
+sync_script_str = """#!/bin/bash
+
+loop=false
+
+if [ "$3" == "-l" ]; then
+  loop=true;
+fi
+echo "loop: $loop"
+sleep_duration=${SYNC_SLEEP:-60}
+if [ "$OCI_IAM_TYPE" = 'api_key' ]; then
+  auth_method=api_key
+else
+  auth_method=resource_principal
+fi
+echo "auth method: $auth_method"
+echo "OCI__SYNC_DIR dir: $OCI__SYNC_DIR"
+echo "sleep duration is $sleep_duration"
+
+bucket=$1
+prefix=$2/sync/$JOB_OCID/$JOB_RUN_OCID/
+
+while true; do
+  if [[ -d $OCI__SYNC_DIR && -n "$(ls -A $OCI__SYNC_DIR)" ]]; then
+    echo "syncing $OCI__SYNC_DIR to $bucket/$prefix"
+    $HOME/bin/oci os object sync --auth $auth_method --bucket-name $bucket --prefix $prefix --src-dir $OCI__SYNC_DIR
+  else
+    echo "nothing to sync in $OCI__SYNC_DIR"
+  fi
+  if [ "$loop" = false ] ; then
+    break
+  fi
+  sleep $sleep_duration
+done
+    """

ads/opctl/distributed/common/abstract_framework_spec_builder.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABCMeta
+from typing import List
+
+
+class AbstractFrameworkSpecBuilder(metaclass=ABCMeta):
+    """
+    Provides contract for implementing Framework specific Cluster Spec Builder
+
+    In the example of jobs, this class handles adding framework specific environment variables to the job definition.
+
+    NOTE: This class is not invoked while the cluster is running. Only after a call to `ads opctl`.
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def update(self):
+        new_env_vars = []
+        # Call DaskFramework().build_env_vars
+        # FrameworkFactory.get("kind").build_env_vars("spec, framework")
+        user_entrypoint = self.config["spec"]["Runtime"]["spec"]["entrypoint"]
+        args = self.config["spec"]["Runtime"]["spec"]["args"]
+
+        # TODO:
+        # kwargs = self.config["spec"]["Runtime"]["spec"]["kwargs"]
+
+        nanny_port = self.config["spec"]["Framework"]["spec"]["schedulerConfig"][
+            "nannyPortRange"
+        ]
+        worker_port = self.config["spec"]["Framework"]["spec"]["schedulerConfig"][
+            "workerPortRange"
+        ]
+        worker_timeout = self.config["spec"]["Framework"]["spec"]["workerConfig"][
+            "timeout"
+        ]
+
+        framework = self.config["spec"]["Framework"]["kind"]
+        work_dir = self.config["spec"]["Framework"]["spec"]["workDir"]
+        ephemeral = self.config["spec"]["Infrastructure"]["spec"]["cluster"][
+            "ephemeral"
+        ]
+        worker_count = self.config["spec"]["Infrastructure"]["spec"]["cluster"][
+            "numWorkers"
+        ]
+
+        new_env_vars.extend(
+            [
+                {"name": "OCI__ENTRY_SCRIPT", "value": user_entrypoint},
+                {"name": "OCI__ENTRY_SCRIPT_ARGS", "value": args},  # TODO check with QQ
+                {
+                    "name": "OCI__ENTRY_SCRIPT_KWARGS",
+                    "value": None,
+                },  # TODO check with QQ
+                {"name": "OCI__NANNY_PORT", "value": nanny_port},
+                {"name": "OCI__WORKER_PORT", "value": worker_port},
+                {"name": "OCI__WORKER_TIMEOUT", "value": worker_timeout},
+                {"name": "OCI__WORKER_COUNT", "value": worker_count},
+                {"name": "OCI__CLUSTER_TYPE", "value": framework},
+                {
+                    "name": "OCI__EPHEMERAL",
+                    "value": ephemeral,
+                },  # TODO: remove from yaml or build out
+                {"name": "OCI__LIFE_SPAN", "value": ""},  # TODO: for testing later
+                {"name": "OCI__WORK_DIR", "value": work_dir},
+            ]
+        )
+
+        new_env_vars = self._add_framework_envs(new_env_vars)
+        return update_env_vars(self.config, new_env_vars)
+
+    def _add_framework_envs(self, new_env_vars: List):
+        # Overwrite this function in your subclass in order to add/subtract environment variables
+        return new_env_vars
+
+
+def update_env_vars(config, env_vars: List):
+    """
+    env_vars: List, should be formatted as [{"name": "OCI__XXX", "value": YYY},]
+    """
+    # TODO move this to a class which checks the version, kind, type, etc.
+    config["spec"]["Runtime"]["spec"]["environmentVariables"].extend(env_vars)
+    return config

ads/opctl/distributed/common/cluster_config_helper.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+OCI__RUNTIME_TYPE = "OCI__RUNTIME_TYPE"
+OCI__RUNTIME_URI = "OCI__RUNTIME_URI"
+OCI__RUNTIME_PYTHON_PATH = "OCI__RUNTIME_PYTHON_PATH"
+OCI__RUNTIME_GIT_BRANCH = "OCI__RUNTIME_GIT_BRANCH"
+OCI__RUNTIME_GIT_COMMIT = "OCI__RUNTIME_GIT_COMMIT"
+OCI__RUNTIME_GIT_SECRET_ID = "OCI__RUNTIME_GIT_SECRET_ID"
+OCI__CODE_DIR = "OCI__CODE_DIR"
+
+
+class ClusterConfigToJobSpecConverter:
+    def __init__(self, cluster_info):
+        self.cluster_info = cluster_info
+
+    def job_def_info(self):
+        job = {}
+        job["infrastructure"] = self.cluster_info.infrastructure
+        cluster = self.cluster_info.cluster
+        job["image"] = cluster.image or cluster.main.image or cluster.worker.image
+        job["name"] = cluster.name or self.cluster_info.infrastructure.get(
+            "displayName"
+        )
+        job["envVars"] = cluster.config.envVars
+        job["envVars"]["OCI__WORK_DIR"] = cluster.work_dir
+        job["envVars"]["OCI__EPHEMERAL"] = cluster.ephemeral
+        job["envVars"]["OCI__CLUSTER_TYPE"] = cluster.type.upper()
+        job["envVars"]["OCI__WORKER_COUNT"] = (
+            cluster.worker.replicas if cluster.worker is not None else 0
+        )
+        if cluster.ps is not None:
+            job["envVars"]["OCI__PS_COUNT"] = cluster.ps.replicas
+
+        job["envVars"]["OCI__START_ARGS"] = cluster.config.cmd_args.strip()
+        job["envVars"]["OCI__ENTRY_SCRIPT"] = self.cluster_info.runtime.entry_point
+
+        if self.cluster_info.runtime.type:
+            optional_runtime_fields = [
+                {"attr": "type", "env": OCI__RUNTIME_TYPE},
+                {"attr": "uri", "env": OCI__RUNTIME_URI},
+                {"attr": "branch", "env": OCI__RUNTIME_GIT_BRANCH},
+                {"attr": "commit", "env": OCI__RUNTIME_GIT_COMMIT},
+                {"attr": "git_secret_id", "env": OCI__RUNTIME_GIT_SECRET_ID},
+                {"attr": "code_dir", "env": OCI__CODE_DIR},
+                {"attr": "python_path", "env": OCI__RUNTIME_PYTHON_PATH},
+            ]
+            for field in optional_runtime_fields:
+                val = getattr(self.cluster_info.runtime, field["attr"], None)
+                if val:
+                    job["envVars"][field["env"]] = val
+
+        runtime_args = self.cluster_info.runtime.args
+        if isinstance(self.cluster_info.runtime.args, list):
+            runtime_args = " ".join([str(v) for v in self.cluster_info.runtime.args])
+        if runtime_args:
+            job["envVars"]["OCI__ENTRY_SCRIPT_ARGS"] = runtime_args
+        if self.cluster_info.runtime.kwargs:
+            job["envVars"][
+                "OCI__ENTRY_SCRIPT_KWARGS"
+            ] = self.cluster_info.runtime.kwargs
+        job["envVars"].update(self.cluster_info.runtime.envVars)
+        job["envVars"] = {k: str(job["envVars"][k]) for k in job["envVars"]}
+
+        if self.cluster_info.cluster.certificate:
+            job["envVars"][
+                "OCI__CERTIFICATE_OCID"
+            ] = self.cluster_info.cluster.certificate.cert_ocid
+            job["envVars"][
+                "OCI__CERTIFICATE_KEY_DOWNLOAD_LOCATION"
+            ] = self.cluster_info.cluster.certificate.key_download_location
+            job["envVars"][
+                "OCI__CERTIFICATE_DOWNLOAD_LOCATION"
+            ] = self.cluster_info.cluster.certificate.cert_download_location
+            job["envVars"][
+                "OCI__CERTIFICATE_AUTHORITY_OCID"
+            ] = self.cluster_info.cluster.certificate.ca_ocid
+            job["envVars"][
+                "OCI__CA_DOWNLOAD_LOCATION"
+            ] = self.cluster_info.cluster.certificate.ca_download_location
+
+        return job
+
+    def job_run_info(self, jobType):
+        jobrun = {}
+        jobTypeConfig = getattr(self.cluster_info.cluster, jobType)
+        if jobTypeConfig is not None:
+            jobrun["name"] = jobTypeConfig.name or jobType
+            jobrun["envVars"] = jobTypeConfig.config.envVars
+            if jobTypeConfig.config.cmd_args:
+                jobrun["envVars"][
+                    "OCI__START_ARGS"
+                ] = jobTypeConfig.config.cmd_args.strip()
+
+            jobrun["envVars"]["OCI__MODE"] = jobType.upper()
+            jobrun["envVars"] = {
+                k: str(jobrun["envVars"][k]) for k in jobrun["envVars"]
+            }
+        return jobrun

ads/opctl/distributed/common/cluster_provider_factory.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class ClusterProviderFactory:
+    """
+    Factory class for creating provider instance.
+    """
+
+    provider = {}
+
+    @classmethod
+    def register(cls, cluster_type, provider_class):
+        ClusterProviderFactory.provider[cluster_type.upper()] = provider_class
+
+    @staticmethod
+    def get_provider(key, *args, **kwargs):
+        return ClusterProviderFactory.provider[key](*args, **kwargs)