oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/opctl/config/merger.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+from string import Template
+from typing import Dict
+
+import yaml
+
+from ads.common.auth import AuthType, ResourcePrincipal
+from ads.opctl import logger
+from ads.opctl.config.base import ConfigProcessor
+from ads.opctl.config.utils import _DefaultNoneDict, read_from_ini
+from ads.opctl.constants import (
+    ADS_CONFIG_FILE_NAME,
+    ADS_DATAFLOW_CONFIG_FILE_NAME,
+    ADS_JOBS_CONFIG_FILE_NAME,
+    ADS_LOCAL_BACKEND_CONFIG_FILE_NAME,
+    ADS_ML_PIPELINE_CONFIG_FILE_NAME,
+    ADS_MODEL_DEPLOYMENT_CONFIG_FILE_NAME,
+    BACKEND_NAME,
+    DEFAULT_ADS_CONFIG_FOLDER,
+    DEFAULT_CONDA_PACK_FOLDER,
+    DEFAULT_NOTEBOOK_SESSION_CONDA_DIR,
+    DEFAULT_OCI_CONFIG_FILE,
+    DEFAULT_PROFILE,
+)
+from ads.opctl.utils import get_service_pack_prefix, is_in_notebook_session
+
+
+class ConfigMerger(ConfigProcessor):
+    """Merge configurations from command line args, YAML, ini and default configs.
+    The order of precedence is
+    command line args > YAML + conf.ini > .ads_ops/config.ini + .ads_ops/ml_job_config.ini or .ads_ops/dataflow_config.ini
+    Detailed examples can be found at the last section on this page:
+    https://bitbucket.oci.oraclecorp.com/projects/ODSC/repos/advanced-ds/browse/ads/opctl/README.md?at=refs%2Fheads%2Fads-ops-draft
+    """
+
+    def process(self, **kwargs) -> None:
+        for key, value in self.config.items():
+            if isinstance(value, str):  # Substitute only if the value is a string
+                self.config[key] = Template(value).safe_substitute(os.environ)
+
+        if "runtime" not in self.config:
+            self.config["runtime"] = {}
+
+        # 1. merge and overwrite values from command line args
+        self._merge_config_with_cmd_args(kwargs)
+        # 1.5 merge environment variables
+        # TODO
+
+        # 2. fill in values from conf file
+        self._fill_config_from_conf()
+
+        ads_config_path = os.path.abspath(
+            os.path.expanduser(
+                self.config["execution"].pop("ads_config", DEFAULT_ADS_CONFIG_FOLDER)
+            )
+        )
+        # 3. fill in values from default files under ~/.ads_ops
+        self._fill_config_with_defaults(ads_config_path)
+
+        self._config_flex_shape_details()
+
+        return self
+
+    def _merge_config_with_cmd_args(self, cmd_args: Dict) -> None:
+        # overwrite config with command line args
+        # if a command line arg value is None or empty collection, then it is ignored
+        def _overwrite(cfg, args):
+            for k, v in cfg.items():
+                if isinstance(v, dict):
+                    _overwrite(v, args)
+                elif k in args:
+                    if (
+                        isinstance(args[k], bool)
+                        or (isinstance(args[k], list) and len(args[k]) > 0)
+                        or args[k]
+                    ):
+                        cfg[k] = args.pop(k)
+                    else:
+                        args.pop(k)
+
+        _overwrite(self.config, cmd_args)
+
+        # save everything else from command line in "execution" section
+        if "execution" not in self.config:
+            self.config["execution"] = {}
+        for k, v in cmd_args.items():
+            if isinstance(v, bool) or (isinstance(v, list) and len(v) > 0) or v:
+                self.config["execution"][k] = v
+
+    def _fill_config_from_conf(self) -> None:
+        if self.config["execution"].get("conf_file"):
+            conf_file = self.config["execution"]["conf_file"]
+            conf_profile = self.config["execution"].get("conf_profile", DEFAULT_PROFILE)
+            logger.info(f"Reading from {conf_profile} using profile {conf_profile}")
+            parser = read_from_ini(conf_file)
+            extra_configs = dict(os.environ)
+            extra_configs.update(parser[conf_profile])
+            config_string = yaml.dump(self.config)
+            # _DefaultNoneDict is used so that if $variable is not found in a section in .ini, None value is filled in.
+            self.config = yaml.safe_load(
+                Template(config_string).substitute(_DefaultNoneDict(**extra_configs))
+            )
+
+    def _fill_config_with_defaults(self, ads_config_path: str) -> None:
+        exec_config = self._get_default_config()
+        exec_config_from_conf = self._get_config_from_config_ini(ads_config_path)
+        exec_config.update(
+            {k: v for k, v in exec_config_from_conf.items() if v is not None}
+        )
+        # set default auth
+        if not self.config["execution"].get("auth", None):
+            if ResourcePrincipal.supported():
+                self.config["execution"]["auth"] = (
+                    exec_config.get("auth") or AuthType.RESOURCE_PRINCIPAL
+                )
+            else:
+                self.config["execution"]["auth"] = (
+                    exec_config.get("auth") or AuthType.API_KEY
+                )
+        # determine profile
+        if self.config["execution"]["auth"] in (
+            AuthType.RESOURCE_PRINCIPAL,
+            AuthType.INSTANCE_PRINCIPAL,
+        ):
+            profile = self.config["execution"]["auth"].upper()
+            exec_config.pop("oci_profile", None)
+            self.config["execution"]["oci_profile"] = None
+        else:
+            profile = self.config["execution"].get("oci_profile") or exec_config.get(
+                "oci_profile"
+            )
+            self.config["execution"]["oci_profile"] = profile
+        # loading config for corresponding profile
+        infra_config = self._get_service_config(profile, ads_config_path)
+        if infra_config.get(
+            "conda_pack_os_prefix"
+        ):  # this is a field that appeared both in config.ini and ml_job_config.ini
+            exec_config["conda_pack_os_prefix"] = infra_config.pop(
+                "conda_pack_os_prefix"
+            )
+        for k, v in exec_config.items():
+            if v and not self.config["execution"].get(k):
+                self.config["execution"][k] = v
+        if not self.config.get("infrastructure"):
+            self.config["infrastructure"] = {}
+        for k, v in infra_config.items():
+            if v and not self.config["infrastructure"].get(k):
+                self.config["infrastructure"][k] = v
+
+    def _get_default_config(self) -> Dict:
+        if is_in_notebook_session():
+            conda_pack_os_prefix = get_service_pack_prefix()
+            return {
+                "oci_config": DEFAULT_OCI_CONFIG_FILE,
+                "oci_profile": DEFAULT_PROFILE,
+                "conda_pack_folder": DEFAULT_NOTEBOOK_SESSION_CONDA_DIR,
+                "conda_pack_os_prefix": conda_pack_os_prefix,
+            }
+        else:
+            return {
+                "oci_config": DEFAULT_OCI_CONFIG_FILE,
+                "oci_profile": DEFAULT_PROFILE,
+                "conda_pack_folder": DEFAULT_CONDA_PACK_FOLDER,
+            }
+
+    @staticmethod
+    def _get_config_from_config_ini(ads_config_folder: str) -> Dict:
+        if os.path.exists(os.path.join(ads_config_folder, ADS_CONFIG_FILE_NAME)):
+            parser = read_from_ini(
+                os.path.join(ads_config_folder, ADS_CONFIG_FILE_NAME)
+            )
+            return {
+                "oci_config": parser["OCI"].get("oci_config"),
+                "oci_profile": parser["OCI"].get("oci_profile"),
+                "auth": parser["OCI"].get("auth"),
+                "conda_pack_folder": parser["CONDA"].get("conda_pack_folder"),
+                "conda_pack_os_prefix": parser["CONDA"].get("conda_pack_os_prefix"),
+            }
+        else:
+            logger.debug(
+                f"{os.path.join(ads_config_folder, 'config.ini')} does not exist. No config loaded."
+            )
+            return {}
+
+    def _get_service_config(self, oci_profile: str, ads_config_folder: str) -> Dict:
+        backend = self.config["execution"].get("backend", None)
+        backend_config = {
+            BACKEND_NAME.JOB: ADS_JOBS_CONFIG_FILE_NAME,
+            BACKEND_NAME.DATAFLOW: ADS_DATAFLOW_CONFIG_FILE_NAME,
+            BACKEND_NAME.PIPELINE: ADS_ML_PIPELINE_CONFIG_FILE_NAME,
+            BACKEND_NAME.LOCAL: ADS_LOCAL_BACKEND_CONFIG_FILE_NAME,
+            BACKEND_NAME.MODEL_DEPLOYMENT: ADS_MODEL_DEPLOYMENT_CONFIG_FILE_NAME,
+        }
+        config_file = backend_config.get(backend, ADS_JOBS_CONFIG_FILE_NAME)
+
+        if os.path.exists(os.path.join(ads_config_folder, config_file)):
+            parser = read_from_ini(os.path.join(ads_config_folder, config_file))
+            if oci_profile in parser:
+                return parser[oci_profile]
+            if DEFAULT_PROFILE in parser:
+                return parser[DEFAULT_PROFILE]
+        else:
+            logger.debug(
+                f"{os.path.join(ads_config_folder, config_file)} does not exist. No config loaded."
+            )
+        return {}
+
+    def _config_flex_shape_details(self):
+        infrastructure = self.config["infrastructure"]
+        backend = self.config["execution"].get("backend", None)
+        if backend == BACKEND_NAME.JOB or backend == BACKEND_NAME.MODEL_DEPLOYMENT:
+            shape_name = infrastructure.get("shape_name", "")
+            if shape_name.endswith(".Flex"):
+                if (
+                    "ocpus" not in infrastructure
+                    or "memory_in_gbs" not in infrastructure
+                ):
+                    raise ValueError(
+                        "Parameters `ocpus` and `memory_in_gbs` must be provided for using flex shape. "
+                        "Call `ads opctl configure` to specify."
+                    )
+                infrastructure["shape_config_details"] = {
+                    "ocpus": infrastructure.pop("ocpus"),
+                    "memory_in_gbs": infrastructure.pop("memory_in_gbs"),
+                }
+        elif backend == BACKEND_NAME.DATAFLOW:
+            executor_shape = infrastructure.get("executor_shape", "")
+            driver_shape = infrastructure.get("driver_shape", "")
+            data_flow_shape_config_details = [
+                "driver_shape_memory_in_gbs",
+                "driver_shape_ocpus",
+                "executor_shape_memory_in_gbs",
+                "executor_shape_ocpus",
+            ]
+            # executor_shape and driver_shape must be the same shape family
+            if executor_shape.endswith(".Flex") or driver_shape.endswith(".Flex"):
+                for parameter in data_flow_shape_config_details:
+                    if parameter not in infrastructure:
+                        raise ValueError(
+                            f"Parameters {parameter} must be provided for using flex shape. "
+                            "Call `ads opctl configure` to specify."
+                        )
+                infrastructure["driver_shape_config"] = {
+                    "ocpus": infrastructure.pop("driver_shape_ocpus"),
+                    "memory_in_gbs": infrastructure.pop("driver_shape_memory_in_gbs"),
+                }
+                infrastructure["executor_shape_config"] = {
+                    "ocpus": infrastructure.pop("executor_shape_ocpus"),
+                    "memory_in_gbs": infrastructure.pop("executor_shape_memory_in_gbs"),
+                }

ads/opctl/config/resolver.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import base64
+import glob
+import json
+import os
+from typing import Dict, Tuple
+
+import yaml
+
+from ads.common.auth import create_signer
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.opctl import logger
+from ads.opctl.config.base import ConfigProcessor
+from ads.opctl.config.utils import NotSupportedError, convert_notebook
+from ads.opctl.constants import (
+    BACKEND_NAME,
+    ML_JOB_GPU_IMAGE,
+    ML_JOB_IMAGE,
+)
+from ads.opctl.utils import (
+    get_namespace,
+    get_region_key,
+    list_ads_operators,
+    parse_conda_uri,
+)
+
+
+class ConfigResolver(ConfigProcessor):
+    """Infer and fill in fields necessary for running an operator based on user inputs.
+    For instance, if user pass in an ADS operator name, we will fill in image name, command,
+    volumes to mount for local run, source folder path, etc.
+
+    Please check functions with name `_resolve_<thing>`.
+
+    List of things being resolved (and brief description):
+    - operator name (could be passed in directly or in YAML)
+    - conda (type is determined on whether a slug or uri is passed in. slug -> service, uri -> published. also do some additional checks.)
+    - source folder (mainly for ADS operator)
+    - command (docker command)
+    - image name (mainly for ADS operator)
+    - env vars
+    - mount volumes (for local runs)
+
+    There are additional comments in some of the functions below.
+    """
+
+    def __init__(self, config: Dict = None) -> None:
+        super().__init__(config)
+        self.ads_operators = list_ads_operators()
+
+    def process(self):
+        # this function should be run after merging configs
+        # conda pack scenarios --
+        # - user runs their own scripts and commands -> source_folder + entrypoint + (command)
+        # - user runs ADS operator -> name/YAML
+
+        # docker image scenarios --
+        # - user runs their own docker image -> image (to run) + (entrypoint) + (command)
+        # - user runs ADS operator -> name/YAML
+
+        if self.config["execution"].get("job_id"):
+            return self
+
+        if self.config.get("kind") == "pipeline":
+            # For pipelines, properties like the conda slug and source folder don't apply to the entire pipeline,
+            # so we can skip all this validation. Each individual step will have its own config with the required
+            # values set.
+            return self
+
+        self._resolve_operator_name()
+
+        if not (
+            self._is_ads_operator()
+            or self.config["execution"].get("source_folder")
+            or self.config["execution"].get("image")
+        ):
+            raise ValueError(
+                "Either an ADS operator or a source folder or a docker image needs to be given."
+            )
+
+        self._resolve_conda()
+        if not (
+            self.config["execution"].get("conda_slug")
+            or self.config["execution"].get("image")
+            or self.config["execution"]["backend"] == BACKEND_NAME.DATAFLOW
+        ):
+            raise ValueError(
+                "If not running an operator, conda pack info or image name needs to be given."
+            )
+        if self.config["execution"].get("conda_slug") and self.config["execution"].get(
+            "image"
+        ):
+            raise ValueError(
+                "Both conda pack info and image name are given. It is ambiguous which one to use."
+            )
+
+        self._resolve_source_folder_path()
+        self._resolve_entry_script()
+        self._resolve_command()
+        self._resolve_image_name()
+        self._resolve_env_vars()
+        self._resolve_mounted_volumes()
+        self._resolve_job_name()
+
+        return self
+
+    def _is_ads_operator(self) -> bool:
+        return self.config["execution"].get("operator_name") in self.ads_operators
+
+    def _resolve_operator_name(self) -> None:
+        self.config["execution"]["operator_name"] = self.config["execution"].get(
+            "operator_name"
+        ) or self.config.get("name")
+
+    @runtime_dependency(module="inflection", install_from=OptionalDependency.OPCTL)
+    def _resolve_source_folder_path(self) -> None:
+        # this should be run after resolve_operator_name()
+        # resolve ADS operator source folder path
+        exec_config = self.config["execution"]
+        if exec_config.get("source_folder"):
+            self.config["execution"]["source_folder"] = os.path.abspath(
+                os.path.normpath(exec_config["source_folder"])
+            )
+            if not os.path.exists(self.config["execution"]["source_folder"]):
+                raise FileNotFoundError(
+                    f"{self.config['execution']['source_folder']} is not found."
+                )
+        elif self._is_ads_operator():
+            curr_dir = os.path.dirname(os.path.abspath(__file__))
+            self.config["execution"]["source_folder"] = os.path.normpath(
+                os.path.join(
+                    curr_dir,
+                    "..",
+                    "operators",
+                    inflection.underscore(self.config["execution"]["operator_name"]),
+                )
+            )
+        else:
+            self.config["execution"]["source_folder"] = None
+
+    def _resolve_entry_script(self) -> None:
+        # this should be run after _resolve_source_folder_path
+        if not self._is_ads_operator():
+            if (
+                self.config["execution"].get("entrypoint")
+                and os.path.splitext(self.config["execution"]["entrypoint"])[1]
+                == ".ipynb"
+            ):
+                input_path = os.path.join(
+                    self.config["execution"]["source_folder"],
+                    self.config["execution"]["entrypoint"],
+                )
+                exclude_tags = self.config["execution"].get("exclude_tag")
+                self.config["execution"]["entrypoint"] = os.path.basename(
+                    convert_notebook(
+                        input_path,
+                        {},
+                        exclude_tags,
+                        overwrite=self.config["execution"].get("overwrite", False),
+                    )
+                )
+
+    def _resolve_command(self) -> None:
+        # this should be run after resolve_operator_name and resolve_conda
+        # resolve command for operators
+        exec_config = self.config["execution"]
+        if self._is_ads_operator():
+            logger.info("Building command for operator runs...")
+            operator_name = exec_config.get("operator_name")
+            command = f"-n {operator_name} -c {exec_config['oci_config']} -p {exec_config['oci_profile']}"
+            if "spec" in self.config:
+                encoded_spec = base64.b64encode(
+                    json.dumps(self.config["spec"]).encode()
+                ).decode()
+                command += f" -s {encoded_spec}"
+            self.config["execution"]["command"] = command
+        else:
+            # cmd_args is for running scripts, either locally, with jobs or dataflow; command is for docker command
+            self.config["execution"]["command"] = exec_config.get(
+                "cmd_args"
+            ) or exec_config.get("command")
+
+    def _resolve_conda(self) -> None:
+        # this should be run after resolve_operator_name
+        exec_config = self.config["execution"]
+        # two scenarios that indicate conda pack should be used:
+        # - conda slug is found
+        # - use-conda is turned on explicitly. in this case for ads operators, look up index.yaml to fill in conda info
+        slug = None
+        conda_type = None
+        if exec_config.get("conda_uri"):
+            slug = parse_conda_uri(exec_config["conda_uri"])[-1]
+            conda_type = "published"
+        elif exec_config.get("conda_slug"):
+            slug = exec_config["conda_slug"]
+            conda_uri, conda_type = self._get_conda_uri_and_type(slug)
+            self.config["execution"]["conda_uri"] = conda_uri
+            if conda_type and conda_type == "data_science":
+                conda_type = "service"
+            conda_type = conda_type or "service"
+        elif exec_config.get("use_conda", False):
+            operator_name = exec_config.get("operator_name")
+            if self._is_ads_operator():
+                if self.ads_operators[operator_name].get("conda_slug"):
+                    slug = self.ads_operators[operator_name]["conda_slug"]
+                    conda_type = "service"
+                else:
+                    raise NotSupportedError(
+                        f"Conda pack is not supported for ADS operator {operator_name}."
+                    )
+        else:
+            logger.info("Conda pack slug info is not found.")
+        self.config["execution"]["conda_slug"] = slug
+        self.config["execution"]["conda_type"] = conda_type
+
+    def _get_conda_uri_and_type(self, slug: str) -> Tuple[str, str]:
+        conda_pack_path = os.path.join(
+            os.path.expanduser(self.config["execution"]["conda_pack_folder"]), slug
+        )
+        if os.path.exists(conda_pack_path):
+            manifest_path = glob.glob(os.path.join(conda_pack_path, "*_manifest.yaml"))[
+                0
+            ]
+            with open(manifest_path) as f:
+                manifest = yaml.safe_load(f.read())["manifest"]
+            return manifest.get("pack_uri"), manifest.get("type")
+        else:
+            return None, None
+
+    def _resolve_image_name(self) -> None:
+        # this should be run after resolve_conda() and resolve_command()
+        exec_config = self.config["execution"]
+        if exec_config.get("conda_slug", None):
+            self.config["execution"]["image"] = (
+                ML_JOB_GPU_IMAGE if exec_config.get("gpu", False) else ML_JOB_IMAGE
+            )
+        elif self._is_ads_operator():
+            image = self.ads_operators[exec_config["operator_name"]].get("image")
+            oci_auth = create_signer(
+                self.config["execution"].get("auth"),
+                self.config["execution"].get("oci_config"),
+                self.config["execution"].get("oci_profile"),
+            )
+            region_key = get_region_key(oci_auth).lower()
+            if exec_config.get("namespace"):
+                namespace = exec_config["namespace"]
+            else:
+                namespace = get_namespace(oci_auth)
+            if self.config.get("infrastructure", {}).get("docker_registry"):
+                self.config["execution"]["image"] = os.path.join(
+                    self.config["infrastructure"]["docker_registry"], image
+                )
+            else:
+                self.config["execution"]["image"] = (
+                    f"{region_key}.ocir.io/{namespace}/{image}"
+                )
+
+    def _resolve_env_vars(self) -> None:
+        env_vars = self.config["execution"].get("env_vars", {})
+        if self.config["execution"].get("env_var"):
+            for ev in self.config["execution"]["env_var"]:
+                if "=" in ev:
+                    k, v = ev.split("=")
+                    env_vars[k.strip()] = v.strip()
+                elif ev in os.environ:
+                    env_vars[ev] = os.environ[ev]
+                else:
+                    raise ValueError(
+                        "Please provide environment variable as VAR=value or "
+                        "give an environment variable that is set locally."
+                    )
+        self.config["execution"]["env_vars"] = env_vars
+
+    def _resolve_mounted_volumes(self) -> None:
+        volumes = self.config["execution"].get("volumes", {})
+        if self.config["execution"].get("volume"):
+            for v in self.config["execution"]["volume"]:
+                host, remote = v.split(":")
+                if not os.path.exists(host.strip()):
+                    raise FileNotFoundError(f"{host} does not exist")
+                volumes[os.path.abspath(host.strip())] = {"bind": remote.strip()}
+        self.config["execution"]["volumes"] = volumes
+
+    def _resolve_job_name(self) -> None:
+        # this should be run after resolve_operator_name
+        if not self.config["execution"].get("job_name"):
+            if self.config["execution"].get("operator_name"):
+                self.config["execution"]["job_name"] = self.config["execution"][
+                    "operator_name"
+                ]

ads/opctl/config/utils.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import configparser
+import os
+from typing import List
+
+import urllib.parse
+import fsspec
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+class OperatorNotFound(Exception):   # pragma: no cover
+    pass
+
+
+class CondaPackInfoNotProvided(Exception):   # pragma: no cover
+    pass
+
+
+class NotSupportedError(Exception):   # pragma: no cover
+    pass
+
+
+class _DefaultNoneDict(dict):
+    def __missing__(self, key):
+        return ""
+
+
+def read_from_ini(path: str) -> configparser.ConfigParser:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"File {path} is not found.")
+    parser = configparser.ConfigParser(default_section=None)
+    parser.optionxform = str  # preserve case
+    parser.read(path)
+    return parser
+
+
+@runtime_dependency(module="nbformat", install_from=OptionalDependency.OPCTL)
+@runtime_dependency(module="nbconvert", install_from=OptionalDependency.OPCTL)
+def convert_notebook(
+    input_path,
+    auth,
+    exclude_tags: List = None,
+    output_path: str = None,
+    overwrite: bool = False,
+) -> str:
+    with fsspec.open(input_path, **auth) as f:
+        nb = nbformat.reads(f.read(), nbformat.NO_CONVERT)
+
+    from nbconvert.preprocessors import TagRemovePreprocessor
+    from nbconvert.exporters import PythonExporter
+
+    exporter = PythonExporter()
+    if exclude_tags:
+        exporter.register_preprocessor(
+            TagRemovePreprocessor(remove_cell_tags=exclude_tags), enabled=True
+        )
+
+    output, _ = exporter.from_notebook_node(nb)
+    output = output.strip("\n")
+    output_path = output_path or os.path.splitext(input_path)[0] + ".py"
+    file_system_clz = fsspec.get_filesystem_class(
+        urllib.parse.urlparse(output_path).scheme or "file"
+    )
+    file_system = file_system_clz(**auth)
+    if not overwrite and file_system.exists(output_path):
+        raise FileExistsError(
+            f"{output_path} exists. Please rename your notebook or use overwrite option."
+        )
+    with fsspec.open(output_path, mode="wb", **auth) as f:
+        f.write(output.encode("utf-8"))
+    return output_path

ads/opctl/config/validator.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.opctl.config.base import ConfigProcessor
+
+
+class ConfigValidator(ConfigProcessor):
+    def process(self):
+        # TODO: add validation using pydantic + datamodel-code-generator in future PR
+
+        # called to update command, part of which is encoded spec
+        # and spec might have been updated during the call above
+        # self.["execution"]["command"] = _resolve_command(self.config)
+        return self