oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/config/versioner.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import base64
-import json
-import os
-from typing import Dict
-from typing import Tuple
-
-import inflection
-import yaml
-import glob
-
-from ads.common.auth import create_signer
-from ads.opctl import logger
-from ads.opctl.config.base import ConfigProcessor
-from ads.opctl.config.utils import NotSupportedError, convert_notebook
-from ads.opctl.constants import (
-    ML_JOB_GPU_IMAGE,
-    ML_JOB_IMAGE,
-)
-from ads.opctl.utils import (
-    list_ads_operators,
-    parse_conda_uri,
-    get_region_key,
-    get_namespace,
-)
-
-
-YAML_STRUCTURES = {
-    "v0": {
-        "infrastructure": ["infrastructure"],
-        "image": ["execution", "image"],
-    },
-    "v1": {
-        "infrastructure": ["spec", "Infrastructure", "spec"],
-    },
-}
-
-
-class ConfigVersioner(ConfigProcessor):
-    """
-    ads opctl supports multiple yaml file types, kinds and versions.
-    Each has its own slightly different yaml structure.
-    This goal of this class is to provide a translation from the generic term for a variable
-        to it's specific path in it's yaml.
-    """
-
-    def __init__(self, config: Dict = None) -> None:
-        super().__init__(config)
-        self.ads_operators = list_ads_operators()
-
-    def process(self):
-        # this function should be run after merging configs
-        # conda pack scenarios --
-        # - user runs their own scripts and commands -> source_folder + entrypoint + (command)
-        # - user runs ADS operator -> name/YAML
-
-        # docker image scenarios --
-        # - user runs their own docker image -> image (to run) + (entrypoint) + (command)
-        # - user runs ADS operator -> name/YAML
-
-        # TODO: build this out
-
-        return self

ads/opctl/config/yaml_parsers/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-from ads.opctl.config.yaml_parsers.base import YamlSpecParser
-from ads.opctl.config.yaml_parsers.distributed import DistributedSpecParser

ads/opctl/config/yaml_parsers/base.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from logging import getLogger
-from collections import namedtuple
-import fsspec
-import yaml
-import importlib
-
-logger = getLogger("ads.yaml")
-
-
-class YamlSpecParser:
-    def translate_config(self, config: list):
-        """
-        Translates config to three element tuple of dictionary- (file, envVars, concatenated args)
-
-        All elements that are of type `env` goes to envVars
-        All elements that are of type `args` are concatenated as `--key1=value1 --key2=value2 ..` string
-        All elements of type `file` are currently ignored
-        """
-        # TODO parse file type
-
-        parsed_config = namedtuple(
-            "ClusterConfig", field_names=["files", "envVars", "cmd_args"]
-        )
-        envVars = {}
-        files = {}
-        cmd_args = ""
-        if config:
-            if config.get("env"):
-                for item in config["env"]:
-                    envVars[item["name"]] = item["value"]
-            cmd_args = " ".join(config.get("startOptions", [""]))
-        return parsed_config(files=files, envVars=envVars, cmd_args=cmd_args)
-
-    def parse(self):
-        pass
-
-    @classmethod
-    def parse_content(cls, file):
-        yaml_spec = {}
-        if isinstance(file, dict):
-            yaml_spec = file
-        elif isinstance(file, str):
-            with fsspec.open(file) as yf:
-                yaml_spec = yaml.load(yf.read(), yaml.SafeLoader)
-        kind = yaml_spec.get("kind")
-        parsed_output = None
-        if kind:
-            className = f"{kind[0].upper()}{kind[1:]}SpecParser"
-            m = importlib.import_module("ads.opctl.config.yaml_parsers")
-            parser = getattr(m, className)
-            parsed_output = parser(yaml_spec).parse()
-        return parsed_output

ads/opctl/config/yaml_parsers/distributed/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.opctl.config.yaml_parsers.distributed.yaml_parser import DistributedSpecParser

ads/opctl/config/yaml_parsers/distributed/yaml_parser.py

@@ -1,201 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from logging import getLogger
-from collections import namedtuple
-from ads.opctl.config.yaml_parsers import YamlSpecParser
-
-logger = getLogger("ads.yaml")
-
-
-class DistributedSpecParser(YamlSpecParser):
-    def __init__(self, distributed):
-        # TODO: validate yamlInput
-        self.distributed = distributed
-
-    def parse(self):
-        ClusterInfo = namedtuple(
-            "ClusterInfo", field_names=["infrastructure", "cluster", "runtime"]
-        )
-        self.distributed_spec = self.distributed["spec"]
-        infrastructure = self.distributed_spec["infrastructure"]
-        cluster_def = self.distributed_spec["cluster"]
-        cluster = self.parse_cluster(cluster_def)
-        runtime = self.parse_runtime(self.distributed_spec.get("runtime"))
-        return ClusterInfo(
-            infrastructure=infrastructure, cluster=cluster, runtime=runtime
-        )
-
-    def parse_cluster(self, cluster_def):
-        Cluster = namedtuple(
-            "Cluster",
-            field_names=[
-                "name",
-                "type",
-                "image",
-                "work_dir",
-                "config",
-                "main",
-                "worker",
-                "ps",
-                "ephemeral",
-                "certificate",
-            ],
-        )
-        cluster_spec = cluster_def["spec"]
-        name = cluster_spec.get("name")
-        cluster_type = cluster_def.get("kind")
-        image = cluster_spec.get("image")
-        work_dir = cluster_spec.get("workDir")
-        ephemeral = cluster_spec.get("ephemeral")
-        cluster_default_config = cluster_spec.get("config")
-        main = self.parse_main(cluster_spec.get("main"))
-        worker = self.parse_worker(cluster_spec.get("worker"))
-        ps = self.parse_ps(cluster_spec.get("ps"))
-        translated_config = self.translate_config(cluster_default_config)
-        certificate = self.parse_certificate(cluster_spec.get("certificate"))
-        logger.debug(
-            f"Cluster: [name: {name}, type: {cluster_type}, image: {image}, work_dir: {work_dir}, config: {translated_config}, main: {main}, worker: {worker}, ps: {ps}]"
-        )
-        return Cluster(
-            name=name,
-            type=cluster_type,
-            image=image,
-            work_dir=work_dir,
-            config=translated_config,
-            main=main,
-            worker=worker,
-            ps=ps,
-            ephemeral=ephemeral,
-            certificate=certificate,
-        )
-
-    def parse_main(self, main):
-        Main = namedtuple("Main", field_names=["name", "image", "replicas", "config"])
-        main_spec = main
-        name = main_spec.get("name")
-        replicas = main_spec.get("replicas") or 1
-        if replicas > 1:
-            logger.warn(
-                "`replicas` greater than 1 is currently not supported. This will be default to 1"
-            )
-        image = main_spec.get("image")
-        config = main_spec.get("config")
-        translated_config = self.translate_config(config)
-        logger.debug(
-            f"main: [name: {name}, image: {image}, replicas: {replicas}, config: {translated_config}]"
-        )
-        return Main(name=name, image=image, replicas=replicas, config=translated_config)
-
-    def parse_worker_params(self, worker_spec):
-        name = worker_spec.get("name")
-        replicas = worker_spec.get("replicas") or 1
-        image = worker_spec.get("image")
-        config = worker_spec.get("config")
-        translated_config = self.translate_config(config)
-        logger.debug(
-            f"Worker: [name: {name}, image: {image}, replicas: {replicas}, config: {translated_config}]"
-        )
-        return name, image, replicas, translated_config
-
-    def parse_worker(self, worker):
-        if not worker:
-            return None
-        Worker = namedtuple("Worker", field_names=["name", "image", "replicas", "config"])
-        name, image, replicas, translated_config = self.parse_worker_params(worker)
-        logger.debug(
-            f"Worker: [name: {name}, image: {image}, replicas: {replicas}, config: {translated_config}]"
-        )
-        return Worker(name=name, image=image, replicas=replicas, config=translated_config)
-
-    def parse_ps(self, worker):
-        if not worker:
-            return None
-        Ps = namedtuple("PS", field_names=["name", "image", "replicas", "config"])
-        name, image, replicas, translated_config = self.parse_worker_params(worker)
-        logger.debug(
-            f"PS: [name: {name}, image: {image}, replicas: {replicas}, config: {translated_config}]"
-        )
-        return Ps(name=name, image=image, replicas=replicas, config=translated_config)
-
-    def parse_runtime(self, runtime):
-        PythonRuntime = namedtuple(
-            "PythonRuntime",
-            field_names=[
-                "entry_point",
-                "args",
-                "kwargs",
-                "envVars",
-                "type",
-                "uri",
-                "branch",
-                "commit",
-                "git_secret_id",
-                "code_dir",
-                "python_path",
-            ],
-        )
-        python_spec = runtime["spec"]
-        envVars = {}
-        if python_spec.get("env"):
-            envVars = {k["name"]: k["value"] for k in python_spec.get("env")}
-        return PythonRuntime(
-            entry_point=python_spec.get("entryPoint"),
-            args=python_spec.get("args"),
-            kwargs=python_spec.get("kwargs"),
-            envVars=envVars,
-            type=runtime.get("type"),
-            uri=python_spec.get("uri"),
-            branch=python_spec.get("branch"),
-            commit=python_spec.get("commit"),
-            git_secret_id=python_spec.get("gitSecretId"),
-            code_dir=python_spec.get("codeDir"),
-            python_path=python_spec.get("pythonPath"),
-        )
-
-    def parse_certificate(self, certificate):
-        """
-        Expected yaml schema:
-            cluster:
-                spec:
-                    certificate:
-                        caCert:
-                            id: oci.xxxx.<ca_cert_ocid>
-                            downloadLocation:  /code/ca.pem
-                            cert:
-                                id: oci.xxxx.<cert_ocid>
-                                certDownloadLocation: /code/cert.pem
-                                keyDownloadLocation: /code/key.pem
-        """
-        if certificate and certificate.get("caCert") and certificate.get("cert"):
-            Certificate = namedtuple(
-                "Certificate",
-                field_names=[
-                    "ca_ocid",
-                    "ca_download_location",
-                    "cert_ocid",
-                    "cert_download_location",
-                    "key_download_location",
-                ],
-            )
-            ca_ocid = certificate["caCert"]["id"]
-            ca_download_location = certificate["caCert"].get(
-                "downloadLocation", "ca-cert.pem"
-            )
-            cert_ocid = certificate["cert"]["id"]
-            cert_download_location = certificate["cert"].get(
-                "certDownloadLocation", "cert.pem"
-            )
-            key_download_location = certificate["cert"].get(
-                "keyDownloadLocation", "key.pem"
-            )
-            return Certificate(
-                ca_ocid=ca_ocid,
-                ca_download_location=ca_download_location,
-                cert_ocid=cert_ocid,
-                cert_download_location=cert_download_location,
-                key_download_location=key_download_location,
-            )

ads/opctl/constants.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.common.extended_enum import ExtendedEnum
-
-DEFAULT_OCI_CONFIG_FILE = "~/.oci/config"
-DEFAULT_PROFILE = "DEFAULT"
-DEFAULT_CONDA_PACK_FOLDER = "~/conda"
-DEFAULT_MODEL_FOLDER = "~/.ads_ops/models"
-CONDA_PACK_OS_PREFIX_FORMAT = "oci://<bucket>@<namespace>/<prefix>"
-DEFAULT_ADS_CONFIG_FOLDER = "~/.ads_ops"
-ML_JOB_IMAGE = "ml-job"
-ML_JOB_GPU_IMAGE = "ml-job-gpu"
-DEFAULT_MANIFEST_VERSION = "1.0"
-ADS_CONFIG_FILE_NAME = "config.ini"
-ADS_JOBS_CONFIG_FILE_NAME = "ml_job_config.ini"
-ADS_DATAFLOW_CONFIG_FILE_NAME = "dataflow_config.ini"
-ADS_ML_PIPELINE_CONFIG_FILE_NAME = "ml_pipeline.ini"
-ADS_LOCAL_BACKEND_CONFIG_FILE_NAME = "local_backend.ini"
-ADS_MODEL_DEPLOYMENT_CONFIG_FILE_NAME = "model_deployment_config.ini"
-DEFAULT_IMAGE_HOME_DIR = "/home/datascience"
-DEFAULT_IMAGE_SCRIPT_DIR = "/etc/datascience"
-DEFAULT_IMAGE_CONDA_DIR = "/opt/conda/envs"
-DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR = "/home/datascience/spark_conf_dir"
-DEFAULT_NOTEBOOK_SESSION_CONDA_DIR = "/home/datascience/conda"
-DEFAULT_SPECIFICATION_FILE_NAME = "oci-datascience-template.yaml"
-DEFAULT_MODEL_DEPLOYMENT_FOLDER = "/opt/ds/model/deployed_model/"
-
-# OPERATOR
-OPERATOR_MODULE_PATH = "ads.opctl.operator.lowcode"
-OPERATOR_IMAGE_WORK_DIR = "/etc/operator"
-OVERRIDE_KWARGS = "override_kwargs"
-
-
-class RUNTIME_TYPE(ExtendedEnum):
-    PYTHON = "python"
-    CONTAINER = "container"
-    NOTEBOOK = "notebook"
-    GITPYTHON = "gitPython"
-    OPERATOR = "operator"
-    SCRIPT = "script"
-    DATAFLOW = "dataFlow"
-    DATAFLOWNOTEBOOK = "dataFlowNotebook"
-    CONDA = "conda"
-
-
-class RESOURCE_TYPE(ExtendedEnum):
-    JOB = "job"
-    DATAFLOW = "dataflow"
-    PIPELINE = "pipeline"
-    MODEL_DEPLOYMENT = "deployment"
-    MARKETPLACE = "marketplace"
-    OPERATOR = "operator"
-
-
-class BACKEND_NAME(ExtendedEnum):
-    JOB = "job"
-    DATAFLOW = "dataflow"
-    PIPELINE = "pipeline"
-    MODEL_DEPLOYMENT = "deployment"
-    LOCAL = "local"
-    OPERATOR_LOCAL = "operator.local"
-    MARKETPLACE = "marketplace"

ads/opctl/decorator/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2023 Oracle and its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/decorator/common.py

@@ -1,129 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from functools import wraps
-from typing import Callable, Dict, List
-
-import click
-
-from ads.common.auth import AuthContext
-from ads.opctl import logger
-from ads.opctl.config.base import ConfigProcessor
-from ads.opctl.config.merger import ConfigMerger
-from ads.opctl.constants import OVERRIDE_KWARGS
-
-RUN_ID_FIELD = "run_id"
-
-
-class OpctlEnvironmentError(Exception):
-    """The custom error to validate OPCTL environment."""
-
-    NOT_SUPPORTED_ENVIRONMENTS = (
-        "Notebook Sessions",
-        "Data Science Jobs",
-        "ML Pipelines",
-        "Data Flow Applications",
-    )
-
-    def __init__(self):
-        super().__init__(
-            "This operation cannot be executed in the current environment. "
-            f"It is not supported in: {', '.join(self.NOT_SUPPORTED_ENVIRONMENTS)}."
-        )
-
-
-def print_watch_command(func: callable) -> Callable:
-    """The decorator to help build the `opctl watch` command."""
-
-    @wraps(func)
-    def wrapper(*args: List, **kwargs: Dict) -> Dict:
-        result = func(*args, **kwargs)
-        if result and isinstance(result, Dict) and RUN_ID_FIELD in result:
-            msg_header = (
-                f"{'*' * 40} To monitor the progress of the task, "
-                f"execute the following command {'*' * 40}"
-            )
-            print(msg_header)
-            print(f"ads opctl watch {result[RUN_ID_FIELD]}")
-            print("*" * len(msg_header))
-        return result
-
-    return wrapper
-
-
-def validate_environment(func: callable) -> Callable:
-    """Validates whether an opctl command can be executed in the current environment."""
-
-    @wraps(func)
-    def wrapper(*args: List, **kwargs: Dict) -> Dict:
-        try:
-            import docker
-
-            docker.from_env().version()
-        except Exception as ex:
-            logger.debug(ex)
-            raise OpctlEnvironmentError()
-
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def click_options(options):
-    """The decorator to help group the click options."""
-
-    def _add_options(func):
-        for option in reversed(options):
-            func = option(func)
-        return func
-
-    return _add_options
-
-
-def with_auth(func: Callable) -> Callable:
-    """The decorator to add AuthContext to the method."""
-
-    @wraps(func)
-    def wrapper(*args, **kwargs) -> Dict:
-        p = ConfigProcessor().step(ConfigMerger, **kwargs)
-
-        with AuthContext(
-            **{
-                key: value
-                for key, value in {
-                    "auth": p.config["execution"]["auth"],
-                    "oci_config_location": p.config["execution"]["oci_config"],
-                    "profile": p.config["execution"]["oci_profile"],
-                }.items()
-                if value
-            }
-        ):
-            return func(*args, **kwargs)
-
-    return wrapper
-
-
-def with_click_unknown_args(func: Callable) -> Callable:
-    """The decorator to parse the click unknown arguments and put them into kwargs."""
-
-    @wraps(func)
-    def wrapper(*args, **kwargs) -> Dict:
-        kwargs[OVERRIDE_KWARGS] = {}
-        try:
-            click_context = next(
-                item for item in args if isinstance(item, click.core.Context)
-            )
-            kwargs[OVERRIDE_KWARGS] = {
-                key[2:]: value
-                for key, value in zip(click_context.args[::2], click_context.args[1::2])
-            }
-        except Exception as ex:
-            logger.debug(ex)
-            pass
-
-        return func(*args, **kwargs)
-
-    return wrapper

ads/opctl/diagnostics/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/diagnostics/__main__.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.opctl.diagnostics.check_requirements import RequirementChecker
-from ads.opctl.diagnostics.check_distributed_job_requirements import (
-    PortsChecker,
-    Default,
-)
-import argparse
-
-RequirementChecker.register_provider("cluster-ports-check", PortsChecker)
-RequirementChecker.register_provider("cluster-generic", Default)
-
-parser = argparse.ArgumentParser(description="Run diagnostics to check your setup")
-parser.add_argument(
-    "-t",
-    "--type",
-    help="What type of diagnostics to run. Eg. `distributed` to run diagnostics for distributed training setup",
-)
-
-args = parser.parse_args()
-RequirementChecker().verify_requirements(args.type)