oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/common/decorator/utils.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from typing import Callable
+
+
+def _get_original_func(func: callable) -> Callable:
+    """The helper to retrieve the original function from the decorated one."""
+    if func and hasattr(func, "__wrapped__"):
+        return _get_original_func(func.__wrapped__)
+    return func
+
+
+class class_or_instance_method(classmethod):
+    """Converts a function to be a class method or an instance depending on how it is called at runtime.
+
+    To declare a class method, use this idiom:
+
+        class C:
+            @classmethod
+            def f(obj, *args, **kwargs):
+                ...
+    It can be called either on the class (e.g. C.f()) or on an instance (e.g. C().f()).
+    If it is called on the class C.f(), the first argument (obj) will be the class (aka. cls).
+    If it is called on the instance C().f(), the first argument (obj) will be the instance (aka. self).
+
+    """
+
+    def __get__(self, instance, type_):
+        delegate_get = super().__get__ if instance is None else self.__func__.__get__
+        return delegate_get(instance, type_)

ads/common/dsc_file_system.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import ads
+import oci
+import os
+import ipaddress
+
+from dataclasses import dataclass
+
+FILE_STORAGE_TYPE = "FILE_STORAGE"
+OBJECT_STORAGE_TYPE = "OBJECT_STORAGE"
+
+
+@dataclass
+class DSCFileSystem:
+
+    src: str = None
+    dest: str = None
+    storage_type: str = None
+    destination_directory_name: str = None
+    destination_path: str = None
+
+    def update_to_dsc_model(self) -> dict:
+        """Updates arguments to dsc model.
+
+        Returns
+        -------
+        dict:
+            A dictionary of arguments.
+        """
+        pass
+
+    @classmethod
+    def update_from_dsc_model(cls, dsc_model) -> dict:
+        """Updates arguments from dsc model.
+
+        Parameters
+        ----------
+        dsc_model: oci.data_science.models.JobStorageMountConfigurationDetails
+            An instance of oci.data_science.models.JobStorageMountConfigurationDetails.
+
+        Returns
+        -------
+        dict
+            A dictionary of arguments.
+        """
+        pass
+
+    @staticmethod
+    def get_destination_path_and_name(dest: str) -> (str, str):
+        """Gets the destination path and destination directory name from dest.
+        Example:
+            dir - "fss" & path - "/opc" : mount happens under "/opc/fss"
+            dir - "fss" & path - "/" : mount happens under "/fss"
+            dir - "fss" & path - omitted : mount happens under "/mnt/fss" (for backward compatibility)
+
+        Parameters
+        ----------
+        dest: str
+            The dest path to which to mount the file system.
+
+        Returns
+        -------
+        tuple
+            A tuple of destination path and destination directory name.
+        """
+        return (
+            os.path.dirname(dest.rstrip("/")) or None, # when destination path is omitted, oci api requires it to be None
+            os.path.basename(dest.rstrip("/"))
+        )
+
+
+@dataclass
+class OCIFileStorage(DSCFileSystem):
+
+    mount_target_id: str = None
+    mount_target: str = None
+    export_id: str = None
+    export_path: str = None
+    storage_type: str = FILE_STORAGE_TYPE        
+
+    def update_to_dsc_model(self) -> dict:
+        """Updates arguments to dsc model.
+
+        Returns
+        -------
+        dict:
+            A dictionary of arguments.
+        """
+        path, directory_name = self.get_destination_path_and_name(self.dest)
+        arguments = {
+            "destinationDirectoryName" : directory_name,
+            "destinationPath" : path,
+            "storageType" : self.storage_type
+        }
+        
+        self._get_mount_target_and_export_ids(arguments)
+
+        return arguments
+
+    def _get_mount_target_and_export_ids(self, arguments: dict):
+        """Gets the mount target id and export id from src.
+
+        Parameters
+        ----------
+        arguments: dict
+            A dictionary of arguments.
+        """
+        resource_client = oci.resource_search.ResourceSearchClient(**ads.auth.default_signer())
+        src_list = self.src.split(":")
+        first_segment = src_list[0]
+        second_segment = src_list[1]
+
+        if first_segment.startswith("ocid") and "mounttarget" in first_segment:
+            arguments["mountTargetId"] = first_segment
+        else:
+            ip_resource = resource_client.search_resources(
+                search_details=oci.resource_search.models.FreeTextSearchDetails(
+                    text=first_segment,
+                    matching_context_type="NONE"
+                )
+            ).data.items
+
+            ip_resource = sorted(ip_resource, key=lambda resource_summary: resource_summary.time_created)
+
+            if not ip_resource or not hasattr(ip_resource[-1], "identifier"):
+                raise ValueError(f"Can't find the identifier from ip {first_segment}. Specify a valid `src`.")
+
+            mount_target_resource = resource_client.search_resources(
+                search_details=oci.resource_search.models.FreeTextSearchDetails(
+                    text=ip_resource[-1].identifier,
+                    matching_context_type="NONE"
+                )
+            ).data.items
+
+            mount_targets = [
+                mount_target.identifier
+                for mount_target in mount_target_resource
+                if mount_target.resource_type == "MountTarget"
+            ]
+            if len(mount_targets) == 0:
+                raise ValueError(
+                    f"No `mount_target_id` found under ip {first_segment}. Specify a valid `src`."
+                )
+            if len(mount_targets) > 1:
+                raise ValueError(
+                    f"Multiple `mount_target_id` found under ip {first_segment}. Specify the `mount_target_id` in `src` instead."
+                )
+
+            arguments["mountTargetId"] = mount_targets[0]
+        
+        if second_segment.startswith("ocid") and "export" in second_segment:
+            arguments["exportId"] = second_segment
+        else:
+            export_resource = resource_client.search_resources(
+                search_details=oci.resource_search.models.FreeTextSearchDetails(
+                    text=second_segment,
+                    matching_context_type="NONE"
+                )
+            ).data.items
+
+            exports = [
+                export.identifier
+                for export in export_resource
+                if export.resource_type == "Export"
+            ]
+            if len(exports) == 0:
+                raise ValueError(
+                    f"No `export_id` found with `export_path` {second_segment}. Specify a valid `src`."
+                )
+            if len(exports) > 1:
+                raise ValueError(
+                    f"Multiple `export_id` found with `export_path` {second_segment}. Specify the `export_id` in `src` instead."
+                )
+
+            arguments["exportId"] = exports[0]
+
+    @classmethod
+    def update_from_dsc_model(cls, dsc_model) -> dict:
+        """Updates arguments from dsc model.
+
+        Parameters
+        ----------
+        dsc_model: oci.data_science.models.JobStorageMountConfigurationDetails
+            An instance of oci.data_science.models.JobStorageMountConfigurationDetails.
+
+        Returns
+        -------
+        dict
+            A dictionary of arguments.
+        """
+        if not dsc_model.mount_target_id:
+            raise ValueError(
+                "Missing parameter `mount_target_id` from service. Check service log to see the error."
+            )
+        if not dsc_model.export_id:
+            raise ValueError(
+                "Missing parameter `export_id` from service. Check service log to see the error."
+            )
+        if not dsc_model.destination_directory_name:
+            raise ValueError(
+                "Missing parameter `destination_directory_name` from service. Check service log to see the error."
+            )
+        
+        dest = dsc_model.destination_directory_name
+        if dsc_model.destination_path:
+            dest = f"{dsc_model.destination_path.rstrip('/')}/{dsc_model.destination_directory_name}"
+
+        return {
+            "src" : f"{dsc_model.mount_target_id}:{dsc_model.export_id}",
+            "dest" : dest
+        }
+
+@dataclass
+class OCIObjectStorage(DSCFileSystem):
+
+    storage_type: str = OBJECT_STORAGE_TYPE
+
+    def update_to_dsc_model(self) -> dict:
+        path, directory_name = self.get_destination_path_and_name(self.dest)
+        arguments = {
+            "destinationDirectoryName" : directory_name,
+            "destinationPath" : path,
+            "storageType" : self.storage_type
+        }
+        src_list = self.src.split("@")
+        bucket_segment = src_list[0]
+        namespace_segment = src_list[1].strip("/")
+        arguments["bucket"] = bucket_segment[6:]
+        if "/" in namespace_segment:
+            first_slash_index = namespace_segment.index("/")
+            arguments["namespace"] = namespace_segment[:first_slash_index]
+            arguments["prefix"] = namespace_segment[first_slash_index+1:]
+        else:
+            arguments["namespace"] = namespace_segment
+        return arguments
+
+    @classmethod
+    def update_from_dsc_model(cls, dsc_model) -> dict:
+        if not dsc_model.namespace:
+            raise ValueError(
+                "Missing parameter `namespace` from service. Check service log to see the error."
+            )
+        if not dsc_model.bucket:
+            raise ValueError(
+                "Missing parameter `bucket` from service. Check service log to see the error."
+            )
+        if not dsc_model.destination_directory_name:
+            raise ValueError(
+                "Missing parameter `destination_directory_name` from service. Check service log to see the error."
+            )
+
+        dest = dsc_model.destination_directory_name
+        if dsc_model.destination_path:
+            dest = f"{dsc_model.destination_path.rstrip('/')}/{dsc_model.destination_directory_name}"
+
+        return {
+            "src" : f"oci://{dsc_model.bucket}@{dsc_model.namespace}/{dsc_model.prefix or ''}",
+            "dest" : dest
+        }
+
+
+class DSCFileSystemManager:
+
+    @classmethod
+    def initialize(cls, arguments: dict) -> dict:
+        """Initialize and update arguments to dsc model.
+
+        Parameters
+        ----------
+        arguments: dict
+            A dictionary of arguments.
+        """
+        if "src" not in arguments:
+            raise ValueError(
+                "Parameter `src` is required for mounting file storage system."
+            )
+
+        if "dest" not in arguments:
+            raise ValueError(
+                "Parameter `dest` is required for mounting file storage system."
+            )
+
+        # case oci://bucket@namespace/prefix
+        if arguments["src"].startswith("oci://") and "@" in arguments["src"]:
+            return OCIObjectStorage(**arguments).update_to_dsc_model()
+
+        first_segment = arguments["src"].split(":")[0]
+        # case <mount_target_id>:<export_id>
+        if first_segment.startswith("ocid"):
+            return OCIFileStorage(**arguments).update_to_dsc_model()
+
+        # case <ip_address>:<export_path>
+        try:
+            ipaddress.IPv4Network(first_segment)
+            return OCIFileStorage(**arguments).update_to_dsc_model()
+        except:
+            pass
+
+        raise ValueError("Invalid dict for mounting file systems. Specify a valid one.")

ads/common/error.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class ChangesNotCommitted(Exception):   # pragma: no cover
+    def __init__(self, path):
+        msg = f"""
+            File(s) at {path} are either dirty or untracked.
+            Please commit changes and then save the model, or set `ignore_pending_changes=True`.
+        """
+        super().__init__(msg)

ads/common/extended_enum.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from abc import ABCMeta
+
+
+class ExtendedEnumMeta(ABCMeta):
+    """
+    A helper metaclass to extend functionality of a generic "Enum-like" class.
+
+    Methods
+    -------
+    __contains__(cls, item) -> bool:
+        Checks if `item` is among the attribute values of the class.
+        Case-insensitive if `item` is a string.
+    values(cls) -> tuple:
+        Returns the tuple of class attribute values.
+    keys(cls) -> tuple:
+        Returns the tuple of class attribute names.
+    """
+
+    def __contains__(cls, item: object) -> bool:
+        """
+        Checks if `item` is a member of the class's values.
+
+        - If `item` is a string, does a case-insensitive match against any string
+          values stored in the class.
+        - Otherwise, does a direct membership test.
+        """
+        # Gather the attribute values
+        attr_values = cls.values()
+
+        # If item is a string, compare case-insensitively to any str-type values
+        if isinstance(item, str):
+            return any(
+                isinstance(val, str) and val.lower() == item.lower()
+                for val in attr_values
+            )
+        else:
+            # For non-string items (e.g., int), do a direct membership check
+            return item in attr_values
+
+    def __iter__(cls):
+        # Make the class iterable by returning an iterator over its values
+        return iter(cls.values())
+
+    def values(cls) -> tuple:
+        """
+        Gets the tuple of class attribute values, excluding private or special
+        attributes and any callables (methods, etc.).
+        """
+        return tuple(
+            value
+            for key, value in cls.__dict__.items()
+            if not key.startswith("_") and not callable(value)
+        )
+
+    def keys(cls) -> tuple:
+        """
+        Gets the tuple of class attribute names, excluding private or special
+        attributes and any callables (methods, etc.).
+        """
+        return tuple(
+            key
+            for key, value in cls.__dict__.items()
+            if not key.startswith("_") and not callable(value)
+        )
+
+
+class ExtendedEnum(metaclass=ExtendedEnumMeta):
+    """The base class to extend functionality of a generic Enum.
+
+    Examples
+    --------
+    >>> class TestEnum(ExtendedEnum):
+    ...    KEY1 = "v1"
+    ...    KEY2 = "v2"
+    """

ads/common/function/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/common/function/fn_util.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import yaml
+import os
+from jinja2 import Environment, PackageLoader
+
+# FunctionAttributes = namedtuple('FunctionAttributes',['schema_version','name','version','runtime','entrypoint','memory'])
+fn_config = None
+env = Environment(loader=PackageLoader("ads", "templates"))
+
+
+def prepare_fn_attributes(
+    func_name: str,
+    schema_version=20180708,
+    version=None,
+    python_runtime=None,
+    entry_point=None,
+    memory=None,
+) -> dict:
+    """
+    Workaround for collections.namedtuples. The defaults are not supported.
+    """
+    function_attributes = {}
+    function_attributes["schema_version"] = schema_version
+    function_attributes["name"] = func_name
+    function_attributes["version"] = (
+        version if version else get_function_config()["fn_conf"]["version"]
+    )
+    function_attributes["runtime"] = (
+        python_runtime
+        if python_runtime
+        else get_function_config()["fn_conf"]["runtime"]
+    )
+    function_attributes["entrypoint"] = (
+        entry_point if entry_point else get_function_config()["fn_conf"]["entrypoint"]
+    )
+    function_attributes["memory"] = (
+        memory if memory else get_function_config()["fn_conf"]["memory"]
+    )
+
+    return function_attributes
+
+
+def get_function_config() -> dict:
+    """
+    Returns dictionary loaded from func_conf.yaml
+    """
+    global fn_config
+    if fn_config:
+        return fn_config
+    with open(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), "func_conf.yaml")
+    ) as fn_config_file:
+        func_config = yaml.load(fn_config_file, Loader=yaml.FullLoader)
+    return func_config
+
+
+def generate_fn_artifacts(
+    path: str,
+    fn_name: str = None,
+    fn_attributes=None,
+    artifact_type_generic=False,
+    **kwargs,
+):
+    """
+    Generates artifacts for fn (https://fnproject.io) at the provided path -
+        * func.py
+        * func.yaml
+        * requirements.txt if not there. If exists appends fdk to the file.
+        * score.py
+
+    Parameters
+    ----------
+        path: str
+            Target folder where the artifacts are placed.
+        fn_attributes: dict
+            dictionary specifying all the function attributes as described in https://github.com/fnproject/docs/blob/master/fn/develop/func-file.md
+        artifact_type_generic: bool
+            default is False. This attribute decides which template to pick for score.py. If True, it is assumed that the code to load is provided by the user.
+
+    """
+    assert fn_name or fn_attributes, (
+        "Must provide either fn_name or fn_attributes. You may use "
+        "ads.common.artifact.fn_util.prepare_fn_attributes for creating fn_attributes "
+    )
+    function_attributes = prepare_fn_attributes(fn_name) if fn_name else fn_attributes
+
+    progress = kwargs.get("progress", None)
+    if progress:
+        progress.update("Writing func.yaml")
+    with open(os.path.join(path, "func.yaml"), "w") as fnyaml_file:
+        yaml.dump(function_attributes, fnyaml_file)
+
+    if progress:
+        progress.update("Writing func.py")
+    func_template = env.get_template("func.jinja2")
+    with open(os.path.join(path, "func.py"), "w") as func_fl:
+        func_fl.write(func_template.render(score_module="score"))
+
+
+def write_score(path, **kwargs):
+    serializer = kwargs.get("serializer", "default")
+    if serializer not in get_function_config()["models"]:
+        serializer = "default"
+
+    model_name = kwargs.get(
+        "model_name", get_function_config()["models"][serializer]["file_name"]
+    )
+    transformer_name = (
+        get_function_config()["models"][serializer]["transformer_name"]
+        if serializer in get_function_config()["models"]
+        else "onnx_data_transformer.json"
+    )
+    misc_imports = (
+        kwargs.get("misc_imports", list())
+        + get_function_config()["models"][serializer]["misc_imports"]
+    )
+    underlying_model = kwargs.get("underlying_model", "NOTFOUND")
+    sklearn_model = underlying_model in ["sklearn", "xgboost"]
+
+    if underlying_model == "automl":
+        jinja_template_filename = "score_oracle_automl"
+    else:
+        jinja_template_filename = kwargs.get(
+            "input_file",
+            get_function_config()["models"][serializer].get("input_file", "score"),
+        )
+    scorefn_template = env.get_template(f"{jinja_template_filename}.jinja2")
+    with open(os.path.join(path, "score.py"), "w") as sfl:
+        sfl.write(
+            scorefn_template.render(
+                model_file_name=model_name,
+                transfromer_file_name=transformer_name,
+                misc_imports=misc_imports,
+                sklearn_model=sklearn_model,
+                underlying_model_type=kwargs.get("_underlying_model", None),
+            )
+        )

ads/common/function/func_conf.yaml

@@ -0,0 +1,25 @@
+models:
+  onnx:
+    file_name:  'model.onnx'
+    transformer_name: 'onnx_data_transformer.json'
+    input_file: 'score'
+    misc_imports: []
+  default:
+    file_name:  'model.pkl'
+    transformer_name: ''
+    input_file: 'score-pkl'
+    misc_imports: []
+  unknown:
+    file_name:  'model.pkl'
+    transformer_name: ''
+    input_file: 'score-pkl'
+    misc_imports: []
+requires:
+  onnx: ['numpy', 'pandas', 'scikit-learn', 'onnxruntime']
+  default: ['numpy', 'pandas', 'scikit-learn', 'onnxruntime']
+  functions: ['fdk']
+fn_conf:
+  version:  '0.0.1'
+  runtime:  'python3.6'
+  entrypoint: '/python/bin/fdk /function/func.py handler'
+  memory: 1024

ads/common/ipython.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+import sys
+
+# TODO - Revisit this as part of ADS logging changes https://jira.oci.oraclecorp.com/browse/ODSC-36245
+# Use a unique logger that we can individually configure without impacting other log statements.
+# We don't want the logger name to mention "ads", since this logger will report any exception that happens in a
+# notebook cell, and we don't want customers incorrectly assuming that ADS is somehow responsible for every error.
+logger = logging.getLogger("ipython.traceback")
+# Set propagate to False so logs aren't passed back up to the root logger handlers. There are some places in ADS
+# where logging.basicConfig() is called. This changes root logger configurations. The user could import/use code that
+# invokes the logging.basicConfig() function at any time, making the behavior of the root logger unpredictable.
+logger.propagate = False
+logger.handlers.clear()
+traceback_handler = logging.StreamHandler()
+traceback_handler.setFormatter(logging.Formatter("%(levelname)s - %(message)s"))
+logger.addHandler(traceback_handler)
+
+
+def _log_traceback(self, exc_tuple=None, **kwargs):
+    try:
+        etype, value, tb = self._get_exc_info(exc_tuple)
+    except ValueError:
+        print("No traceback available to show.", file=sys.stderr)
+        return
+    msg = etype.__name__, str(value)
+    # User a generic message that makes no mention of ADS.
+    logger.error("Exception", exc_info=(etype, value, tb))
+    sys.stderr.write("{0}: {1}".format(*msg))
+
+
+def set_ipython_traceback():
+    pass
+
+
+def configure_plotting():
+    try:
+        import IPython
+        from IPython import get_ipython
+        from IPython.core.error import UsageError
+
+        ipy = get_ipython()
+        if ipy is not None:
+            try:
+                # show matplotlib plots inline
+                ipy.run_line_magic("matplotlib", "inline")
+            except UsageError:
+                #  ignore error and use the default matplotlib mode
+                pass
+    except ModuleNotFoundError:
+        import matplotlib as mpl
+
+        mpl.rcParams["backend"] = "agg"
+        import matplotlib.pyplot as plt
+
+        plt.switch_backend("agg")
+
+
+try:
+    import IPython
+
+    global orig_ipython_traceback
+    if IPython.core.interactiveshell.InteractiveShell.showtraceback != _log_traceback:
+        orig_ipython_traceback = (
+            IPython.core.interactiveshell.InteractiveShell.showtraceback
+        )
+
+    # Override the default showtraceback behavior of ipython, to show only the error message and log the stacktrace
+    IPython.core.interactiveshell.InteractiveShell.showtraceback = _log_traceback
+except ModuleNotFoundError:
+    pass