oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/dataset/target.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import numpy as np
+from sklearn import preprocessing
+
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.common import utils
+from ads.type_discovery.typed_feature import CategoricalTypedFeature
+from ads.type_discovery.typed_feature import ContinuousTypedFeature
+from ads.type_discovery.typed_feature import OrdinalTypedFeature
+
+ABSOLUTE_SKEWNESS_THRESHOLD = 0.5
+# between 0 and 1, ratio of least / most represented class
+CLASS_IMBALANCE_THRESHOLD = 0.5
+
+
+class TargetVariable:
+    """
+    This class provides target specific APIs.
+    """
+
+    @runtime_dependency(module="scipy", install_from=OptionalDependency.VIZ)
+    def __init__(self, sampled_ds, target, target_type):
+        from scipy.stats import skew
+
+        self.sampled_ds = sampled_ds
+        self.name = target
+        self.type = target_type
+        self.target_vals = None
+        if isinstance(self.type, CategoricalTypedFeature):
+            self.target_vals = self.sampled_ds.sampled_df[target].unique().tolist()
+            val_counts = self.sampled_ds.sampled_df[target].value_counts()
+            self.class_imbalance_ratio = val_counts.min() / val_counts.max()
+        elif isinstance(self.type, ContinuousTypedFeature):
+            try:
+                self.skewness = np.abs(self.sampled_ds.sampled_df[target].skew())
+            except TypeError as e:
+                self.skewness = np.abs(
+                    self.sampled_ds.sampled_df[target].astype("float").skew()
+                )
+        else:
+            # can also be DateTimeTypedFeature IPAddressTypedFeature PhoneNumberTypedFeature GISTypedFeature
+            # AddressTypedFeature DocumentTypedFeature ZipcodeTypedFeature UnknownTypedFeature ConstantTypedFeature
+            # CreditCardTypedFeature OrdinalTypedFeature
+            self.skewness = None
+        self.numeric_columns = (
+            self.sampled_ds.sampled_df._get_numeric_data().columns.values
+        )
+
+    def show_in_notebook(self, feature_names=None):  # pragma: no cover
+        """
+        Plot target distribution or target versus feature relation.
+
+        Parameters
+        ----------
+        feature_names: list, Optional
+            Plot target against a list of features.
+            Display target distribution if feature_names is not provided.
+        """
+        if not utils.is_notebook():
+            print("show_in_notebook called but not in notebook environment")
+            return
+
+        verbose = True
+        if feature_names is not None:
+            for feature_name in feature_names:
+                self.sampled_ds.plot(
+                    feature_name, self.name, verbose=verbose
+                ).show_in_notebook()
+                verbose = False
+        else:
+            self.sampled_ds.plot(self.name, verbose=False).show_in_notebook()
+
+    def is_balanced(
+        self,
+        skewness_threshold=ABSOLUTE_SKEWNESS_THRESHOLD,
+        class_imbalance_threshold=CLASS_IMBALANCE_THRESHOLD,
+    ):
+        """
+        Returns True if the target is balanced, False otherwise.
+
+        Returns
+        -------
+        is_balanced: bool
+        """
+        if isinstance(self.type, ContinuousTypedFeature):
+            return self.skewness < skewness_threshold
+        elif isinstance(self.type, CategoricalTypedFeature):
+            return self.class_imbalance_ratio > class_imbalance_threshold
+        else:
+            return True

ads/dataset/timeseries.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class Timeseries:
+    def __init__(self, col_name, df, date_range=None, min=None, max=None):
+        self.col_name = col_name
+        self.df = df
+        self.date_range = date_range
+        self.min = min
+        self.max = max
+
+    def plot(self, **kwargs):
+        # this could be improved :)
+        pass

ads/dbmixin/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/dbmixin/db_pandas_accessor.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.bds.big_data_service import ADSHiveConnection
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from typing import Dict, Optional, Union, Iterator
+from pandas import DataFrame
+
+
+class ConnectionFactory:
+    connectionprovider = {"hive": ADSHiveConnection}
+
+    try:
+        from ads.oracledb.oracle_db import OracleRDBMSConnection
+
+        connectionprovider["oracle"] = OracleRDBMSConnection
+    except:
+        pass
+
+    try:
+        from ads.mysqldb.mysql_db import MySQLRDBMSConnection
+
+        connectionprovider["mysql"] = MySQLRDBMSConnection
+    except:
+        pass
+
+    @classmethod
+    def get(cls, engine="oracle"):
+        Connection = cls.connectionprovider.get(engine, None)
+
+        if not Connection:
+            if engine == "mysql":
+                print("Requires mysql-connector-python package to use mysql engine")
+            elif engine == "oracle":
+                print(
+                    f"The `oracledb` or `cx_Oracle` module was not found. Please run "
+                    f"`pip install {OptionalDependency.DATA}`."
+                )
+            raise Exception(
+                f"Engine {engine} does not have either required dependency or is not supported."
+            )
+        return Connection
+
+
+class DBAccessMixin:
+    @staticmethod
+    def read_sql(
+        sql: str,
+        connection_parameters: dict,
+        bind_variables: Dict = {},
+        chunksize: Optional[int] = None,
+        engine="oracle",
+    ) -> Union["DataFrame", Iterator["DataFrame"]]:
+        """Read SQL query from oracle database into a DataFrame.
+
+        Parameters
+        ----------
+        sql: str
+            SQL query to be executed.
+        connection_parameters: dict
+            A dictionary of connection_parameters - {"user_name":"", "password":"", "service_name":"", "wallet_location":""}
+        bind_variables: Optional[Dict]
+            Key value of pair of bind variables and corresponding values
+        chunksize: Optional[int], default None
+            If specified, return an iterator where `chunksize` is the number of rows to include in each chunk.
+        engine: {'oracle', 'mysql', 'hive'}, default 'oracle'
+            Select the database type - MySQL/Oracle/Hive to store the data
+
+        Returns
+        -------
+            DataFrame or Iterator[DataFrame]
+                DataFrame or Iterator[DataFrame].
+
+        Examples
+        --------
+        >>> connection_parameters = {
+                "user_name": "<username>",
+                "password": "<password>",
+                "service_name": "{service_name}_{high|med|low}",
+                "wallet_location": "/full/path/to/my_wallet.zip",
+            }
+        >>> import pandas as pd
+        >>> import ads
+        >>> df = pd.DataFrame.ads.read_sql("SELECT * from Employee", connection_parameters=connection_parameters)
+        >>> df_with_bind = pd.DataFrame.ads.read_sql("SELECT * from EMPLOYEE WHERE EMPLOYEE_ID = :ID", bind_variables={"ID":"121212", connection_parameters=connection_parameters)
+
+
+        """
+        Connection = ConnectionFactory.get(engine)
+
+        return Connection(**connection_parameters).query(
+            sql, bind_variables=bind_variables, chunksize=chunksize
+        )
+
+    def to_sql(
+        self,
+        table_name: str,
+        connection_parameters: dict,
+        if_exists: str = "fail",
+        batch_size=100000,
+        engine="oracle",
+        encoding="utf-8",
+    ):
+        """To save the dataframe df to database.
+
+        Parameters
+        ----------
+        table_name: str
+            Name of SQL table.
+        connection_parameters: dict
+            A dictionary of connection_parameters - {"user_name":"", "password":"", "service_name":"", "wallet_location":""}
+        if_exists: : {'fail', 'replace', 'append'}, default 'fail'
+            How to behave if the table already exists.
+            * fail: Raise a ValueError. If table exists, do nothing
+            * replace: Drop the table before inserting new values. If table exists, drop it, recreate it, and insert data.
+            * append: Insert new values to the existing table. If table exists, insert data. Create if does not exist.
+        batch_size: int, default 100000
+            Inserting in batches improves insertion performance. Choose this value based on available memore and network bandwidth.
+        engine: {'oracle', 'mysql'}, default 'oracle'
+            Select the database type - MySQL or Oracle to store the data
+        encoding: str, default is "utf-8"
+            Encoding provided will be used for ecoding all columns, when inserting into table
+
+
+        Returns
+        -------
+            None
+                Nothing.
+        Examples
+        --------
+        >>> connection_parameters = {
+                "user_name": "<username>",
+                "password": "<password>",
+                "service_name": "{service_name}_{high|med|low}",
+                "wallet_location": "/full/path/to/my_wallet.zip",
+            }
+        >>> import pandas as pd
+        >>> import ads
+        >>> df2 = pd.read_csv("my/data/csv")
+        >>> df2.ads.to_sql("MY_DATA_CSV", connection_parameters=connection_parameters)
+        """
+        if if_exists not in ["fail", "replace", "append"]:
+            raise ValueError(
+                f"Unknown option `if_exists`={if_exists}. Valid options are 'fail', 'replace', 'append'"
+            )
+
+        Connection = ConnectionFactory.get(engine)
+        return Connection(**connection_parameters).insert(
+            table_name, self._obj, if_exists, batch_size, encoding
+        )

ads/environment/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+
+logger = logging.getLogger(__name__)

ads/environment/ml_runtime.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+
+from ads.common.decorator.deprecate import deprecated
+from multiprocessing.pool import ThreadPool
+
+try:
+    import dask
+    from dask.distributed import Client
+except ImportError as e:
+    raise ModuleNotFoundError("Install dask to use the MLRuntime class.") from e
+
+_default_dask_threadpool_size = 50
+_client = None
+_storage = None
+
+
+class MLRuntime(object):
+    @deprecated("2.5.2")
+    def __init__(self):
+        dask_threadpool_size = (
+            os.environ.get("DASK_THREADPOOL_SIZE") or _default_dask_threadpool_size
+        )
+        dask.config.set(pool=ThreadPool(int(dask_threadpool_size)))
+        if "DASK_SCHEDULER" in os.environ:
+            self.dask_client = Client(address=os.environ["DASK_SCHEDULER"])
+        elif os.environ.get("DASK_LOCAL_CLIENT", "False").lower() == "true":
+            self.dask_client = Client(
+                processes=False,
+                local_directory="/tmp",  # use directory with write permits
+            )
+        else:
+            # first let's see if the default client is already running, if it is
+            # connect to it, else connect a new Dask scheduler
+            # localhost:8786 for dask scheduler
+            try:
+                self.dask_client = Client("127.0.0.1:8786", timeout=1)
+            except:
+                self.dask_client = Client(
+                    processes=False,  # use threads, avoid for inproc cluster IPC overhead
+                    name="Oracle Data Science Local Dask",
+                    heartbeat_interval=60000,  # one mimnute heartbeats
+                    local_directory="/tmp",  # use directory with write permits
+                )
+
+    def get_compute_accelerator(self):
+        return self.dask_client
+
+
+def init():
+    global _client
+
+    # ensures the client is created only once
+    _client = MLRuntime().get_compute_accelerator()
+
+
+def compute_accelerator():
+    global _client
+    if _client is None:
+        init()
+    return _client

ads/evaluations/README.md

@@ -0,0 +1,14 @@
+### Model Quality Evaluation Capabilities
+
+At this time, we support evaluation for various supervised ML models.
+The following evaluations have been added at this time:
+
+1. Regression Models: For cases where the target variable is a
+continuous function of dependent variables.
+
+2. Classification Models: For cases where the target variable is a
+discrete function, and consists of a (usually small) finite set of
+values, called the classes. We support the following specific cases:
+ * Binary Classification: Target consists of two unique values
+ * Multi-class Classification: Target consists of more than two
+ (but definitely finite) unique values.

ads/evaluations/__init__.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+from typing import List
+from numpy.typing import ArrayLike
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorMixin:
+    def evaluate(
+        self,
+        X: ArrayLike,
+        y: ArrayLike,
+        y_pred: ArrayLike = None,
+        y_score: ArrayLike = None,
+        X_train: ArrayLike = None,
+        y_train: ArrayLike = None,
+        classes: List = None,
+        positive_class: str = None,
+        legend_labels: dict = None,
+        perfect: bool = True,
+        filename: str = None,
+        use_case_type: str = None,
+    ):
+        """Creates an ads evaluation report.
+
+        Parameters
+        ----------
+        X : DataFrame-like
+            The data used to make a prediction.
+            Can be set to None if `y_preds` is given. (And `y_scores` for more thorough analysis).
+        y : array-like
+            The true values corresponding to the input data
+        y_pred : array-like, optional
+            The predictions from each model in the same order as the models
+        y_score : array-like, optional
+            The predict_probas from each model in the same order as the models
+        X_train : DataFrame-like, optional
+            The data used to train the model
+        y_train : array-like, optional
+            The true values corresponding to the input training data
+        classes : List or None, optional
+            A List of the possible labels for y, when evaluating a classification use case
+        positive_class : str or int, optional
+            The class to report metrics for binary dataset. If the target classes is True or False,
+            positive_class will be set to True by default. If the dataset is multiclass or multilabel,
+            this will be ignored.
+        legend_labels : dict, optional
+            List of legend labels. Defaults to `None`.
+            If legend_labels not specified class names will be used for plots.
+        use_case_type : str, optional
+            The type of problem this model is solving. This can be set during `prepare()`.
+            Examples: "binary_classification", "regression", "multinomial_classification"
+            Full list of supported types can be found here: `ads.common.model_metadata.UseCaseType`
+        filename: str, optional
+            If filename is given, the html report will be saved to the location specified.
+
+        Examples
+        --------
+        >>> import tempfile
+        >>> from ads.evaluations.evaluator import Evaluator
+        >>> from sklearn.tree import DecisionTreeClassifier
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.model_selection import train_test_split
+        >>> from ads.model.framework.sklearn_model import SklearnModel
+        >>> from ads.common.model_metadata import UseCaseType
+        >>>
+        >>> X, y = make_classification(n_samples=1000)
+        >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
+        >>> est = DecisionTreeClassifier().fit(X_train, y_train)
+        >>> model = SklearnModel(estimator=est, artifact_dir=tempfile.mkdtemp())
+        >>> model.prepare(
+                inference_conda_env="generalml_p38_cpu_v1",
+                training_conda_env="generalml_p38_cpu_v1",
+                X_sample=X_test,
+                y_sample=y_test,
+                use_case_type=UseCaseType.BINARY_CLASSIFICATION,
+            )
+        >>> model.evaluate(X_test, y_test, filename="report.html")
+        """
+
+        from ads.evaluations.evaluator import Evaluator
+
+        y_preds, y_scores = (
+            ([y_pred], [y_score]) if y_pred is not None else (None, None)
+        )
+        report = Evaluator(
+            models=[self],
+            X=X,
+            y=y,
+            y_preds=y_preds,
+            y_scores=y_scores,
+            X_train=X_train,
+            y_train=y_train,
+            classes=classes,
+            positive_class=positive_class,
+            legend_labels=legend_labels,
+            use_case_type=use_case_type,
+        )
+        if filename is None:
+            report.display(perfect=perfect)
+        else:
+            report.save(filename=filename, perfect=perfect)
+        return report