oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/feature_engineering/accessor/mixin/eda_mixin.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+This exploratory data analysis (EDA) Mixin is used in the ADS accessor for the Pandas Dataframe.
+The series of purpose-driven methods enable the data scientist to complete analysis on the dataframe.
+
+From the accessor we have access to the pandas object the user is interacting with as well as
+corresponding lists of feature types per column.
+"""
+
+import collections
+import pandas as pd
+import matplotlib.pyplot as plt
+from ads.feature_engineering.accessor.mixin.correlation import (
+    cat_vs_cat,
+    cont_vs_cont,
+    cat_vs_cont,
+)
+from ads.feature_engineering.accessor.mixin.utils import (
+    _continuous_columns,
+    _categorical_columns,
+    _sienna_light_to_dark_color_palette,
+)
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+class EDAMixin:
+    def feature_count(self) -> pd.DataFrame:
+        """
+        Counts the number of columns for each feature type and each primary feature.
+        The column of primary is the number of primary feature types that is assigned to the column.
+
+        Returns
+        -------
+        Dataframe with
+          The number of columns for each feature type
+          The number of columns for each primary feature
+
+        Examples
+        --------
+        >>> df.ads.feature_type
+        {'PassengerId': ['ordinal', 'category'],
+        'Survived': ['ordinal'],
+        'Pclass': ['ordinal'],
+        'Name': ['category'],
+        'Sex': ['category']}
+        >>> df.ads.feature_count()
+            Feature Type        Count       Primary
+        0       category            3             2
+        1        ordinal            3             3
+        """
+
+        feature_count = collections.defaultdict(lambda: [0, 0])
+
+        for _, feature_types in self.feature_type.items():
+            feature_count[feature_types[0]][1] += 1
+            for ft in feature_types:
+                feature_count[ft][0] += 1
+
+        return pd.DataFrame(
+            [
+                (feature_type, count, primary)
+                for feature_type, (count, primary) in feature_count.items()
+            ],
+            columns=["Feature Type", "Count", "Primary"],
+        )
+
+    def feature_stat(self) -> pd.DataFrame:
+        """Summary statistics Dataframe provided.
+
+        This returns feature stats on each column using FeatureType summary method.
+
+        Examples
+        --------
+        >>> df = pd.read_csv('~/advanced-ds/tests/vor_datasets/vor_titanic.csv')
+        >>> df.ads.feature_stat().head()
+                 Column	   Metric	                Value
+        0	PassengerId	    count	                891.000
+        1	PassengerId	    mean	                446.000
+        2	PassengerId	    standard deviation	    257.354
+        3	PassengerId	    sample minimum  	    1.000
+        4	PassengerId	    lower quartile	        223.500
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            Dataframe with 3 columns: name, metric, value
+
+        """
+        stats = []
+        for col_name, col in self._obj.items():
+            for _, row in col.ads.feature_stat().iterrows():
+                stats.append([col_name, row["Metric"], row["Value"]])
+        df_stats = pd.DataFrame(stats, columns=["Column", "Metric", "Value"])
+        df_stats.value = df_stats.Value.round(3)
+        return df_stats
+
+    def feature_plot(self) -> pd.DataFrame:
+        """For every column in the dataframe plot generate a list of summary plots based on the most
+        relevant feature type.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            Dataframe with 2 columns:
+            1. Column - feature name
+            2. Plot - plot object
+        """
+        plots = []
+        for _, col in self._obj.items():
+            try:
+                plot = col.ads.feature_plot()
+            except:
+                plot = None
+            plots.append([col.name, plot])
+        return pd.DataFrame(plots, columns=["Column", "Plot"])
+
+    def pearson(self) -> pd.DataFrame:
+        """Generate a Pearson correlation data frame for all continuous variable pairs.
+
+        Gives a warning for dropped non-numerical columns.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+        Pearson correlation data frame with the following 3 columns:
+            1. Column 1 (name of the first continuous column)
+            2. Column 2 (name of the second continuous column)
+            3. Value (correlation value)
+
+        Note
+        ____
+        Pairs will be replicated. For example for variables x and y, we'd have (x,y), (y,x) both with same correlation value. We'll also have (x,x) and (y,y) with value 1.0.
+        """
+        continuous_cols = _continuous_columns(self._obj.ads.feature_type)
+        return cont_vs_cont(self._obj[continuous_cols])
+
+    @runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
+    def pearson_plot(self) -> plt.Axes:
+        """Generate a heatmap of the Pearson correlation for all continuous variable pairs.
+
+        Returns
+        -------
+        Plot object
+            Pearson correlation plot object that can be updated by the customer
+        """
+        ax = plt.axes()
+        df = (
+            self.pearson()
+            .pivot_table(index="Column 1", columns="Column 2", values="Value")
+            .rename_axis("")
+            .rename_axis("", axis="columns")
+        )
+        ax.set_title("Pearson's Correlation")
+        return seaborn.heatmap(df, cmap=_sienna_light_to_dark_color_palette(), ax=ax)
+
+    def cramersv(self) -> pd.DataFrame:
+        """Generate a Cramer's V correlation data frame for all categorical variable pairs.
+
+        Gives a warning for dropped non-categorical columns.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            Cramer's V correlation data frame with the following 3 columns:
+                1. Column 1 (name of the first categorical column)
+                2. Column 2 (name of the second categorical column)
+                3. Value (correlation value)
+        Note
+        ____
+        Pairs will be replicated. For example for variables x and y, we would have (x,y), (y,x) both with same correlation value. We will also have (x,x) and (y,y) with value 1.0.
+        """
+        categorical_cols = _categorical_columns(self._obj.ads.feature_type)
+        return cat_vs_cat(self._obj[categorical_cols])
+
+    @runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
+    def cramersv_plot(self) -> plt.Axes:
+        """Generate a heatmap of the Cramer's V correlation for all categorical variable pairs.
+
+        Gives a warning for dropped non-categorical columns.
+
+        Returns
+        -------
+        Plot object
+            Cramer's V correlation plot object that can be updated by the customer
+        """
+        ax = plt.axes()
+        df = (
+            self.cramersv()
+            .pivot_table(index="Column 1", columns="Column 2", values="Value")
+            .rename_axis("")
+            .rename_axis("", axis="columns")
+        )
+        ax.set_title("Cramer's V")
+        return seaborn.heatmap(df, cmap=_sienna_light_to_dark_color_palette(), ax=ax)
+
+    def correlation_ratio(self) -> pd.DataFrame:
+        """Generate a Correlation Ratio data frame for all categorical-continuous variable pairs.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+        Correlation Ratio correlation data frame with the following 3 columns:
+            1. Column 1 (name of the first categorical/continuous column)
+            2. Column 2 (name of the second categorical/continuous column)
+            3. Value (correlation value)
+
+        Note
+        ____
+        Pairs will be replicated. For example for variables x and y, we would have (x,y), (y,x) both with same correlation value. We will also have (x,x) and (y,y) with value 1.0.
+        """
+        categorical_cols = _categorical_columns(self._obj.ads.feature_type)
+        continuous_cols = _continuous_columns(self._obj.ads.feature_type)
+        return cat_vs_cont(self._obj, categorical_cols, continuous_cols)
+
+    @runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
+    def correlation_ratio_plot(self) -> plt.Axes:
+        """Generate a heatmap of the Correlation Ratio correlation for all categorical-continuous variable
+        pairs.
+
+        Returns
+        -------
+        Plot object
+            Correlation Ratio correlation plot object that can be updated by the customer
+        """
+        ax = plt.axes()
+        df = (
+            self.correlation_ratio()
+            .pivot_table(index="Column 1", columns="Column 2", values="Value")
+            .rename_axis("")
+            .rename_axis("", axis="columns")
+        )
+        ax.set_title("Correlation Ratio")
+        return seaborn.heatmap(df, cmap=_sienna_light_to_dark_color_palette(), ax=ax)
+
+    def warning(self) -> pd.DataFrame:
+        """Generates a data frame that lists feature specific warnings.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            The list of feature specific warnings.
+
+        Examples
+        --------
+        >>> df.ads.warning()
+            Column    Feature Type         Warning               Message       Metric    Value
+        --------------------------------------------------------------------------------------
+        0      Age      continuous           Zeros      Age has 38 zeros        Count       38
+        1      Age      continuous           Zeros   Age has 12.2% zeros   Percentage    12.2%
+        """
+        common_columns = ["Feature Type", "Warning", "Message", "Metric", "Value"]
+        result_df = pd.DataFrame((), columns=["Column"] + common_columns)
+        for col in self._obj.columns:
+            warning_df = self._obj[col].ads.warning()
+            if warning_df is not None:
+                warning_df.insert(0, "Column", col)
+                result_df = pd.concat([result_df, warning_df])
+        return result_df.reset_index(drop=True)

ads/feature_engineering/accessor/mixin/eda_mixin_series.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+This exploratory data analysis (EDA) Mixin is used in the ADS accessor for the Pandas Series.
+The series of purpose-driven methods enable the data scientist to complete univariate analysis.
+
+From the accessor we have access to the pandas object the user is interacting with as well as
+corresponding list of feature types.
+"""
+
+import pandas as pd
+import matplotlib.pyplot as plt
+
+
+class EDAMixinSeries:
+    def feature_stat(self) -> pd.DataFrame:
+        """Summary statistics Dataframe provided.
+
+        This returns feature stats on series using FeatureType summary method.
+
+        Examples
+        --------
+        >>> df = pd.read_csv('~/advanced-ds/tests/vor_datasets/vor_titanic.csv')
+        >>> df['Cabin'].ads.feature_stat()
+            Metric      Value
+        0	count	    891
+        1	unqiue	    147
+        2	missing	    687
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            Dataframe with 2 columns and rows for different metric values
+
+        """
+        for feature_type in self._feature_type:
+            if hasattr(feature_type, "feature_stat"):
+                stat = feature_type.feature_stat(self._obj).reset_index()
+                stat.columns = ["Metric", "Value"]
+                return stat
+        return None
+
+    def feature_plot(self) -> plt.Axes:
+        """For the series generate a summary plot based on the most relevant feature type.
+
+        Returns
+        -------
+        matplotlib.axes._subplots.AxesSubplot
+           Plot object for the series based on the most relevant feature type.
+        """
+        for feature_type in self._feature_type:
+            if hasattr(feature_type, "feature_plot"):
+                return feature_type.feature_plot(self._obj)
+        return None
+
+    def warning(self) -> pd.DataFrame:
+        """Generates a data frame that lists feature specific warnings.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            The list of feature specific warnings.
+
+        Examples
+        --------
+        >>> df["Age"].ads.warning()
+          Feature Type       Warning               Message         Metric      Value
+         ---------------------------------------------------------------------------
+        0   continuous         Zeros      Age has 38 zeros          Count         38
+        1   continuous         Zeros   Age has 12.2% zeros     Percentage      12.2%
+        """
+        common_columns = ["Feature Type", "Warning", "Message", "Metric", "Value"]
+        result_df = pd.DataFrame((), columns=common_columns)
+        for feature_type in self._feature_type:
+            if hasattr(feature_type, "warning"):
+                warning_df = feature_type.warning(self._obj)
+                if warning_df is not None:
+                    warning_df.insert(0, "Feature Type", feature_type.name)
+                    result_df = pd.concat([result_df, warning_df])
+        result_df.reset_index(drop=True, inplace=True)
+        return result_df

ads/feature_engineering/accessor/mixin/feature_types_mixin.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+The module that represents the ADS Feature Types Mixin class that extends
+Pandas Series and Dataframe accessors.
+
+Classes
+-------
+    ADSFeatureTypesMixin
+        ADS Feature Types Mixin class that extends Pandas Series and Dataframe accessors.
+"""
+import inspect
+from typing import Union
+
+import pandas as pd
+import tabulate
+from ads.feature_engineering.feature_type.base import FeatureType
+from ads.feature_engineering.feature_type_manager import (
+    FeatureTypeManager as feature_type_manager,
+)
+from ads.feature_engineering.feature_type_manager import _feature_type_by_dtype
+
+
+class ADSFeatureTypesMixin:
+    """ADS Feature Types Mixin class that extends Pandas Series and DataFrame accessors.
+
+    Methods
+    -------
+    warning_registered(cls) -> pd.DataFrame
+        Lists registered warnings for registered feature types.
+    validator_registered(cls) -> pd.DataFrame
+        Lists registered validators for registered feature types.
+    help(self, prop: str = None) -> None
+        Help method that prints either a table of available properties or, given a property,
+        returns its docstring.
+    """
+
+    @staticmethod
+    def _feature_type_by_dtype(dtype) -> FeatureType:
+        """Determines feature type by DataFrame dtype.
+
+        Parameters
+        ----------
+        dtype: pd.DataFrame.dtypes
+            The Pandas series data type.
+
+        Returns
+        -------
+        FeatureType
+            The subclass of FeatureType.
+        """
+        return _feature_type_by_dtype(dtype)
+
+    @staticmethod
+    def _is_type_registered(feature_type: Union[FeatureType, str]) -> bool:
+        """Checks if provided feature type is registered in the system.
+
+        Parameters
+        ----------
+        feature_type: Union[FeatureType, str]
+            The FeatureType subclass or a str indicating feature type.
+
+        Returns
+        -------
+        bool
+            True if provided feature type registered, False otherwise.
+        """
+        return feature_type_manager.is_type_registered(feature_type)
+
+    @staticmethod
+    def _get_type(feature_type: Union[FeatureType, str]) -> FeatureType:
+        """Gets a feature type by class object or name.
+
+        Parameters
+        ----------
+        feature_type: Union[FeatureType, str]
+            The FeatureType subclass or a str indicating feature type.
+
+        Returns
+        -------
+        FeatureType
+            Found feature type.
+        """
+        return feature_type_manager.feature_type_object(feature_type)
+
+    def warning_registered(self) -> pd.DataFrame:
+        """Lists registered warnings for all registered feature types.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            The list of registered warnings for registered feature types.
+
+        Examples
+        --------
+        >>> df.ads.warning_registered()
+               Column    Feature Type             Warning                    Handler
+           -------------------------------------------------------------------------
+           0      Age      continuous               zeros              zeros_handler
+           1      Age      continuous    high_cardinality   high_cardinality_handler
+
+        >>> df["Age"].ads.warning_registered()
+               Feature Type             Warning                    Handler
+           ---------------------------------------------------------------
+           0     continuous               zeros              zeros_handler
+           1     continuous    high_cardinality   high_cardinality_handler
+        """
+        common_columns = ["Feature Type", "Warning", "Handler"]
+        if isinstance(self._obj, pd.DataFrame):
+            result_df = pd.DataFrame((), columns=["Column"] + common_columns)
+            for col in self._obj.columns:
+                feature_type_df = self._obj[col].ads.warning_registered()
+                feature_type_df.insert(0, "Column", col)
+                result_df = pd.concat([result_df, feature_type_df])
+        else:
+            result_df = pd.DataFrame((), columns=common_columns)
+            for feature_type in self._feature_type:
+                feature_type_df = feature_type.warning.registered()
+                feature_type_df.insert(0, "Feature Type", feature_type.name)
+                feature_type_df = feature_type_df.rename(columns={"Name": "Warning"})
+                result_df = pd.concat([result_df, feature_type_df])
+        result_df.reset_index(drop=True, inplace=True)
+        return result_df
+
+    def validator_registered(self) -> pd.DataFrame:
+        """Lists registered validators for registered feature types.
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            The list of registered validators for registered feature types
+
+        Examples
+        --------
+        >>> df.ads.validator_registered()
+                 Column     Feature Type        Validator                 Condition                    Handler
+        ------------------------------------------------------------------------------------------------------
+        0   PhoneNumber    phone_number   is_phone_number                        ()            default_handler
+        1   PhoneNumber    phone_number   is_phone_number    {'country_code': '+7'}   specific_country_handler
+        2    CreditCard    credit_card     is_credit_card                        ()            default_handler
+
+        >>> df['PhoneNumber'].ads.validator_registered()
+            Feature Type            Validator                 Condition                     Handler
+        -------------------------------------------------------------------------------------------
+        0   phone_number      is_phone_number                        ()             default_handler
+        1   phone_number      is_phone_number    {'country_code': '+7'}    specific_country_handler
+        """
+        common_columns = ["Feature Type", "Validator", "Condition", "Handler"]
+        if isinstance(self._obj, pd.DataFrame):
+            result_df = pd.DataFrame((), columns=["Column"] + common_columns)
+            for col in self._obj.columns:
+                feature_type_df = self._obj[col].ads.validator_registered()
+                feature_type_df.insert(0, "Column", col)
+                result_df = pd.concat([result_df, feature_type_df])
+        else:
+            result_df = pd.DataFrame((), columns=common_columns)
+            for feature_type in self._feature_type:
+                feature_type_df = feature_type.validator.registered()
+                feature_type_df.insert(0, "Feature Type", feature_type.name)
+                feature_type_df = feature_type_df.rename(columns={"Name": "Validator"})
+                result_df = pd.concat([result_df, feature_type_df])
+        result_df.reset_index(drop=True, inplace=True)
+        return result_df
+
+    def help(self, prop: str = None) -> None:
+        """Help method that prints either a table of available properties or, given an individual property,
+        returns its docstring.
+
+        Parameters
+        ----------
+        prop : str
+            The Name of property.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+        if prop:
+            if hasattr(self, prop):
+                print(inspect.getdoc(getattr(self, prop)))
+                return
+            print(f"Property {prop} not found.")
+            return
+
+        methods = set()
+        attrs = set()
+
+        def get_attr(c):
+            for attr in dir(c):
+                if not attr.startswith("__") and not attr.startswith("_"):
+                    attr_obj = getattr(c, attr)
+                    if callable(attr_obj):
+                        doc = inspect.getdoc(attr_obj)
+                        if doc and len(doc.split(".")) > 0:
+                            methods.add((attr, doc.split(".")[0]))
+                        else:
+                            methods.add((attr, "method"))
+                    else:
+                        if hasattr(c.__class__, attr):
+                            attr_obj = getattr(c.__class__, attr)
+                            doc = inspect.getdoc(attr_obj)
+                            attrs.add((attr, doc.split(".")[0]))
+
+        get_attr(self)
+        props = sorted(list(methods) + list(attrs))
+        print(tabulate.tabulate(props, headers=("Property", "Description")))

ads/feature_engineering/accessor/mixin/utils.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Dict, List
+from matplotlib.colors import LinearSegmentedColormap
+
+
+def _sienna_light_to_dark_color_palette():
+    return LinearSegmentedColormap.from_list("", ["#FDF3E4", "#512C1B"])
+
+
+def _continuous_columns(feature_types: Dict) -> List[str]:
+    """
+    Parameters
+    ----------
+    feature_types : Dict
+        Column name mapping to list of feature types ordered by most to least relevant.
+
+    Returns
+    -------
+    List[str]
+        List of columns that have continuous or ordinal in the feature type list.
+
+    Note
+    ____
+    if a column has both ordinal/continuous and categorical pick whichever comes first.
+    """
+    continuous_cols = []
+    for col in feature_types:
+        for feature_type in feature_types[col]:
+            if feature_type == "continuous" or feature_type == "ordinal":
+                continuous_cols.append(col)
+                break
+            if feature_type == "category":
+                break
+    return continuous_cols
+
+
+def _categorical_columns(feature_types: Dict) -> List[str]:
+    """
+    Parameters
+    ----------
+    feature_types : Dict
+        column name mapping to list of feature types ordered by most to least relevant.
+
+    Returns
+    -------
+    List[str]
+        List of columns that have categorical in the feature type list.
+    Note
+    ____
+    if a column has both ordinal/continuous and categorical pick whichever comes first.
+    """
+    categorical_cols = []
+    for col in feature_types:
+        for feature_type in feature_types[col]:
+            if feature_type == "category":
+                categorical_cols.append(col)
+                break
+            if feature_type == "continuous" or feature_type == "ordinal":
+                break
+    return categorical_cols