oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/dataset/capabilities.md

@@ -0,0 +1,51 @@
+### Dataset Capabilities
+
+The Dataset is an abstraction atop a variety of sources
+   - local file system via Pandas dataframe
+   - remote file systems like s3, Oracle Storage
+   - advanced possibilities include using pyarrow for Apache Arrow datsets
+   - interoperate with Dask.dataframes
+
+The sdk's dataset provides the following capabilities:
+- Support loading dataset from pandas, remote sources, clipboard etc
+  ```
+  df = pd.load_csv(...)
+  ds = sdk.Dataset.fromPandas(df)
+  ```
+- Visualizing of dataset
+    - Show all features, their data types and counts, meta data for a dataset, for example if the dataset is time-series or not.
+    - Show (heatmap) of correlated features
+      ```
+      show_in_notebook(ds)
+      ```
+    - Show distribution of values for features, using a datatype-dependent
+      visualizer (text vs categorical vs numerical will all render differently)
+      ```
+      show_in_notebook(ds, features=["col1", "col2", "col3"])
+      ```
+
+    look at http://pandas.pydata.org/pandas-docs/stable/style.html for some pandas
+    examples, also seaborn https://stackoverflow.com/questions/39409866/correlation-heatmap
+
+- Data cleaning
+    - Support redaction of features
+    - Handle missing values
+    - Identify outliers and inconsistencies
+    - Imputation
+- Identify data type of features
+    - Define a data type hierarchy that includes special patterns like credit cards, zip codes, phone number, etc.
+- Feature selection
+  - features used in a model is part of AutoML, however, for modeling that's not using AutoML the dataset
+    should have feature selection capabilities using a plugin (initial should be information theoretic) - see
+    http://scikit-learn.org/stable/modules/classes.html#module-sklearn.feature_selection for examples
+- Sampling
+  - can be used to limit data used for modeling, can also be used to balance and unbalanced target variable.
+  - "auto mode" determines if up/down is required
+- Features can hold feature encoding hints
+   -  these hints are part of the dataset, not the underlying pandas dataframe
+- Discretization
+  - of target variable for regression to classification problem
+
+- Persistence & interchange
+  - support save as snapshot (possibly parquet format), a snapshot generates a shareable URI
+  - load from snapshot URI

ads/dataset/classification_dataset.py

@@ -0,0 +1,339 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import pandas as pd
+import warnings
+
+from ads.common import utils, logger
+from ads.dataset import helper
+from ads.dataset.exception import ValidationError
+from ads.dataset.dataset_with_target import ADSDatasetWithTarget
+from sklearn.preprocessing import FunctionTransformer
+from ads.dataset.helper import deprecate_variable, deprecate_default_value
+
+
+class ClassificationDataset(ADSDatasetWithTarget):
+    """
+    Dataset for classification task
+    """
+
+    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
+        ADSDatasetWithTarget.__init__(
+            self, df=df, sampled_df=sampled_df, target=target, target_type=target_type, shape=shape, **kwargs
+        )
+
+    def auto_transform(
+        self,
+        fix_imbalance: bool = True,
+        correlation_threshold: float = 0.7,
+        frac: float = 1.0,
+        correlation_methods: str = "pearson",
+    ):
+        """
+        Return transformed dataset with several optimizations applied automatically.
+        The optimizations include:
+
+        - Dropping constant and primary key columns, which has no predictive quality,
+        - Imputation, to fill in missing values in noisy data:
+
+            - For continuous variables, fill with mean if less than 40% is missing, else drop,
+            - For categorical variables, fill with most frequent if less than 40% is missing, else drop,
+
+        - Dropping strongly co-correlated columns that tend to produce less generalizable models,
+        - Balancing dataset using up or down sampling.
+
+        Parameters
+        ----------
+        fix_imbalance : bool, defaults to True.
+            Fix imbalance between classes in dataset. Used only for classification datasets.
+        correlation_threshold: float, defaults to 0.7. It must be between 0 and 1, inclusive.
+            The correlation threshold where columns with correlation higher than the threshold will
+            be considered as strongly co-correlated and recommended to be taken care of.
+        frac: float, defaults to 1.0. Range -> (0, 1].
+            What fraction of the data should be used in the calculation?
+        correlation_methods: Union[list, str], defaults to 'pearson'.
+
+            - 'pearson': Use Pearson's Correlation between continuous features,
+            - 'cramers v': Use Cramer's V correlations between categorical features,
+            - 'correlation ratio': Use Correlation Ratio Correlation between categorical and continuous features,
+            - 'all': Is equivalent to ['pearson', 'cramers v', 'correlation ratio'].
+
+            Or a list containing any combination of these methods, for example, ['pearson', 'cramers v'].
+
+        Returns
+        -------
+        transformed_dataset : ADSDatasetWithTarget
+            The dataset after transformation
+
+        Examples
+        --------
+        >>> ds_clean = ds.auto_transform(correlation_threshold=0.6)
+        """
+        frac = deprecate_default_value(
+            frac,
+            None,
+            1,
+            f"<code>frac=None</code> is deprecated. Use <code>frac=1.0</code> instead.",
+            FutureWarning,
+        )
+        with utils.get_progress_bar(7) as progress:
+            df, sampled_df, transformer_pipeline = self._transform(
+                progress=progress,
+                fix_imbalance=fix_imbalance,
+                correlation_threshold=correlation_threshold,
+                frac=frac,
+                correlation_methods=correlation_methods,
+            )
+            return self._build_new_dataset(
+                df,
+                sampled_df=sampled_df,
+                transformers=transformer_pipeline.steps,
+                progress=progress,
+            )
+
+    def convert_to_text_classification(self, text_column: str):
+        """
+        Builds a new dataset with the given text column as the only feature besides target.
+
+        Parameters
+        ----------
+        text_column: str
+            Feature name to use for text classification task
+
+        Returns
+        -------
+        ds: TextClassificationDataset
+            Dataset with one text feature and a classification target
+
+        Examples
+        --------
+        >>> review_ds = DatasetFactory.open("review_data.csv")
+        >>> ds_text_class = review_ds.convert_to_text_classification('reviews')
+        """
+
+        def _select_features(df, feature_names, target):
+            if target in df.columns:
+                feature_names = feature_names + [target]
+            return df[feature_names]
+
+        transformer = (
+            f"convert_to_text_classification using feature {text_column}",
+            FunctionTransformer(
+                func=_select_features,
+                validate=False,
+                kw_args={
+                    "feature_names": [self.target.name, text_column],
+                    "target": self.target.name,
+                },
+            ).fit(self.sampled_df),
+        )
+        if utils.is_same_class(self, BinaryClassificationDataset):
+            new_ds = BinaryTextClassificationDataset(
+                self.df[[self.target.name, text_column]],
+                self.sampled_df[[self.target.name, text_column]],
+                self.target.name,
+                self.target.type,
+                (len(self.df), 2),
+                **self.init_kwargs,
+            )
+
+        else:
+            new_ds = MultiClassTextClassificationDataset(
+                self.df[[self.target.name, text_column]],
+                self.sampled_df[[self.target.name, text_column]],
+                self.target.name,
+                self.target.type,
+                (len(self.df), 2),
+                **self.init_kwargs,
+            )
+        new_ds.transformer_pipeline = self._update_transformer_pipeline(transformer)
+        return new_ds
+
+    def down_sample(self, sampler=None):
+        """
+        Fixes an imbalanced dataset by down-sampling.
+
+        Parameters
+        ----------
+        sampler: An instance of SamplerMixin
+            Should implement fit_resample(X,y) method. If None, does random down sampling.
+
+        Returns
+        -------
+        down_sampled_ds: ClassificationDataset
+            A down-sampled dataset.
+
+        Examples
+        --------
+        >>> ds = DatasetFactory.open("some_data.csv")
+        >>> ds_balanced_small = ds.down_sample()
+        """
+        return self._build_new_dataset(
+            helper.down_sample(self.df, self.target.name)
+            if sampler is None
+            else helper.sample(
+                sampler,
+                self.df.drop(self.target.name, axis=1),
+                self.df[self.target.name],
+            )
+        )
+
+    def up_sample(self, sampler="default"):
+        """
+        Fixes imbalanced dataset by up-sampling
+
+        Parameters
+        ----------
+        sampler: An instance of SamplerMixin
+            Should implement fit_resample(X,y) method.
+            If 'default', either SMOTE or random sampler will be used
+        fill_missing_type: a string
+            Can either be 'mean', 'mode' or 'median'.
+
+        Returns
+        -------
+        up_sampled_ds: ClassificationDataset
+            an up-sampled dataset
+
+        Examples
+        --------
+        >>> ds = DatasetFactory.open("some_data.csv")
+        >>> ds_balanced_large = ds.up_sample()
+        """
+        return self._build_new_dataset(
+            helper.up_sample(
+                self.df,
+                self.target.name,
+                sampler=sampler,
+                feature_types=self.feature_types,
+            )
+        )
+
+
+class BinaryClassificationDataset(ClassificationDataset):
+    """
+    Dataset for binary classification
+    """
+
+    def __init__(
+        self, df, sampled_df, target, target_type, shape, positive_class=None, **kwargs
+    ):
+        if positive_class is not None:
+            # map positive_class to True
+            update_arg = lambda x: x == positive_class
+
+            def mapper(df, column_name, arg):
+                df[column_name] = df[column_name].map(arg)
+                return df
+
+            df = mapper(df, target, update_arg)
+            sampled_df = mapper(sampled_df, target, update_arg)
+        ClassificationDataset.__init__(
+            self, df, sampled_df, target, target_type, shape, **kwargs
+        )
+
+    def set_positive_class(self, positive_class, missing_value=False):
+        """
+        Return new dataset with values in target column mapped to True or False
+        in accordance with the specified positive label.
+
+        Parameters
+        ----------
+        positive_class : same dtype as target
+            The target label which should be identified as positive outcome from model.
+        missing_value : bool
+            missing values will be converted to this
+
+        Returns
+        -------
+        dataset: same type as the caller
+
+        Raises
+        ------
+        ValidationError
+             if the positive_class is not present in target
+
+        Examples
+        --------
+        >>> ds = DatasetFactory.open("iris.csv")
+        >>> ds_with_target = ds.set_target('class')
+        >>> ds_with_pos_class = ds.set_positive_class('setosa')
+        """
+        if positive_class not in self.target.target_vals:
+            raise ValidationError(
+                "Positive label '%s' not in target values '%s'"
+                % (positive_class, self.target.target_vals)
+            )
+
+        return self.assign_column(
+            self.target.name,
+            lambda x: pd.isnull(x) and missing_value or x == positive_class,
+        )
+
+
+class MultiClassClassificationDataset(ClassificationDataset):
+    """
+    Dataset for multi-class classification
+    """
+
+    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
+        ClassificationDataset.__init__(
+            self, df, sampled_df, target, target_type, shape, **kwargs
+        )
+
+
+class BinaryTextClassificationDataset(BinaryClassificationDataset):
+    """
+    Dataset for binary text classification
+    """
+
+    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
+        BinaryClassificationDataset.__init__(
+            self, df, sampled_df, target, target_type, shape, **kwargs
+        )
+
+    def auto_transform(self):
+        """
+        Automatically chooses the most effective dataset transformation
+        """
+        logger.info("No optimizations.")
+        return self
+
+    def select_best_features(self, score_func=None, k=12):
+        """
+        Automatically chooses the best features and removes the rest
+        """
+        logger.info(
+            "There are an insufficient number of features to do feature selection."
+        )
+        return self
+
+
+class MultiClassTextClassificationDataset(MultiClassClassificationDataset):
+    """
+    Dataset for multi-class text classification
+    """
+
+    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
+        MultiClassClassificationDataset.__init__(
+            self, df, sampled_df, target, target_type, shape, **kwargs
+        )
+
+    def auto_transform(self):
+        """
+        Automatically chooses the most effective dataset transformation
+        """
+        logger.info("No optimizations.")
+        return self
+
+    def select_best_features(self, score_func=None, k=12):
+        """
+        Automatically chooses the best features and removes the rest
+        """
+        logger.info(
+            "There are an insufficient number of features to do feature selection."
+        )
+        return self

ads/dataset/correlation.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from __future__ import print_function, absolute_import
+
+import numpy as np
+import pandas as pd
+
+from ads.common import logger
+from ads.dataset.exception import ValidationError
+from collections import defaultdict, Counter
+from itertools import product, combinations
+from typing import Tuple
+
+
+def _cat_vs_cat(df: pd.core.frame.DataFrame, categorical_columns: list) -> pd.DataFrame:
+    """
+    calc the correlation of all pairs of categorical features and categorical features
+    """
+    if not categorical_columns:
+        return pd.DataFrame()
+    categorical_pairs = list(combinations(categorical_columns, 2))
+    corr_list = []
+    for col in categorical_pairs:
+        cat1_name = col[0]
+        cat2_name = col[1]
+        _check_if_same_type(df[col[0]], cat1_name)
+        _check_if_same_type(df[col[1]], cat2_name)
+        corr_list.append(
+            _cramers_v(np.array(df[col[0]].values), np.array(df[col[1]].values))
+        )
+    correlation_matrix = _list_to_dataframe(categorical_pairs, corr_list)
+    return correlation_matrix
+
+
+def _cat_vs_cts(
+    df: pd.core.frame.DataFrame, categorical_columns: list, continuous_columns: list
+) -> pd.DataFrame:
+    """
+    calc the correlation of all pairs of categorical features and continuous features
+    """
+    numerical_categorical_pairs = list(product(categorical_columns, continuous_columns))
+    corr_list = []
+    for col in numerical_categorical_pairs:
+        corr_list.append(
+            _correlation_ratio(np.array(df[col[0]].values), np.array(df[col[1]].values))
+        )
+    correlation_matrix = _list_to_dataframe(numerical_categorical_pairs, corr_list)
+    return correlation_matrix
+
+
+def _list_to_dataframe(name_list: list, corr_list: list) -> pd.DataFrame:
+    corr_dict = defaultdict(dict)
+    for idx, corr in zip(name_list, corr_list):
+        row_name = idx[0]
+        col_name = idx[1]
+        corr_dict[row_name][col_name] = corr_dict[col_name][row_name] = round(corr, 4)
+        corr_dict[row_name][row_name] = corr_dict[col_name][col_name] = 1.0000
+    correlation_matrix = pd.DataFrame.from_dict(corr_dict).sort_index()
+    correlation_matrix = correlation_matrix.loc[:, correlation_matrix.index]
+    return correlation_matrix
+
+
+def _correlation_ratio(cat: np.ndarray, cts: np.ndarray):
+    """
+    calc the correlation of a pair of a categorical feature and a continuous feature
+    using correlation ratio when input are two numpy arrays
+    """
+    keep_cts = ~pd.isnull(cts)
+    cat_no_nan = cat[keep_cts]
+    cts_no_nan = cts[keep_cts]
+
+    keep_cat = ~pd.isnull(cat_no_nan)
+    cat_no_none = cat_no_nan[keep_cat]
+    cts_no_none = cts_no_nan[keep_cat]
+
+    unq_cat, tags, group_count = np.unique(
+        list(cat_no_none), return_inverse=1, return_counts=1
+    )
+    group_mean = np.bincount(tags, cts_no_none) / group_count
+    overall_mean = np.nanmean(cts_no_none)
+    n = len(cts_no_none)
+
+    dispersion_within = np.dot(group_count, np.square(group_mean - overall_mean))
+    dispersion_population = cts_no_none.var() * n
+    ratio = dispersion_within / dispersion_population
+
+    return np.sqrt(ratio)
+
+
+def _count_occurrence(
+    cat1: np.ndarray, cat2: np.ndarray
+) -> Tuple[np.ndarray, int, int]:
+    """
+    calc the contingency table of two arrays
+    """
+    occurance_cnt = Counter([(x, y) for x, y in zip(cat1, cat2)])
+    nunique_cat1 = np.unique(cat1[~pd.isnull(cat1)])
+    nunique_cat2 = np.unique(cat2[~pd.isnull(cat2)])
+    r = len(nunique_cat1)
+    k = len(nunique_cat2)
+    contigency_table = np.zeros((r, k))
+    for row, num1 in enumerate(nunique_cat1):
+        for col, num2 in enumerate(nunique_cat2):
+            contigency_table[row, col] = occurance_cnt[(num1, num2)]
+
+    return contigency_table, r, k
+
+
+def _chi_squared(count_matrix: np.ndarray, n_obs: int) -> float:
+    """
+    Compute Chi-squared when given a contingency table
+    """
+    row_sums = np.tile(np.sum(count_matrix, axis=1), (count_matrix.shape[1], 1)).T
+    col_sums = np.tile(np.sum(count_matrix, axis=0), (count_matrix.shape[0], 1))
+    return np.sum(
+        np.square(count_matrix - row_sums * col_sums / n_obs)
+        / (row_sums * col_sums / n_obs)
+    )
+
+
+def _cramers_v(cat1: np.ndarray, cat2: np.ndarray) -> float:
+    """
+    calc the cramers v of two numpy arrays
+    """
+    n = len(cat1)
+    if n == 1:
+        return 0
+    contigency_table, r, k = _count_occurrence(cat1, cat2)
+
+    if r == 0:
+        return 0.0000
+
+    chi2 = _chi_squared(contigency_table, n)
+    phi2 = chi2 / n
+
+    phi2corr = max(0, phi2 - ((k - 1) * (r - 1)) / (n - 1))
+    rcorr = r - (np.square(r - 1)) / (n - 1)
+    kcorr = k - (np.square(k - 1)) / (n - 1)
+    denominator = min((kcorr - 1), (rcorr - 1))
+    if denominator == 0:
+        return np.nan
+    return np.sqrt(phi2corr / denominator)
+
+
+def _get_columns_by_type(
+    feature_types_df: pd.DataFrame, threshold: float = 0.8
+) -> Tuple[list, list, list]:
+    """
+    return the categorical columns, continuous columns and columns of other types
+    """
+    missing = feature_types_df.loc[:, "missing_percentage"] > threshold
+    not_missing = feature_types_df.loc[:, "missing_percentage"] <= threshold
+    missing_columns = list(feature_types_df.loc[missing, "feature_name"].values)
+    constant_columns = list(
+        feature_types_df.loc[
+            (feature_types_df.loc[:, "type"].isin(["constant"])), "feature_name"
+        ].values
+    )
+    categorical_columns = list(
+        feature_types_df.loc[
+            (feature_types_df.loc[:, "type"].isin(["categorical", "zipcode"]))
+            & not_missing,
+            "feature_name",
+        ].values
+    )
+    continuous_columns = list(
+        feature_types_df.loc[
+            (feature_types_df.loc[:, "type"].isin(["continuous", "ordinal"]))
+            & not_missing,
+            "feature_name",
+        ].values
+    )
+    other_columns = list(
+        set(feature_types_df.index.values)
+        - set(categorical_columns)
+        - set(continuous_columns)
+        - set(missing_columns)
+        - set(constant_columns)
+    )
+
+    if missing_columns:
+        logger.info(
+            f"The columns {missing_columns} are not included because more than {threshold}% of the values are missing. "
+        )
+        logger.info(
+            f"The columns {missing_columns} are not included because more than {threshold}% of the values are missing. "
+            f"Adjust this threshold using the `nan_threshold` parameter."
+        )
+    if constant_columns:
+        logger.info(
+            " The constant columns {} are not included.".format(constant_columns)
+        )
+    if other_columns:
+        logger.info(
+            f" The columns {other_columns} are not included because more than {threshold}% of the values are missing, or "
+            f"they are not one of the following types: "
+            f"`categorical`, `zipcode`, `continuous`"
+            f", or `ordinal`."
+        )
+
+    return categorical_columns, continuous_columns, other_columns
+
+
+def _validate_correlation_methods(correlation_methods):
+    if isinstance(correlation_methods, str):
+        correlation_methods = [correlation_methods]
+    for method in correlation_methods:
+        if method not in ["all", "pearson", "cramers v", "correlation ratio"]:
+            raise ValidationError(f"{method} is not supported.")
+    if "all" in [method for method in correlation_methods]:
+        correlation_methods = ["pearson", "cramers v", "correlation ratio"]
+    return correlation_methods
+
+
+def _check_if_same_type(series, col_name):
+    col = series.dropna().values
+    if len(col) > 0:
+        col_type = type(col[0])
+        if not all([isinstance(x, col_type) for x in col]):
+            raise TypeError(
+                f"More than one data type in the column `{col_name}`. Keep all the values in that column the same type."
+            )