oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/dataset/dataset_with_target.py

@@ -1,995 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import absolute_import, print_function
-
-import abc
-import importlib
-from collections import defaultdict
-from numbers import Number
-from typing import Tuple, Union
-
-import pandas as pd
-from ads.common import utils, logger
-from ads.common.data import ADSData
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
-from ads.dataset import helper
-from ads.dataset.dataset import ADSDataset
-from ads.dataset.feature_engineering_transformer import FeatureEngineeringTransformer
-from ads.dataset.feature_selection import FeatureImportance
-from ads.dataset.helper import (
-    DatasetDefaults,
-    deprecate_default_value, 
-    deprecate_variable, 
-    generate_sample,
-    get_target_type,
-    is_text_data,
-)
-from ads.dataset.label_encoder import DataFrameLabelEncoder
-from ads.dataset.pipeline import TransformerPipeline
-from ads.dataset.progress import DummyProgressBar
-from ads.dataset.recommendation import Recommendation
-from ads.dataset.recommendation_transformer import RecommendationTransformer
-from ads.dataset.target import TargetVariable
-from ads.type_discovery.typed_feature import (
-    CategoricalTypedFeature,
-    ContinuousTypedFeature,
-    DocumentTypedFeature,
-    GISTypedFeature,
-    OrdinalTypedFeature,
-    TypedFeature,
-    DateTimeTypedFeature, 
-    TypedFeature
-)
-from sklearn.model_selection import train_test_split
-from pandas.io.formats.printing import pprint_thing
-from sklearn.preprocessing import FunctionTransformer
-from abc import ABCMeta
-
-
-class ADSDatasetWithTarget(ADSDataset, metaclass=ABCMeta):
-    """
-    This class provides APIs for preparing dataset for modeling.
-    """
-
-    def __init__(
-        self,
-        df,
-        target,
-        sampled_df=None,
-        shape=None,
-        target_type=None,
-        sample_max_rows=-1,
-        type_discovery=True,
-        types={},
-        parent=None,
-        name="",
-        metadata=None,
-        transformer_pipeline=None,
-        description=None,
-        progress=DummyProgressBar(),
-        **kwargs,
-    ):
-        self.recommendation_transformer = None
-        if shape is None:
-            shape = df.shape
-        if sampled_df is None:
-            sampled_df = generate_sample(
-                df,
-                shape[0],
-                DatasetDefaults.sampling_confidence_level,
-                DatasetDefaults.sampling_confidence_interval,
-                **kwargs,
-            )
-
-        if parent is None:
-            cols = sampled_df.columns.tolist()
-            cols.insert(0, cols.pop(cols.index(target)))
-            ADSDataset.__init__(
-                self,
-                df,
-                sampled_df[[*cols]],
-                shape,
-                name=name,
-                description=description,
-                type_discovery=type_discovery,
-                types=types,
-                progress=progress,
-                metadata=metadata,
-                transformer_pipeline=transformer_pipeline,
-                sample_max_rows=sample_max_rows,
-            )
-        else:
-            self.__dict__ = parent.__dict__.copy()
-            cols = self.sampled_df.columns.tolist()
-            cols.insert(0, cols.pop(cols.index(target)))
-
-            self.sampled_df = parent.sampled_df[[*cols]]
-
-            # if parent has already been built, just reorder the columns to display the plot for target at beginning
-            if parent.correlation is None:
-                self.corr_futures = parent.corr_futures
-            else:
-                corr_cols = parent.sampled_df.select_dtypes(
-                    exclude=["object"]
-                ).columns.values.tolist()
-                corr_cols.insert(0, corr_cols.pop(corr_cols.index(target)))
-                self.correlation = parent.correlation.reindex(corr_cols)[[corr_cols]]
-            self.feature_types = parent.feature_types
-            self.feature_dist_html_dict = {}
-            if len(parent.feature_dist_html_dict) > 0:
-                parent_feature_dist_html_dict = parent.feature_dist_html_dict.copy()
-                self.feature_dist_html_dict = {
-                    target: parent_feature_dist_html_dict.pop(target)
-                }
-                self.feature_dist_html_dict.update(parent_feature_dist_html_dict)
-
-        # drop all rows where target is nan
-        target = target.strip().replace(" ", "_")
-
-        #
-        # as an optimization only dropna and regenerate sample when the target
-        # has na values
-        #
-
-        if self.df[target].isna().sum():
-            #
-            # remove rows for which the target is null
-            #
-            self.df = self.df.dropna(subset=[target])
-
-            #
-            # we cannot simply drop null values from the sampled_df after a change
-            # to the df - we must rebuild the sample from the new df
-            #
-            self.sampled_df = helper.generate_sample(
-                self.df,
-                sampled_df.shape[0],
-                helper.DatasetDefaults.sampling_confidence_level,
-                helper.DatasetDefaults.sampling_confidence_interval,
-            )
-            #
-            # after regenerating the sample we need to move the target back to the head
-            #
-            cols = self.sampled_df.columns.tolist()
-            cols.insert(0, cols.pop(cols.index(target)))
-            self.sampled_df = self.sampled_df[[*cols]]
-
-        if target_type is None:
-            target_type = get_target_type(target, sampled_df, **kwargs)
-        self.target = TargetVariable(self, target, target_type)
-
-        # remove target from type discovery conversion
-        for step in self.transformer_pipeline.steps:
-            if (
-                step[0] == "type_discovery"
-                and self.target.name in step[1].kw_args["dtypes"]
-            ):
-                step[1].kw_args["dtypes"].pop(self.target.name)
-
-    @staticmethod
-    def from_dataframe(
-        df: pd.DataFrame,
-        target: str,
-        sampled_df: pd.DataFrame = None,
-        shape: Tuple[int, int] = None,
-        target_type: TypedFeature = None,
-        positive_class=None,
-        **init_kwargs,
-    ):
-        from ads.dataset.classification_dataset import (
-            BinaryClassificationDataset, 
-            BinaryTextClassificationDataset, 
-            MultiClassClassificationDataset, 
-            MultiClassTextClassificationDataset
-        )
-        from ads.dataset.forecasting_dataset import ForecastingDataset
-        from ads.dataset.regression_dataset import RegressionDataset
-
-        if sampled_df is None:
-            sampled_df = generate_sample(
-                df,
-                (shape or df.shape)[0],
-                DatasetDefaults.sampling_confidence_level,
-                DatasetDefaults.sampling_confidence_interval,
-                **init_kwargs,
-            )
-
-        if target not in df:
-            raise ValueError(
-                f"{target} column doesn't exist in data frame. Specify a valid one instead."
-            )
-            
-        if target_type is None:
-            target_type = get_target_type(target, sampled_df, **init_kwargs)
-
-        if len(df[target].dropna()) == 0:
-            logger.warning(
-                "It is not recommended to use an empty column as the target variable."
-            )
-            raise ValueError(
-                f"We do not support using empty columns as the chosen target"
-            )
-        if utils.is_same_class(target_type, ContinuousTypedFeature):
-            return RegressionDataset(
-                df=df,
-                sampled_df=sampled_df,
-                target=target,
-                target_type=target_type,
-                shape=shape,
-                **init_kwargs,
-            )
-        elif utils.is_same_class(
-            target_type, DateTimeTypedFeature
-        ) or df.index.dtype.name.startswith("datetime"):
-            return ForecastingDataset(
-                df=df,
-                sampled_df=sampled_df,
-                target=target,
-                target_type=target_type,
-                shape=shape,
-                **init_kwargs,
-            )
-
-        # Adding ordinal typed feature, but ultimately we should rethink how we want to model this type
-        elif utils.is_same_class(target_type, CategoricalTypedFeature) or utils.is_same_class(
-            target_type, OrdinalTypedFeature
-        ):
-            if target_type.meta_data["internal"]["unique"] == 2:
-                if is_text_data(sampled_df, target):
-                    return BinaryTextClassificationDataset(
-                        df=df,
-                        sampled_df=sampled_df,
-                        target=target,
-                        shape=shape,
-                        target_type=target_type,
-                        positive_class=positive_class,
-                        **init_kwargs,
-                    )
-
-                return BinaryClassificationDataset(
-                    df=df,
-                    sampled_df=sampled_df,
-                    target=target,
-                    shape=shape,
-                    target_type=target_type,
-                    positive_class=positive_class,
-                    **init_kwargs,
-                )
-            else:
-                if is_text_data(sampled_df, target):
-                    return MultiClassTextClassificationDataset(
-                        df=df,
-                        sampled_df=sampled_df,
-                        target=target,
-                        target_type=target_type,
-                        shape=shape,
-                        **init_kwargs,
-                    )
-                return MultiClassClassificationDataset(
-                    df=df,
-                    sampled_df=sampled_df,
-                    target=target,
-                    target_type=target_type,
-                    shape=shape,
-                    **init_kwargs,
-                )
-        elif (
-            utils.is_same_class(target, DocumentTypedFeature)
-            or "text" in target_type["type"]
-            or "text" in target
-        ):
-            raise ValueError(
-                f"The column {target} cannot be used as the target column."
-            )
-        elif (
-            utils.is_same_class(target_type, GISTypedFeature)
-            or "coord" in target_type["type"]
-            or "coord" in target
-        ):
-            raise ValueError(
-                f"The column {target} cannot be used as the target column."
-            )
-        # This is to catch constant columns that are boolean. Added as a fix for pd.isnull(), and datasets with a
-        #   binary target, but only data on one instance
-        elif target_type and target_type["low_level_type"] == "bool":
-            return BinaryClassificationDataset(
-                df=df,
-                sampled_df=sampled_df,
-                target=target,
-                shape=shape,
-                target_type=target_type,
-                positive_class=positive_class,
-                **init_kwargs,
-            )
-        raise ValueError(
-            f"Unable to identify problem type. Specify the data type of {target} using 'types'. "
-            f"For example, types = {{{target}: 'category'}}"
-        )
-
-    def rename_columns(self, columns):
-        """
-        Returns a dataset with columns renamed.
-        """
-        if isinstance(columns, list):
-            assert len(columns) == len(
-                self.columns.values
-            ), "columns length do not match the dataset"
-            columns = dict(zip(self.columns.values, columns))
-        assert isinstance(columns, dict)
-        new_target = None
-        if self.target.name in columns:
-            new_target = columns[self.target.name]
-        return self.rename(columns=columns, _new_target=new_target)
-
-    def select_best_features(self, score_func=None, k=12):
-        """
-        Return new dataset containing only the top k features.
-
-        Parameters
-        ----------
-        k: int, default 12
-            The top 'k' features to select.
-        score_func: function
-            Scoring function to use to rank the features. This scoring function should take a 2d array X(features)
-            and an array like y(target) and return a numeric score for each feature in the same order as X.
-
-        Notes
-        -----
-        See also https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_regression.html
-        and https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.f_classif.html
-
-        Examples
-        --------
-        >>> ds = DatasetBrowser("sklearn").open("iris")
-        >>> ds_small = ds.select_best_features(k=2)
-        """
-        tf = self._get_best_features_transformer(score_func=score_func, k=k)
-        return self._build_new_dataset(
-            tf[1].transform(self.df), tf[1].transform(self.sampled_df), transformers=tf
-        )
-
-    def auto_transform(
-        self,
-        correlation_threshold: float = 0.7,
-        frac: float = 1.0,
-        sample_size=1.0,
-        correlation_methods: Union[str, list] = "pearson",
-    ):
-        """
-        Return transformed dataset with several optimizations applied automatically.
-        The optimizations include:
-
-        - Dropping constant and primary key columns, which has no predictive quality,
-        - Imputation, to fill in missing values in noisy data:
-
-            - For continuous variables, fill with mean if less than 40% is missing, else drop,
-            - For categorical variables, fill with most frequent if less than 40% is missing, else drop,
-
-        - Dropping strongly co-correlated columns that tend to produce less generalizable models.
-
-        Parameters
-        ----------
-        correlation_threshold: float, defaults to 0.7. It must be between 0 and 1, inclusive
-            the correlation threshold where columns with correlation higher than the threshold will
-            be considered as strongly co-correlated and recommended to be taken care of.
-        frac: Is superseded by sample_size
-        sample_size: float, defaults to 1.0. Float, Range -> (0, 1]
-            What fraction of the data should be used in the calculation?
-        correlation_methods: Union[list, str], defaults to 'pearson'
-
-            - 'pearson': Use Pearson's Correlation between continuous features,
-            - 'cramers v': Use Cramer's V correlations between categorical features,
-            - 'correlation ratio': Use Correlation Ratio Correlation between categorical and continuous features,
-            - 'all': Is equivalent to ['pearson', 'cramers v', 'correlation ratio'].
-
-            Or a list containing any combination of these methods, for example, ['pearson', 'cramers v'].
-
-        Returns
-        -------
-        transformed_dataset : ADSDatasetWithTarget
-
-        Examples
-        --------
-        >>> ds_clean = ds.auto_transform()
-        """
-        frac = deprecate_default_value(
-            frac,
-            None,
-            1,
-            "<code>frac=None</code> is deprecated. Use <code>sample_size=1.0</code> instead.",
-            FutureWarning,
-        )
-
-        if frac != 1.0:
-            deprecate_frac = deprecate_variable(
-                frac,
-                sample_size,
-                "<code>frac</code> is superseded by <code>sample_size</code>.",
-                DeprecationWarning,
-            )
-            if sample_size == 1.0:
-                sample_size = deprecate_frac
-
-        if correlation_threshold > 1 or correlation_threshold < 0:
-            raise AssertionError("correlation_threshold has to be between 0 and 1.")
-        with utils.get_progress_bar(5) as progress:
-            df, sampled_df, transformer_pipeline = self._transform(
-                progress=progress,
-                correlation_threshold=correlation_threshold,
-                frac=sample_size,
-                correlation_methods=correlation_methods,
-            )
-            return self._build_new_dataset(
-                df,
-                sampled_df=sampled_df,
-                transformers=transformer_pipeline.steps,
-                progress=progress,
-            )
-
-    def visualize_transforms(self):
-        """
-        Render a representation of the dataset's transform DAG.
-        """
-
-        helper.visualize_transformation(
-            self.transformer_pipeline,
-            text="- rows: {}\\l- columns: {}\\l".format(*self.shape),
-        )
-
-    def _suggested_code(self, action, recommendation_type, variable):
-        if action == "Drop":
-            return ".drop_columns([{}])".format('"' + variable + '"')
-        if action == "Do nothing":
-            return ""
-        if "Drop " in action:
-            return ".drop_columns([{}])".format('"' + action.split(" ")[1] + '"')
-        if action == "Down-sample":
-            return ".down_sample()"
-        if action == "Up-sample":
-            if importlib.util.find_spec("imblearn") is None:
-                return ".up_sample(sampler='default') \\n `pip install imbalanced-learn` to use default up-sampler."
-            else:
-                return ".up_sample(sampler='default')"
-        if recommendation_type == "positive_class" and action != "Do nothing":
-            return ".set_positive_class({}, missing_value=False)".format(
-                '"' + action + '"'
-            )
-        if recommendation_type == "imputation":
-            fill_val = helper.get_fill_val(
-                self.feature_types, variable, action, constant="constant"
-            )
-
-            fill_val = (
-                fill_val if isinstance(fill_val, Number) else '"' + fill_val + '"'
-            )
-            return ".fillna({}{}: {}{})".format(
-                "{", '"' + variable + '"', fill_val, "}"
-            )
-        else:
-            return ""
-
-    def suggest_recommendations(
-        self,
-        correlation_methods: Union[str, list] = "pearson",
-        print_code: bool = True,
-        correlation_threshold: float = 0.7,
-        overwrite: bool = None,
-        force_recompute: bool = False,
-        frac: float = 1.0,
-        sample_size: float = 1.0,
-        **kwargs,
-    ):
-        """
-        Returns a pandas dataframe with suggestions for dataset optimization. This includes:
-
-        - Identifying constant and primary key columns, which has no predictive quality,
-        - Imputation, to fill in missing values in noisy data:
-
-            - For continuous variables, fill with mean if less than 40% is missing, else drop,
-            - For categorical variables, fill with most frequent if less than 40% is missing, else drop,
-
-        - Identifying strongly co-correlated columns that tend to produce less generalizable models,
-        - Automatically balancing dataset for classification problems using up or down sampling.
-
-        Parameters
-        ----------
-        correlation_methods: Union[list, str], default to 'pearson'
-
-            - 'pearson': Use Pearson's Correlation between continuous features,
-            - 'cramers v': Use Cramer's V correlations between categorical features,
-            - 'correlation ratio': Use Correlation Ratio Correlation between categorical and continuous features,
-            - 'all': Is equivalent to ['pearson', 'cramers v', 'correlation ratio'].
-
-            Or a list containing any combination of these methods, for example, ['pearson', 'cramers v']
-        print_code: bool, Defaults to True
-            Print Python code for the suggested actions.
-        correlation_threshold: float. Defaults to 0.7. It must be between 0 and 1, inclusive
-            the correlation threshold where columns with correlation higher than the threshold will
-            be considered as strongly co-correated and recommended to be taken care of.
-        frac: Is superseded by sample_size
-        sample_size: float, defaults to 1.0. Float, Range -> (0, 1]
-            What fraction of the data should be used in the calculation?
-        overwrite:
-            Is deprecated and replaced by force_recompute.
-        force_recompute: bool, default to be False
-
-            - If False, it calculates the correlation matrix if there is no cached correlation matrix. Otherwise,
-              it returns the cached correlation matrix.
-            - If True, it calculates the correlation matrix regardless whether there is cached result or not.
-
-        Returns
-        -------
-        suggestion dataframe : pandas.DataFrame
-
-        Examples
-        --------
-        >>> suggestion_df = ds.suggest_recommendations(correlation_threshold=0.7)
-        """
-        frac = deprecate_default_value(
-            frac,
-            None,
-            1,
-            "<code>frac=None</code> is deprecated. Use <code>sample_size=1.0</code>.",
-            FutureWarning,
-        )
-
-        if frac != 1.0:
-            deprecate_frac = deprecate_variable(
-                frac,
-                sample_size,
-                "<code>frac</code> is superseded by <code>sample_size</code>.",
-                DeprecationWarning,
-            )
-            if sample_size == 1.0:
-                sample_size = deprecate_frac
-
-        force_recompute = deprecate_variable(
-            overwrite,
-            force_recompute,
-            f"<code>overwrite=None</code> is deprecated. Use <code>force_recompute</code> instead.",
-            DeprecationWarning,
-        )
-
-        recommended = self._get_recommendations_transformer(
-            auto_transform=False,
-            correlation_threshold=correlation_threshold,
-            correlation_methods=correlation_methods,
-            force_recompute=force_recompute,
-            frac=sample_size,
-            **kwargs,
-        ).fit(self.sampled_df)
-
-        if len(recommended.reco_dict_) == 0:
-            logger.info("No recommendations.")
-            return pd.DataFrame()
-
-        column_names = [
-            "Message",
-            "Variables",
-            "Action",
-            "Selected Action",
-            "Recommendation Type",
-        ]
-
-        df_dict = defaultdict(list)
-
-        for recommendation_type, column_dict in recommended.reco_dict_.items():
-            if recommendation_type == "constant_column":
-                n_constant = len(column_dict)
-                df_dict["Recommendation Type"].extend(
-                    [recommendation_type] * n_constant
-                )
-                df_dict["Variables"].extend(column_dict)
-                df_dict["Message"].extend(["Constant Column"] * n_constant)
-                df_dict["Action"].extend(["Drop"] * n_constant)
-                df_dict["Selected Action"].extend(["Drop"] * n_constant)
-                continue
-
-            for column, details_dict in column_dict.items():
-                max_length = len(details_dict["Action"])
-                for key, value in details_dict.items():
-                    if isinstance(value, list):
-                        df_dict[key].extend(value)
-                    else:
-                        df_dict[key].extend([value] * max_length)
-                df_dict["Recommendation Type"].extend(
-                    [recommendation_type] * max_length
-                )
-                df_dict["Variables"].extend([column] * max_length)
-
-        suggestions_df = pd.DataFrame.from_dict(df_dict)[column_names]
-        suggestions_df["Code"] = suggestions_df.apply(
-            lambda x: self._suggested_code(
-                x["Action"], x["Recommendation Type"], x["Variables"]
-            ),
-            axis=1,
-        )
-        suggestion_df = (
-            suggestions_df.drop(columns=["Recommendation Type"])
-            .rename(columns={"Selected Action": "Suggested"})
-            .set_index(["Message", "Variables", "Suggested", "Action"])
-            .fillna("")
-        )
-        if print_code:
-            columns_to_impute = {}
-            columns_to_drop = []
-            consolidated_code = ""
-            suggestion_df_ = suggestion_df.reset_index()
-            suggested_code = suggestion_df_.loc[
-                suggestion_df_.Suggested == suggestion_df_.Action
-            ].Code.unique()
-            for code in suggested_code:
-                if ".drop_columns" in code:
-                    columns_to_drop.append(code.split("[")[1].split("]")[0][1:-1])
-                elif ".fillna" in code:
-                    impute_pair = code.split("{")[1].split("}")[0]
-                    columns_to_impute[impute_pair.split(":")[0].replace('"', "")] = (
-                        float(impute_pair.split(":")[1].strip())
-                        if impute_pair.split(":")[1].strip().replace(".", "").isdigit()
-                        else impute_pair.split(":")[1].strip().replace('"', "")
-                    )
-                else:
-                    consolidated_code += code
-            consolidated_code = (
-                "No more!" if len(consolidated_code) == 0 else consolidated_code
-            )
-
-            logger.info(f"Suggested columns to drop: {columns_to_drop}.")
-            logger.info(f"Suggested columns to impute: {columns_to_impute}.")
-            logger.info(f"Others: {consolidated_code}.")
-
-        return suggestion_df
-
-    @runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
-    def get_recommendations(
-        self,
-        correlation_methods: str = "pearson",
-        correlation_threshold: float = 0.7,
-        frac: float = 1.0,
-        sample_size: float = 1.0,
-        overwrite: bool = None,
-        force_recompute: bool = False,
-        display_format: str = "widget",
-    ):
-        """
-        Generate recommendations for dataset optimization. This includes:
-
-        - Identifying constant and primary key columns, which has no predictive quality,
-        - Imputation, to fill in missing values in noisy data:
-
-            - For continuous variables, fill with mean if less than 40% is missing, else drop,
-            - For categorical variables, fill with most frequent if less than 40% is missing, else drop,
-
-        - Identifying strongly co-correlated columns that tend to produce less generalizable models,
-        - Automatically balancing dataset for classification problems using up or down sampling.
-
-        Parameters
-        ----------
-        correlation_methods: Union[list, str], default to 'pearson'
-
-            - 'pearson': Use Pearson's Correlation between continuous features,
-            - 'cramers v': Use Cramer's V correlations between categorical features,
-            - 'correlation ratio': Use Correlation Ratio Correlation between categorical and continuous features,
-            - 'all': Is equivalent to ['pearson', 'cramers v', 'correlation ratio'].
-
-            Or a list containing any combination of these methods, for example, ['pearson', 'cramers v'].
-        correlation_threshold: float, defaults to 0.7. It must be between 0 and 1, inclusive
-            The correlation threshold where columns with correlation higher than the threshold will
-            be considered as strongly co-correlated and recommended to be taken care of.
-        frac: Is superseded by sample_size
-        sample_size: float, defaults to 1.0. Float, Range -> (0, 1]
-            What fraction of the data should be used in the calculation?
-        overwrite:
-            Is deprecated and replaced by force_recompute.
-        force_recompute: bool, default to be False
-
-            - If False, it calculates the correlation matrix if there is no cached correlation matrix. Otherwise,
-              it returns the cached correlation matrix.
-            - If True, it calculates the correlation matrix regardless whether there is cached result or not.
-
-        display_format: string, defaults to 'widget'.
-            Should be either 'widget' or 'table'. If 'widget',
-            a GUI style interface is popped out; if 'table', a table of suggestions is shown.
-        """
-        frac = deprecate_default_value(
-            frac,
-            None,
-            1,
-            "<code>frac=None</code> is superseded by <code>sample_size=1.0</code>.",
-            FutureWarning,
-        )
-
-        if frac != 1.0:
-            deprecate_frac = deprecate_variable(
-                frac,
-                sample_size,
-                "<code>frac</code> is superseded by <code>sample_size</code>.",
-                DeprecationWarning,
-            )
-            if sample_size == 1.0:
-                sample_size = deprecate_frac
-
-        force_recompute = deprecate_variable(
-            overwrite,
-            force_recompute,
-            f"<code>overwrite=None</code> is deprecated. Use <code>force_recompute</code> instead.",
-            DeprecationWarning,
-        )
-
-        if display_format == "widget":
-            recommended = Recommendation(
-                self,
-                self._get_recommendations_transformer(
-                    auto_transform=False,
-                    correlation_threshold=correlation_threshold,
-                    correlation_methods=correlation_methods,
-                    frac=sample_size,
-                    force_recompute=force_recompute,
-                ).fit(self.sampled_df),
-            )
-
-            if len(recommended.reco_dict) == 0:
-                logger.info("No recommendations.")
-
-            return recommended
-
-        elif display_format == "table":
-            df_suggestion = self.suggest_recommendations(
-                correlation_threshold=correlation_threshold,
-                frac=sample_size,
-                force_recompute=force_recompute,
-            )
-
-            from IPython.display import HTML, display
-
-            display(
-                HTML(
-                    df_suggestion.to_html()
-                    .replace(" `", "<code>")
-                    .replace("` ", "</code>")
-                    .replace("\\n", "<br>")
-                )
-            )
-
-    def get_transformed_dataset(self):
-        """
-        Return the transformed dataset with the recommendations applied.
-
-        This method should be called after applying the recommendations using the Recommendation#show_in_notebook() API.
-        """
-        if hasattr(self, "new_ds"):
-            return self.new_ds
-        logger.info(
-            "Use `get_recommendations()` to view or update recommendation or `auto_tranform()` first."
-        )
-        logger.warning(
-            "`get_transformed_dataset` is deprecated and will be removed in a future release."
-        )
-        return
-
-    def type_of_target(self):
-        """
-        Return the target type for the dataset.
-
-        Returns
-        -------
-        target_type: TypedFeature
-            an object of TypedFeature
-
-        Examples
-        --------
-        >>> ds = ds.set_target('target_class')
-        >>> assert(ds.type_of_target() == 'categorical')
-        """
-        return self.target.type
-
-    def train_test_split(self, test_size=0.1, random_state=utils.random_state):
-        """
-        Splits  dataset to train and test data.
-
-        Parameters
-        ----------
-        test_size: Union[float, int], optional, default=0.1
-        random_state: Union[int, RandomState], optional, default=None
-
-                - If int, random_state is the seed used by the random number generator;
-                - If RandomState instance, random_state is the random number generator;
-                - If None, the random number generator is the RandomState instance used by np.random.
-
-        Returns
-        -------
-        train_data, test_data: tuple
-            tuple of ADSData instances
-
-        Examples
-        --------
-        >>> ds = DatasetFactory.open("data.csv")
-        >>> train, test = ds.train_test_split()
-        """
-        X_train, X_test, y_train, y_test = train_test_split(
-            self.df.drop(self.target.name, axis=1),
-            self.df[self.target.name],
-            test_size=test_size,
-            train_size=1 - test_size,
-            random_state=random_state,
-        )
-        train = ADSData.build(
-            X=X_train, y=y_train, name="Train Data", dataset_type=self.__class__
-        )
-        train.transformer_pipeline = self.transformer_pipeline
-        test = ADSData.build(
-            X=X_test, y=y_test, name="Test Data", dataset_type=self.__class__
-        )
-        return train, test
-
-    def train_validation_test_split(
-        self, test_size=0.1, validation_size=0.1, random_state=utils.random_state
-    ):
-        """
-        Splits  dataset to train, validation and test data.
-
-        Parameters
-        ----------
-        test_size: Union[float, int], optional, default=0.1
-        validation_size: Union[float, int], optional, default=0.1
-        random_state: Union[int, RandomState], optional, default=None
-
-                - If int, random_state is the seed used by the random number generator;
-                - If RandomState instance, random_state is the random number generator;
-                - If None, the random number generator is the RandomState instance used by np.random.
-
-        Returns
-        -------
-        train_data, validation_data, test_data: tuple
-            tuple of ADSData instances
-
-        Examples
-        --------
-        >>> ds = DatasetFactory.open("data.csv")
-        >>> train, valid, test = ds.train_validation_test_split()
-        """
-        train, test = self.train_test_split(
-            test_size=test_size, random_state=random_state
-        )
-        X_train, X_valid, y_train, y_valid = train_test_split(
-            train.X, train.y, test_size=validation_size, random_state=random_state
-        )
-        train.X = X_train
-        train.y = y_train
-        valid = ADSData.build(
-            X=X_valid, y=y_valid, name="Validation Data", dataset_type=self.__class__
-        )
-        return train, valid, test
-
-    """
-    Internal methods
-    """
-
-    def __repr__(self):
-        rows, cols = self.shape
-        return f"{self.__class__.__name__}(target: {self.target.name}) {rows:,} rows, {cols:,} columns"
-
-    def _transform(
-        self,
-        progress=DummyProgressBar(),
-        fix_imbalance=True,
-        correlation_threshold=0.7,
-        frac=None,
-        correlation_methods="pearson",
-    ):
-        progress.update("Building the transformer pipeline")
-        if self.recommendation_transformer is None:
-            transformer_pipeline = TransformerPipeline(
-                steps=[
-                    (
-                        "recommendations",
-                        self._get_recommendations_transformer(
-                            fix_imbalance=fix_imbalance,
-                            correlation_threshold=correlation_threshold,
-                            frac=frac,
-                            correlation_methods=correlation_methods,
-                        ),
-                    ),
-                    (
-                        "feature_engineering",
-                        FeatureEngineeringTransformer(
-                            feature_metadata=self.feature_types
-                        ),
-                    ),
-                ]
-            )
-        else:
-            # recommendations are already generated using get_recommendations().show_in_notebook() API
-            transformer_pipeline = TransformerPipeline(
-                steps=[
-                    (
-                        "feature_engineering",
-                        FeatureEngineeringTransformer(
-                            feature_metadata=self.feature_types
-                        ),
-                    )
-                ]
-            )
-            transformer_pipeline.steps = [
-                ("recommendations", self.recommendation_transformer)
-            ] + transformer_pipeline.steps
-        sampled_df = self.sampled_df.copy()
-        self.recommendation_transformer = None
-        df = self.df.copy()
-        for step in transformer_pipeline.steps:
-            progress.update("Applying transformation for %s" % step[0])
-            sampled_df = step[1].fit_transform(sampled_df)
-            df = step[1].transform(df, progress=progress, fit_transform=True)
-        return df, sampled_df, transformer_pipeline
-
-    def _get_best_features(self, score_func=None, k=12):
-        if isinstance(self.target.type, DateTimeTypedFeature):
-            return FeatureImportance._get_feature_ranking(
-                self.sampled_df.copy(),
-                self.target.name,
-                self.type_of_target(),
-                score_func=score_func,
-                k=k,
-            )
-        else:
-            return FeatureImportance._get_feature_ranking(
-                self.sampled_df.copy(),
-                self.target.name,
-                self.type_of_target(),
-                score_func=score_func,
-                k=k,
-            )
-
-    def _get_best_features_transformer(self, score_func=None, k=12):
-        feature_set = self._get_best_features(k=k, score_func=score_func)[
-            "features"
-        ].tolist()
-
-        def _select_features(df, feature_names, target):
-            if target in df.columns:
-                feature_names = feature_names + [target]
-            return df[feature_names]
-
-        return (
-            "select_{0}_best_features".format(k),
-            FunctionTransformer(
-                func=_select_features,
-                validate=False,
-                kw_args={"feature_names": feature_set, "target": self.target.name},
-            ).fit(self.sampled_df),
-        )
-
-    def _get_recommendations_transformer(
-        self,
-        fix_imbalance=True,
-        auto_transform=True,
-        correlation_threshold=0.7,
-        **kwargs,
-    ):
-        force_recompute = kwargs.pop("force_recompute", False)
-        frac = kwargs.pop("frac", 1)
-        correlation_methods = kwargs.pop("correlation_methods", "pearson")
-        return RecommendationTransformer(
-            feature_metadata=self.feature_types,
-            correlation=self.corr(
-                force_recompute=force_recompute,
-                frac=frac,
-                correlation_methods=correlation_methods,
-                **kwargs,
-            ),
-            target=self.target.name,
-            target_type=self.target.type,
-            is_balanced=self.target.is_balanced(),
-            feature_ranking=self._get_best_features(k=len(self.sampled_df)),
-            fix_imbalance=fix_imbalance,
-            len=self.__len__(),
-            auto_transform=auto_transform,
-            correlation_threshold=correlation_threshold,
-        )