oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/dataset/helper.py

@@ -0,0 +1,1450 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import ast
+import base64
+import html
+import io
+import math
+import os
+import warnings
+import re
+from collections import defaultdict
+import inspect
+import importlib
+from typing import Callable, List, Tuple, Union
+import fsspec
+
+# from pandas.io.common import _compression_to_extension
+
+from numbers import Number
+from urllib.parse import urlparse
+
+import numpy as np
+import pandas as pd
+
+from pandas.core.dtypes.common import (
+    is_numeric_dtype,
+    is_bool_dtype,
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_float_dtype,
+)
+
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.common import utils
+from ads.dataset import logger
+from ads.type_discovery.type_discovery_driver import TypeDiscoveryDriver
+from ads.type_discovery.typed_feature import (
+    ContinuousTypedFeature,
+    DateTimeTypedFeature,
+    CategoricalTypedFeature,
+    GISTypedFeature,
+    TypedFeature,
+    UnknownTypedFeature,
+    OrdinalTypedFeature,
+    DocumentTypedFeature,
+)
+
+
+class DatasetDefaults:
+    sampling_confidence_level = 95
+    sampling_confidence_interval = 1.0
+
+
+_known_db_protocols = {"sqlite", "ADB", "oracle+cx_oracle"}
+
+
+def concatenate(X, y):
+    if isinstance(X, pd.DataFrame) or isinstance(X, pd.Series):
+        return pd.concat([X, y], axis=1)
+    else:
+        return X.assign(**{y.name: y})
+
+
+def fix_column_names(X):
+    X.columns = X.columns.astype("str").str.strip().str.replace(" ", "_")
+    return X
+
+
+def convert_columns(df, feature_metadata=None, dtypes=None):
+    if feature_metadata is not None:
+        dtypes = {}
+        for feature in feature_metadata:
+            dtype = get_dtype(feature_metadata[feature], df[feature].dtype)
+            if dtype is not None:
+                dtypes[feature] = dtype
+    return df.astype(dtypes)
+
+
+def get_dtype(feature_type, dtype):
+    if isinstance(feature_type, ContinuousTypedFeature) or isinstance(
+        feature_type, OrdinalTypedFeature
+    ):
+        return dtype.name if is_numeric_dtype(dtype) else "float"
+    elif isinstance(feature_type, DateTimeTypedFeature):
+        return "datetime64[ns]" if not dtype.name.startswith("datetime") else dtype
+    elif isinstance(feature_type, CategoricalTypedFeature):
+        return "bool" if is_bool_dtype(dtype) else "category"
+
+
+def get_feature_type(name, series):
+    if is_bool_dtype(series) or is_categorical_dtype(series):
+        return CategoricalTypedFeature.build(name, series)
+    elif is_numeric_dtype(series):
+        if is_float_dtype(series):
+            return ContinuousTypedFeature.build(name, series)
+        else:
+            return OrdinalTypedFeature.build(name, series)
+    elif is_datetime64_any_dtype(series):
+        return DateTimeTypedFeature.build(name, series)
+    else:
+        return UnknownTypedFeature.build(name, series)
+
+
+def convert_to_html(plot):
+    img = io.BytesIO()
+    plot.savefig(img, format="png", bbox_inches="tight")
+    img.seek(0)
+    encoded = base64.b64encode(img.getvalue())
+    return '<img width=95%" src="data:image/png;base64, {}"><hr><br>'.format(
+        encoded.decode("utf-8")
+    )
+
+
+def _num_partitions_for_dataframe(df):
+    # takes pandas dataframe, guesses good number of partitions
+    return utils.get_cpu_count() if df.shape[0] > 1000 * utils.get_cpu_count() else 1
+
+
+class ElaboratedPath:
+    """
+    The Elaborated Path class unifies all of the operations and information related to a path or pathlist.
+    Whether the user wants to
+    An Elaborated path can accept any of the following as a valid source:
+    * A single path
+    * A glob pattern path
+    * A directory
+    * A list of paths (Note: all of these paths must be from the same filesystem AND have the same format)
+    * A sqlalchemy connection url
+    """
+
+    def __init__(
+        self,
+        source: Union[str, List[str]],
+        format: str = None,
+        name: str = None,
+        **kwargs,
+    ):
+        """
+        :param source:
+        :param format:
+        :param kwargs:
+
+        By the end of this method, this class needs to have paths, format, and name ready
+        """
+        self._kwargs = kwargs
+        self._format = format
+        self._name = name
+        if isinstance(source, str):
+            self._original_source = source
+            self._determine_protocol_type()
+            if self._type == "db":
+                self._paths = [self._original_source]
+            else:
+                self._elaborate_path()
+        elif isinstance(source, list) and all(isinstance(file, str) for file in source):
+            assert len(source) > 0, "Error, the source you passed in was an empty list."
+            self._original_source = source[0]
+            self._paths = source
+            self._type = "list"
+        else:
+            raise ValueError(f"Source argument not understood: {source}")
+        if self.num_paths == 0:
+            raise FileNotFoundError(
+                f"Error: We could not find any files associated with the source: "
+                f"{source}. Double check that this source is a valid glob pattern,"
+                f" directory, or path."
+            )
+        self._determine_format()
+        self._determine_name()
+
+    @property
+    def paths(self) -> List[str]:
+        """
+        :return: a list of str
+            Each element will be a valid path
+        """
+        return self._paths
+
+    @property
+    def num_paths(self) -> int:
+        """
+        This method will return the number of paths found with the associated original glob, folder, or path.
+        If this returns 0,
+        :return:
+        """
+        return len(self._paths)
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def format(self) -> str:
+        return self._format
+
+    def _determine_name(self):
+        if self._name is None:
+            if self._type == "list":
+                self._name = (
+                    f"DataFrame from [{os.path.basename(self._original_source)}, ...]"
+                )
+            elif self._type == "glob":
+                self._name = f"DataFrame from {os.path.basename(self._original_source)}"
+            else:
+                self._name = f"DataFrame from {urlparse(self._original_source).scheme}"
+
+    def _determine_format(self):
+        """
+        Infer format from the path.
+
+        If its  a compressed file, returns the extension before compression extension.
+        If the extension cannot be inferred,  returns None
+
+        Parameters
+        ----------
+        path : ElaboratedPath
+            an ElaboratedPath object
+
+        Returns
+        -------
+        format : str
+        """
+        if self._format in [None, "infer"]:
+            format_keys = []
+            for i in range(min(self.num_paths, 5)):
+                format_keys.append(self._remove_compressions(self.paths[i]))
+            if len(format_keys) == 0:
+                raise ValueError(
+                    f"Could not determine the format key for source: {self._original_source}"
+                )
+            if format_keys.count(format_keys[0]) != len(format_keys):
+                raise ValueError(
+                    f"Got multiple formats from the source: {self._original_source}. Run again "
+                    f'using the format parameter. Ex: format=<your format key, like: "csv", "hdf", etc.>'
+                )
+            self._format = format_keys[0]
+        else:
+            self._format = self._format.lower()
+
+    def _elaborate_path(self):
+        self._paths = self._fs.glob(self._original_source)
+        if self._protocol != "":
+            self._paths = [f"{self._protocol}://{p}" for p in self._paths]
+
+    def _determine_protocol_type(self):
+        self._protocol = urlparse(self._original_source).scheme
+
+        if self._kwargs.get("fs") is not None:
+            self._fs = self._kwargs.pop("fs")
+            self._type = "glob"
+        elif self._original_source.startswith("oracle+cx_oracle://"):
+            self._protocol = "oracle+cx_oracle"
+            self._type = "db"
+        else:
+            try:
+                self._fs = fsspec.filesystem(
+                    self._protocol, **self._kwargs.get("storage_options", dict())
+                )
+                self._type = "glob"
+            except ValueError:
+                try:
+                    self.engine = utils.get_sqlalchemy_engine(
+                        self._original_source, **self._kwargs
+                    )
+                    self._type = "db"
+                except:
+                    if self._protocol in _known_db_protocols:
+                        self._type = "db"
+                    else:
+                        raise ValueError(
+                            f"Error in trying to understand the protocol for source: "
+                            f"{self._original_source}. The protocol found: {self._protocol} is not "
+                            f"registered with fsspec or sqlalchemy"
+                        )
+
+    @staticmethod
+    def _remove_compressions(filename: str):
+        _compression_to_extension = [
+            ".gz",
+            ".bz2",
+            ".zip",
+            ".xz",
+            ".zst",
+            ".tar",
+            ".tar.gz",
+            ".tar.xz",
+            ".tar.bz2",
+        ]
+        for compression in _compression_to_extension:
+            if filename.strip().endswith(compression):
+                return ElaboratedPath._remove_compressions(
+                    os.path.splitext(filename.rstrip("/*"))[0]
+                )
+        format = os.path.splitext(filename.rstrip("/*"))[1][1:].lower()
+        return format.lower() if format != "" else None
+
+
+class DatasetLoadException(BaseException):
+    def __init__(self, exc_msg):
+        self.exc_msg = exc_msg
+
+    def __str__(self):
+        return self.exc_msg
+
+
+def _get_dtype_from_error(e):
+    error_string = str(e)
+
+    if "mismatched dtypes" in error_string.lower():
+        # For the mismatched dtypes error, dask either returns a error message containing the dtype argument
+        # to specify, or  the found and expected dtypes in a table format, depending on what stage
+        # the type inferencing fails. The below logic supports building the dtype dictionary for both cases
+        found_dtype_dict_str_list = re.findall(
+            r"dtype=({[^{}]+})", error_string, re.MULTILINE
+        )
+        if found_dtype_dict_str_list:
+            found_dtype_dict = ast.literal_eval(found_dtype_dict_str_list[0])
+        else:
+            found_dtype_dict = _find_dtypes_from_table(error_string)
+        if found_dtype_dict:
+            logger.warning(
+                "Dask type-inference/coercion failed. Retrying with "
+                f"dtype={found_dtype_dict}.",
+                exc_info=True,
+            )
+            return found_dtype_dict
+    return None
+
+
+def _find_dtypes_from_table(error_string):
+    error_lines = error_string.splitlines()
+    dtypes = {}
+    # matches '| Column | Found | Expected |'
+    pattern = re.compile(
+        "\\s*\\|\\s*Column\\s*\\|\\s*Found\\s*\\|\\s*Expected\\s*\\|\\s*"
+    )
+    for i, line in enumerate(error_lines):
+        if re.match(pattern, line):
+            for j in range(i + 2, len(error_lines)):
+                # extracts column_name and found_dtype from  '| <column_name> | <found_dtype> | <expected_dtype |'
+                dtype_suggestion = re.compile("\\s*\\|([^\\|]+)\\|([^\\|]+)\\|.*")
+                match_groups = re.match(dtype_suggestion, error_lines[j])
+                if match_groups is None:
+                    break
+                dtypes[match_groups.group(1).strip()] = match_groups.group(2).strip()
+    return dtypes
+
+
+def rename_duplicate_cols(original_cols):
+    seen_col_names = defaultdict(int)
+    new_cols = []
+    for col in original_cols:
+        # remove any spaces form column names
+        if isinstance(col, str):
+            col.replace(" ", "_")
+        if col not in seen_col_names:
+            new_cols.append(col)
+        else:
+            dup_count = seen_col_names[col]
+            new_cols.append(f"{col}.{dup_count}")
+        seen_col_names[col] += 1
+    assert len(new_cols) == len(
+        original_cols
+    ), "There has been an error in re-naming duplicate columns"
+    return new_cols
+
+
+def write_parquet(
+    path,
+    data,
+    engine="fastparquet",
+    metadata_dict=None,
+    compression=None,
+    storage_options=None,
+):
+    """
+    Uses fast parquet to write dask dataframe and custom metadata in parquet format
+
+    Parameters
+    ----------
+    path : str
+        Path to write to
+    data : pandas.DataFrame
+    engine : string
+        "auto" by default
+    metadata_dict : Deprecated, will not pass through
+    compression : {{'snappy', 'gzip', 'brotli', None}}, default 'snappy'
+        Name of the compression to use
+    storage_options : dict, optional
+        storage arguments required to read the path
+
+    Returns
+    -------
+    str : the file path the parquet was written to
+    """
+    assert isinstance(data, pd.DataFrame)
+    if metadata_dict is not None:
+        warnings.warn(
+            "The `metadata_dict` argument is deprecated and has no effect on this method.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    data.to_parquet(
+        path,
+        engine=engine,
+        compression=compression,
+        storage_options=storage_options,
+    )
+    return path
+
+
+def is_text_data(df, target=None):
+    if len(df.columns.values) == 2:
+        feature_name = (
+            list(set(df.columns.values) ^ set([target]))[0]
+            if target
+            else list(set(df.columns.values))[0]
+        )
+    elif len(df.columns.values == 1):
+        feature_name = df.columns.values[0]
+    else:
+        return False
+    return isinstance(
+        TypeDiscoveryDriver().discover(feature_name, df[feature_name]),
+        DocumentTypedFeature,
+    )
+
+
+def generate_sample(
+    df: pd.DataFrame,
+    n: int,
+    confidence_level: int = DatasetDefaults.sampling_confidence_level,
+    confidence_interval: float = DatasetDefaults.sampling_confidence_interval,
+    **kwargs,
+):
+    min_size_to_sample = min(n, 10000)
+
+    sample_size = None
+
+    if "sample_max_rows" in kwargs:
+        requested_sample_size = int(kwargs["sample_max_rows"])
+
+        if requested_sample_size < 0:
+            sample_size = calculate_sample_size(
+                n, min_size_to_sample, confidence_level, confidence_interval
+            )
+        else:
+            if min_size_to_sample < requested_sample_size < n:
+                logger.info(
+                    f"Downsampling from {n} rows, to the user specified {requested_sample_size} rows for graphing."
+                )
+                sample_size = requested_sample_size
+            elif requested_sample_size >= n:
+                logger.info(f"Using the entire dataset of {n} rows for graphing.")
+                sample_size = n
+            else:
+                sample_size = min_size_to_sample
+                logger.info(
+                    f"Downsampling from {n} rows, to {sample_size} rows for graphing."
+                )
+
+    if sample_size and len(df) > sample_size:
+        frac = min(1.0, sample_size * 1.05 / n)
+        df = df.sample(frac=frac, random_state=42)
+        return df.head(sample_size) if len(df) > sample_size else df
+    else:
+        return df
+
+
+def calculate_sample_size(
+    population_size, min_size_to_sample, confidence_level=95, confidence_interval=1.0
+):
+    """Find sample size for a population using Cochran’s Sample Size Formula.
+     With default values for confidence_level (percentage, default: 95%)
+     and confidence_interval (margin of error, percentage, default: 1%)
+
+    SUPPORTED CONFIDENCE LEVELS: 50%, 68%, 90%, 95%, and 99% *ONLY* - this
+    is because the Z-score is table based, and I'm only providing Z
+    for common confidence levels.
+    """
+
+    if population_size < min_size_to_sample:
+        return None
+
+    confidence_level_constant = {
+        50: 0.67,
+        68: 0.99,
+        90: 1.64,
+        95: 1.96,
+        99: 2.57,
+        99.5: 2.807,
+        99.9: 3.291,
+    }
+
+    p = 0.5
+    e = confidence_interval / 100.0
+    N = population_size
+    n_0 = 0.0
+    n = 0.0
+
+    Z = confidence_level_constant.get(confidence_level, 99)
+
+    n_0 = ((Z**2) * p * (1 - p)) / (e**2)
+    n = n_0 / (1 + ((n_0 - 1) / float(N)))
+
+    sample_size = max(int(math.ceil(n)), min_size_to_sample)
+
+    logger.info(f"Downsampling from {population_size} rows to {sample_size} rows.")
+
+    return sample_size
+
+
+def map_types(types):
+    for column in types:
+        if types[column] == "continuous":
+            types[column] = "float64"
+        elif types[column] == "ordinal":
+            types[column] = "int64"
+        elif types[column] == "categorical":
+            types[column] = "category"
+        elif types[column] == "datetime":
+            types[column] = "datetime64[ns]"
+    return types
+
+
+@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
+@runtime_dependency(module="graphviz", install_from=OptionalDependency.VIZ)
+def visualize_transformation(transformer_pipeline, text=None):
+    dot = graphviz.Digraph()
+
+    # show a single node for paritions
+    dot.attr(
+        "node",
+        shape="tab",
+        style="filled",
+        fontname="courier",
+        fontsize="12",
+        fontcolor="white",
+        resolution="144",
+    )
+    if text:
+        dot.node("partitions", text, margin="0.25", fillcolor="dimgray")
+
+    dot.attr(
+        "node",
+        shape="component",
+        style="filled",
+        fontname="courier",
+        fontsize="10",
+        fontcolor="black",
+        resolution="144",
+    )
+    for step in transformer_pipeline.steps:
+        name, clazz, clazzname, is_ads = (
+            step[0],
+            step[1],
+            step[1].__class__.__name__,
+            "ads" in str(step[1].__class__),
+        )
+        ads_node = str(step[1].__class__.__name__) in [
+            "AutoMLPreprocessingTransformer",
+            "DataFrameTransformer",
+            "RecommendationTransformer",
+            "AutoMLFeatureSelection",
+            "FeatureEngineeringTransformer",
+        ]
+        if ads_node:
+            text2html = "< {} >".format(
+                html.escape(step[1].__repr__()).replace("\n", "<br/>")
+            )
+            dot.node(name, text2html, margin="0.25", fillcolor="gold2")
+        else:
+            dot.node(name, name.rsplit("/")[0], fillcolor="azure")
+
+    def format_label(stage):
+        if "FunctionTransformer" in str(transformer_pipeline.steps[stage][1].__class__):
+            return "<<font face='courier' point-size='10'>&nbsp;<b>{}</b>&nbsp;</font>>".format(
+                html.escape(transformer_pipeline.steps[stage][1].func.__name__)
+            )
+        else:
+            is_ads = "ads" in str(transformer_pipeline.steps[stage][1].__class__)
+            return "<<font face='courier' point-size='10'>&nbsp;<b>{}</b>&nbsp;</font>>".format(
+                transformer_pipeline.steps[stage][1].__class__.__name__
+            )
+
+    edges = [x[0] for x in transformer_pipeline.steps]
+    for i, edge in enumerate(list(zip(edges[:-1], edges[1:]))):
+        dot.edge(*edge, len="1.00", label=format_label(i))
+
+    # terminus node
+    dot.node("terminus", "", shape="terminator", fillcolor="white")
+    dot.edge(edges[-1], "terminus", len="1.00", label=format_label(len(edges) - 1))
+
+    graph = graphviz.Source(dot)
+
+    from IPython.core.display import display, SVG
+
+    display(SVG(graph.pipe(format="svg")))
+
+
+def up_sample(df, target, sampler="default", feature_types=None):
+    """
+    Fixes imbalanced dataset by up-sampling
+
+    Parameters
+    ----------
+    df : Union[pandas.DataFrame, dask.dataframe.core.DataFrame]
+    target : name of the target column in df
+    sampler: Should implement fit_resample(X,y) method
+    fillna: a dictionary contains the column name as well as the fill value,
+            only needed when the column has missing values
+
+    Returns
+    -------
+    upsampled_df : Union[pandas.DataFrame, dask.dataframe.core.DataFrame]
+    """
+    if sampler != "default":
+        if inspect.getattr_static(sampler, "fit_resample", None) is None:
+            raise AttributeError("`sampler` object must has method `fit_resample`.")
+        else:
+            # exactly two input args X, y will be passed to fit_resample()
+            # check signature of fit_sample
+            num_no_default_params = 0
+            sig = inspect.signature(sampler.fit_resample)
+            for param in sig.parameters.values():
+                if param.default is param.empty:
+                    num_no_default_params += 1
+            if len(sig.parameters) < 2 or num_no_default_params > 2:
+                raise RuntimeError(
+                    "The signature for `sampler.fit_resample` has to be `fit_resample(X, y)`."
+                )
+
+    X = df.drop(target, axis=1)
+    y = df[target]
+
+    feature_types = feature_types if feature_types is not None else {}
+
+    columns_with_nans = X.columns.values[X.isna().any()]
+    if len(columns_with_nans) > 0:
+        fill_nan_dict = {}
+        for column in columns_with_nans:
+            if column in feature_types and "mode" in feature_types[column]["stats"]:
+                fill_nan_dict[column] = feature_types[column]["stats"]["mode"]
+            elif column in feature_types and "mean" in feature_types[column]["stats"]:
+                fill_nan_dict[column] = feature_types[column]["stats"]["mean"]
+            elif column in feature_types and "median" in feature_types[column]["stats"]:
+                fill_nan_dict[column] = feature_types[column]["stats"]["median"]
+            else:
+                logger.warning(
+                    "Sampling from a column that has missing values may cause an error."
+                )
+        X = X.fillna(fill_nan_dict)
+
+    if sampler == "default":
+        imblearn_found = importlib.util.find_spec("imblearn") is not None
+        if not imblearn_found:
+            raise ModuleNotFoundError(
+                """
+                Required package for up-sampling `imblearn` not found.
+                Install `imblearn` with `pip install imbalanced-learn`
+                and rerun to enable up-sampling.
+            """
+            )
+        else:
+            sampler = _get_imblearn_sampler(X, y)
+    return _sample(sampler, X, y)
+
+
+def _get_imblearn_sampler(X, y):
+    from imblearn.over_sampling import SMOTE, RandomOverSampler
+
+    categorical_feature_indices = [
+        X.columns.get_loc(c)
+        for c in X.select_dtypes(
+            include=["category", "object", "datetime64"]
+        ).columns.values
+    ]
+
+    if len(categorical_feature_indices) > 0:
+        logger.info(
+            """
+            Using the default `RandomOverSampler` sampler. Use `sample` to specify a sampler.
+            Classes will be equalized.
+            You can also pass in other samplers such as `imblearn.SMOTENC` instead, e.g.
+            sampler = SMOTENC(categorical_features=categorical_feature_indices)
+            ds.up_sample(sampler=sampler)
+        """
+        )
+        return RandomOverSampler(random_state=42)
+
+    min_sample_size = y.value_counts().min()
+
+    k_neighbors = min(min_sample_size - 1, 5)
+    if k_neighbors == 0:
+        logger.warning(
+            f"""k_neighbors is 0 as in the target there exists a class label that appeared only once.
+                SMOTE will fail. Default to RandomOverSampler.
+            """
+        )
+        return RandomOverSampler(random_state=42)
+    else:
+        if 5 > k_neighbors > 0:
+            logger.info(
+                f"`k_neighbors()` of SMOTE has changed to {k_neighbors}"
+                " as the target has at least one class which appeared "
+                f"only {min_sample_size} times in the data. "
+            )
+        logger.info("Using SMOTE for over sampling. Classes will be equalized.")
+        return SMOTE(random_state=42, k_neighbors=k_neighbors)
+
+
+def down_sample(df, target):
+    """
+    Fixes imbalanced dataset by down-sampling
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+    target : name of the target column in df
+
+    Returns
+    -------
+    downsampled_df : pandas.DataFrame
+    """
+    dfs = []
+    target_value_counts = df[target].value_counts()
+    min_key = min(target_value_counts.items(), key=lambda k: k[1])
+    for key, value in target_value_counts.items():
+        if key != min_key[0]:
+            dfs.append(
+                df[df[target] == key].sample(frac=1 - ((value - min_key[1]) / value))
+            )
+    dfs.append(df[df[target] == min_key[0]])
+    return pd.concat(dfs)
+
+
+def _sample(sampler, X, y):
+    if isinstance(y, pd.Series) and (
+        isinstance(y[0], bool) or isinstance(y[0], np.bool_)
+    ):
+        y_trans = y.astype(int)  ## Convert to ints to let SMOTE sample properly
+        X_resampled, y_resampled = sampler.fit_resample(X=X, y=y_trans)
+    else:
+        X_resampled, y_resampled = sampler.fit_resample(X=X, y=y)
+
+    if not isinstance(X_resampled, pd.DataFrame):
+        X_resampled = pd.DataFrame(X_resampled, columns=X.columns.values)
+    if not isinstance(y_resampled, pd.Series):
+        y_resampled = pd.DataFrame(y_resampled, columns=[y.name])[y.name]
+
+    for k in X.dtypes.keys():
+        X_resampled[k] = X_resampled[k].astype(X.dtypes[k].name)
+    balanced_df = concatenate(X_resampled, y_resampled)
+    return balanced_df
+
+
+def get_fill_val(feature_types, column, action, constant="constant"):
+    # action can be one of the following
+    # "Fill missing values with mean", "Fill missing values with median",
+    # "Fill missing values with frequent", "Fill missing values with constant"
+    action_ = action.split(" ")[-1]
+    fill_type = "mode" if action_ == "frequent" else action_
+    try:
+        fill_val = (
+            feature_types[column].meta_data["stats"][fill_type]
+            if action_ != "constant"
+            else constant
+        )
+        fill_val = round(fill_val, 4) if isinstance(fill_val, Number) else fill_val
+    except:
+        fill_val = None
+    return fill_val
+
+
+def parse_apache_log_str(x):
+    """
+    Returns the string delimited by two characters.
+
+    Source: https://mmas.github.io/read-apache-access-log-pandas
+    Example:
+        `>>> parse_str('[my string]')`
+        `'my string'`
+    """
+    if x is not None:
+        return x[1:-1]
+    return np.nan
+
+
+def parse_apache_log_datetime(x):
+    """
+    Parses datetime with timezone formatted as:
+        `[day/month/year:hour:minute:second zone]`
+
+    Source: https://mmas.github.io/read-apache-access-log-pandas
+    Example:
+        `>>> parse_datetime('13/Nov/2015:11:45:42 +0000')`
+        `datetime.datetime(2015, 11, 3, 11, 45, 4, tzinfo=<UTC>)`
+
+    Due to problems parsing the timezone (`%z`) with `datetime.strptime`, the
+    timezone will be obtained using the `pytz` library.
+    """
+    import pytz
+    from datetime import datetime
+
+    dt = datetime.strptime(x[1:-7], "%d/%b/%Y:%H:%M:%S")
+    dt_tz = int(x[-6:-3]) * 60 + int(x[-3:-1])
+    return dt.replace(tzinfo=pytz.FixedOffset(dt_tz))
+
+
+def deprecate_variable(old_var, new_var, warning_msg, warning_type):
+    if old_var is not None:
+        warnings.warn(warning_msg, warning_type)
+        return old_var
+    return new_var
+
+
+def deprecate_default_value(var, old_value, new_value, warning_msg, warning_type):
+    if var == old_value:
+        warnings.warn(warning_msg, warning_type)
+        return new_value
+    else:
+        return var
+
+
+def _log_yscale_not_set():
+    logger.info(
+        "`yscale` parameter is not set. Valid values are `'linear'`, `'log'`, `'symlog'`."
+    )
+
+
+def infer_target_type(target, target_series, discover_target_type=True):
+    # if type discovery is turned off, infer type from pandas dtype
+    if discover_target_type:
+        target_type = TypeDiscoveryDriver().discover(
+            target, target_series, is_target=True
+        )
+    else:
+        target_type = get_feature_type(target, target_series)
+    return target_type
+
+
+def get_target_type(target, sampled_df, **init_kwargs):
+    discover_target_type = init_kwargs.get("type_discovery", True)
+    if target in init_kwargs.get("types", {}):
+        sampled_df[target] = sampled_df[target].astype(init_kwargs.get("types")[target])
+        discover_target_type = False
+    return infer_target_type(target, sampled_df[target], discover_target_type)
+
+
+def get_dataset(
+    df: pd.DataFrame,
+    sampled_df: pd.DataFrame,
+    target: str,
+    target_type: TypedFeature,
+    shape: Tuple[int, int],
+    positive_class=None,
+    **init_kwargs,
+):
+    from ads.dataset.classification_dataset import (
+        BinaryClassificationDataset,
+        BinaryTextClassificationDataset,
+        MultiClassClassificationDataset,
+        MultiClassTextClassificationDataset,
+    )
+    from ads.dataset.forecasting_dataset import ForecastingDataset
+    from ads.dataset.regression_dataset import RegressionDataset
+
+    if len(df[target].dropna()) == 0:
+        logger.warning(
+            "It is not recommended to use an empty column as the target variable."
+        )
+        raise ValueError(f"We do not support using empty columns as the chosen target")
+    if utils.is_same_class(target_type, ContinuousTypedFeature):
+        return RegressionDataset(
+            df=df,
+            sampled_df=sampled_df,
+            target=target,
+            target_type=target_type,
+            shape=shape,
+            **init_kwargs,
+        )
+    elif utils.is_same_class(
+        target_type, DateTimeTypedFeature
+    ) or df.index.dtype.name.startswith("datetime"):
+        return ForecastingDataset(
+            df=df,
+            sampled_df=sampled_df,
+            target=target,
+            target_type=target_type,
+            shape=shape,
+            **init_kwargs,
+        )
+
+    # Adding ordinal typed feature, but ultimately we should rethink how we want to model this type
+    elif utils.is_same_class(
+        target_type, CategoricalTypedFeature
+    ) or utils.is_same_class(target_type, OrdinalTypedFeature):
+        if target_type.meta_data["internal"]["unique"] == 2:
+            if is_text_data(sampled_df, target):
+                return BinaryTextClassificationDataset(
+                    df=df,
+                    sampled_df=sampled_df,
+                    target=target,
+                    shape=shape,
+                    target_type=target_type,
+                    positive_class=positive_class,
+                    **init_kwargs,
+                )
+
+            return BinaryClassificationDataset(
+                df=df,
+                sampled_df=sampled_df,
+                target=target,
+                shape=shape,
+                target_type=target_type,
+                positive_class=positive_class,
+                **init_kwargs,
+            )
+        else:
+            if is_text_data(sampled_df, target):
+                return MultiClassTextClassificationDataset(
+                    df=df,
+                    sampled_df=sampled_df,
+                    target=target,
+                    target_type=target_type,
+                    shape=shape,
+                    **init_kwargs,
+                )
+            return MultiClassClassificationDataset(
+                df=df,
+                sampled_df=sampled_df,
+                target=target,
+                target_type=target_type,
+                shape=shape,
+                **init_kwargs,
+            )
+    elif (
+        utils.is_same_class(target, DocumentTypedFeature)
+        or "text" in target_type["type"]
+        or "text" in target
+    ):
+        raise ValueError(f"The column {target} cannot be used as the target column.")
+    elif (
+        utils.is_same_class(target_type, GISTypedFeature)
+        or "coord" in target_type["type"]
+        or "coord" in target
+    ):
+        raise ValueError(f"The column {target} cannot be used as the target column.")
+    # This is to catch constant columns that are boolean. Added as a fix for pd.isnull(), and datasets with a
+    #   binary target, but only data on one instance
+    elif target_type["low_level_type"] == "bool":
+        return BinaryClassificationDataset(
+            df=df,
+            sampled_df=sampled_df,
+            target=target,
+            shape=shape,
+            target_type=target_type,
+            positive_class=positive_class,
+            **init_kwargs,
+        )
+    raise ValueError(
+        f"Unable to identify problem type. Specify the data type of {target} using 'types'. "
+        f"For example, types = {{{target}: 'category'}}"
+    )
+
+
+def open(
+    source,
+    target=None,
+    format="infer",
+    reader_fn: Callable = None,
+    name: str = None,
+    description="",
+    npartitions: int = None,
+    type_discovery=True,
+    html_table_index=None,
+    column_names="infer",
+    sample_max_rows=10000,
+    positive_class=None,
+    transformer_pipeline=None,
+    types={},
+    **kwargs,
+):
+    """
+    Returns an object of ADSDataset or ADSDatasetWithTarget  read from the given path
+
+    .. deprecated:: 2.6.6
+        "Deprecated in favor of using Pandas. Pandas supports reading from object storage directly.
+        Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/loading_data/connect.html",
+
+    Parameters
+    ----------
+    source: Union[str, pandas.DataFrame, h2o.DataFrame, pyspark.sql.dataframe.DataFrame]
+        If str, URI for the dataset. The dataset could be read from local or network file system, hdfs, s3, gcs and optionally pyspark in pyspark
+        conda env
+    target: str, optional
+        Name of the target in dataset.
+        If set an ADSDatasetWithTarget object is returned, otherwise an ADSDataset object is returned which can be
+        used to understand the dataset through visualizations
+    format: str, default: infer
+        Format of the dataset.
+        Supported formats: CSV, TSV, Parquet, libsvm, JSON, XLS/XLSX (Excel), HDF5, SQL, XML,
+        Apache server log files (clf, log), ARFF.
+        By default, the format would be inferred from the ending of the dataset file path.
+    reader_fn: Callable, default: None
+        The user may pass in their own custom reader function.
+        It must accept `(path, **kwarg)` and return a pandas DataFrame
+    name: str, optional default: ""
+    description: str, optional default: ""
+        Text describing the dataset
+    npartitions: int, deprecated
+        Number of partitions to split the data
+        By default this is set to the max number of cores supported by the backend compute accelerator
+    type_discovery: bool, default: True
+        If false, the data types of the dataframe are used as such.
+        By default, the dataframe columns are associated with the best suited data types. Associating the features
+        with the disovered datatypes would impact visualizations and model prediction.
+    html_table_index: int, optional
+        The index of the dataframe table in html content. This is used when the format of dataset is html
+    column_names: 'infer', list of str or None, default: 'infer'
+        Supported only for CSV and TSV.
+        List of column names to use.
+        By default, column names are inferred from the first line of the file.
+        If set to None, column names would be auto-generated instead of inferring from file.
+        If the file already contains a column header, specify header=0 to ignore the existing column names.
+    sample_max_rows: int, default: 10000, use -1 auto calculate sample size, use 0 (zero) for no sampling
+        Sample size of the dataframe to use for visualization and optimization.
+    positive_class: Any, optional
+        Label in target for binary classification problems which should be identified as positive for modeling.
+        By default, the first unique value is considered as the positive label.
+    types: dict, optional
+        Dictionary of <feature_name> : <data_type> to override the data type of features.
+    transformer_pipeline: datasets.pipeline.TransformerPipeline, optional
+        A pipeline of transformations done outside the sdk and need to be applied at the time of scoring
+    storage_options: dict, default: varies by source type
+        Parameters passed on to the backend filesystem class.
+    sep: str
+        Delimiting character for parsing the input file.
+    kwargs: additional keyword arguments that would be passed to underlying dataframe read API
+        based on the format of the dataset
+
+    Returns
+    -------
+    dataset : An instance of ADSDataset
+    (or)
+    dataset_with_target : An instance of ADSDatasetWithTarget
+    """
+    if npartitions:
+        warnings.warn(
+            "Variable `npartitions` is deprecated and will not be used",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+    if (
+        "storage_options" not in kwargs
+        and type(source) is str
+        and len(source) > 6
+        and source[:6] == "oci://"
+    ):
+        kwargs["storage_options"] = {"config": {}}
+
+    if isinstance(source, str) or isinstance(source, list):
+        progress = utils.get_progress_bar(4)
+        progress.update("Opening data")
+        path = ElaboratedPath(source, format=format, **kwargs)
+        reader_fn = (
+            get_format_reader(path=path, **kwargs) if reader_fn is None else reader_fn
+        )
+        df = load_dataset(path=path, reader_fn=reader_fn, **kwargs)
+        name = path.name
+    elif isinstance(source, pd.DataFrame):
+        progress = utils.get_progress_bar(4)
+        progress.update("Partitioning data")
+        df = source
+        name = "User Provided DataFrame" if name is None else name
+    else:
+        raise TypeError(
+            f"The Source type: {type(source)} is not supported for DatasetFactory."
+        )
+    shape = df.shape
+    return build_dataset(
+        df=df,
+        shape=shape,
+        target=target,
+        sample_max_rows=sample_max_rows,
+        type_discovery=type_discovery,
+        types=types,
+        positive_class=positive_class,
+        name=name,
+        transformer_pipeline=transformer_pipeline,
+        description=description,
+        progress=progress,
+        **utils.inject_and_copy_kwargs(
+            kwargs,
+            **{"html_table_index": html_table_index, "column_names": column_names},
+        ),
+    )
+
+
+def build_dataset(
+    df: pd.DataFrame,
+    shape: Tuple[int, int],
+    target: str = None,
+    progress=None,
+    **kwargs,
+):
+    from ads.dataset.dataset import ADSDataset
+
+    n = shape[0]
+    if progress:
+        progress.update("Generating data sample")
+
+    sampled_df = generate_sample(
+        df,
+        n,
+        DatasetDefaults.sampling_confidence_level,
+        DatasetDefaults.sampling_confidence_interval,
+        **kwargs,
+    )
+
+    if target is None:
+        if progress:
+            progress.update("Building the dataset with no target.")
+        result = ADSDataset(df=df, sampled_df=sampled_df, shape=shape, **kwargs)
+        if progress:
+            progress.update("Done")
+        logger.info(
+            "Use `set_target()` to type the dataset for a particular learning task."
+        )
+        return result
+
+    if progress:
+        progress.update("Building dataset")
+
+    discover_target_type = kwargs["type_discovery"]
+    if target in kwargs["types"]:
+        sampled_df[target] = sampled_df[target].astype(kwargs["types"][target])
+        discover_target_type = False
+
+    # if type discovery is turned off, infer type from pandas dtype
+    target_type = infer_target_type(target, sampled_df[target], discover_target_type)
+
+    result = get_dataset(
+        df=df,
+        sampled_df=sampled_df,
+        target=target,
+        target_type=target_type,
+        shape=shape,
+        **kwargs,
+    )
+    if progress:
+        progress.update("Done")
+    logger.info(
+        "Use `suggest_recommendations()` to view and apply recommendations for dataset optimization."
+    )
+    return result
+
+
+class CustomFormatReaders:
+    @staticmethod
+    def read_tsv(path: str, **kwargs) -> pd.DataFrame:
+        return pd.read_csv(
+            path, **utils.inject_and_copy_kwargs(kwargs, **{"sep": "\t"})
+        )
+
+    @staticmethod
+    def read_json(path: str, **kwargs) -> pd.DataFrame:
+        try:
+            return pd.read_json(path, **kwargs)
+        except ValueError as e:
+            return pd.read_json(
+                path, **utils.inject_and_copy_kwargs(kwargs, **{"lines": True})
+            )
+
+    @staticmethod
+    def read_libsvm(path: str, **kwargs) -> pd.DataFrame:
+        from sklearn.datasets import load_svmlight_file
+        from joblib import Memory
+
+        mem = Memory("./mycache")
+
+        @mem.cache
+        def get_data(path):
+            X, y = load_svmlight_file(path)
+            df = pd.DataFrame(X.todense())
+            df["target"] = y
+            return df
+
+        return get_data(path)
+
+    @staticmethod
+    @runtime_dependency(
+        module="pandavro", object="read_avro", install_from=OptionalDependency.DATA
+    )
+    def read_avro(path: str, **kwargs) -> pd.DataFrame:
+        return read_avro(path, **kwargs)
+
+    DEFAULT_SQL_CHUNKSIZE = 12007
+    DEFAULT_SQL_ARRAYSIZE = 50000
+    DEFAULT_SQL_MIL = 128
+    DEFAULT_SQL_CTU = False
+
+    @classmethod
+    def read_sql(cls, path: str, table: str = None, **kwargs) -> pd.DataFrame:
+        """
+
+        :param path: str
+            This is the connection URL that gets passed to sqlalchemy's create_engine method
+        :param table: str
+            This is either the name of a table to select * from or a sql query to be run
+        :param kwargs:
+        :return: pd.DataFrame
+        """
+        if table is None:
+            raise ValueError(
+                "In order to read from a database you need to specify the table using the `table` "
+                "argument."
+            )
+        # check if it's oracle dialect
+        if str(path).lower().startswith("oracle"):
+            kwargs = utils.inject_and_copy_kwargs(
+                kwargs,
+                **{
+                    "arraysize": cls.DEFAULT_SQL_ARRAYSIZE,
+                    "max_identifier_length": cls.DEFAULT_SQL_MIL,
+                    "coerce_to_unicode": cls.DEFAULT_SQL_CTU,
+                },
+            )
+        engine = utils.get_sqlalchemy_engine(path, **kwargs)
+
+        table_name = table.strip()
+        with engine.connect() as connection:
+            # if it's a query expression:
+            if table_name.lower().startswith("select"):
+                sql_query = table_name
+            else:
+                sql_query = f"select * from {table_name}"
+
+            chunks = pd.read_sql_query(
+                sql_query,
+                con=connection,
+                **validate_kwargs(
+                    pd.read_sql_query,
+                    utils.inject_and_copy_kwargs(
+                        kwargs, **{"chunksize": cls.DEFAULT_SQL_CHUNKSIZE}
+                    ),
+                ),
+            )
+            df = pd.DataFrame()
+            from tqdm import tqdm
+
+            with tqdm(chunks, unit=" rows") as t:
+                for chunk in chunks:
+                    df = pd.concat([df, chunk])
+                    t.update(len(chunk))
+
+            df = df.reset_index(drop=True)
+            if df.shape[0] == 0:
+                logger.warning(
+                    "The SQL expression returned zero rows. Therefore, no `ADSdataset` object was created."
+                )
+                raise Exception("The SQL expression returned no rows")
+        return df
+
+    @staticmethod
+    def read_log(path, **kwargs):
+        from ads.dataset.helper import parse_apache_log_str, parse_apache_log_datetime
+
+        df = pd.read_csv(
+            path,
+            # assume_missing=True,
+            sep=r'\s(?=(?:[^"]*"[^"]*")*[^"]*$)(?![^\[]*\])',
+            engine="python",
+            na_values="-",
+            header=None,
+            names=[
+                "host",
+                "identity",
+                "user",
+                "time",
+                "request",
+                "http_code",
+                "response_bytes",
+                "referer",
+                "user_agent",
+                "unknown",
+            ],
+            converters={
+                "time": parse_apache_log_datetime,
+                "request": parse_apache_log_str,
+                "status": int,
+                "size": int,
+                "referer": parse_apache_log_str,
+                "user_agent": parse_apache_log_str,
+            },
+            **kwargs,
+        )
+        return df
+
+    @staticmethod
+    def read_html(path, html_table_index: int = None, **kwargs):
+        if html_table_index is None:
+            return pd.concat(df for df in pd.read_html(path, **kwargs))
+        else:
+            return pd.read_html(path, **kwargs)[html_table_index]
+
+    @staticmethod
+    @runtime_dependency(module="scipy", install_from=OptionalDependency.VIZ)
+    def read_arff(path, **kwargs):
+        from scipy.io import arff
+        import requests
+        from io import BytesIO, TextIOWrapper
+
+        data = None
+        if os.path.isfile(path):
+            data, _ = arff.loadarff(path)
+        else:
+            with requests.get(path) as r:
+                if r.status_code == requests.codes.ok:
+                    f = TextIOWrapper(BytesIO(r.content))
+                    data, _ = arff.loadarff(f)
+        return pd.DataFrame(data)
+
+    @staticmethod
+    def read_xml(path: str, **kwargs) -> pd.DataFrame:
+        """
+        Load data from xml file.
+
+        Parameters
+        ----------
+        path: str
+            Path to XML file
+        storage_options: dict, optional
+            Storage options passed to Pandas to read the file.
+
+        Returns
+        -------
+        dataframe : pandas.DataFrame
+        """
+        import xml.etree.cElementTree as et
+
+        def get_children(df, node, parent, i):
+            for name in node.attrib.keys():
+                df.at[i, parent + name] = node.attrib[name]
+            for child in list(node):
+                if len(list(child)) > 0:
+                    get_children(df, child, parent + child.tag + "/", i)
+                else:
+                    df.at[i, parent + child.tag] = child.text
+
+        storage_options = kwargs.get("storage_options", {})
+
+        file_handles = fsspec.open_files(path, mode="rb", **storage_options)
+        ret_df = pd.DataFrame()
+        last_i = 0
+        for file_handle in file_handles:
+            with file_handle:
+                parsed_xml = et.parse(path)
+                for i, node in enumerate(parsed_xml.getroot()):
+                    get_children(ret_df, node, node.tag + "/", last_i + i)
+                last_i = i
+        return ret_df
+
+
+reader_fns = {
+    "csv": pd.read_csv,
+    "tsv": CustomFormatReaders.read_tsv,
+    "json": CustomFormatReaders.read_json,
+    "jsonl": CustomFormatReaders.read_json,
+    "excel": pd.read_excel,
+    "xls": pd.read_excel,
+    "xlsx": pd.read_excel,
+    "parquet": pd.read_parquet,
+    "libsvm": CustomFormatReaders.read_libsvm,
+    "hdf": pd.read_hdf,  # Todo: re.match(format, "hdf\d*") or format == "h5"
+    "hdf3": pd.read_hdf,
+    "hdf4": pd.read_hdf,
+    "h5": pd.read_hdf,
+    "avro": CustomFormatReaders.read_avro,
+    "avsc": CustomFormatReaders.read_avro,
+    "sql": CustomFormatReaders.read_sql,
+    "db": CustomFormatReaders.read_sql,
+    "log": CustomFormatReaders.read_log,
+    "clf": CustomFormatReaders.read_log,
+    "html": CustomFormatReaders.read_html,
+    "arff": CustomFormatReaders.read_arff,
+    "xml": CustomFormatReaders.read_xml,
+}
+
+
+def validate_kwargs(func: Callable, kwargs):
+    valid_params = inspect.signature(func).parameters
+    if "kwargs" in valid_params:
+        return kwargs
+    else:
+        return {k: v for k, v in kwargs.items() if k in valid_params}
+
+
+def get_format_reader(path: ElaboratedPath, **kwargs) -> Callable:
+    format_key = path.format
+    try:
+        reader_fn = reader_fns[format_key]
+    except (KeyError, NameError):
+        raise ValueError(
+            f"We were unable to load the specified dataset. We have interpreted the format "
+            f"as {format_key}, if this is not correct, call again and set the `format` parameter = "
+            f"to the desired format. Read more here: https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+
+    return reader_fn
+
+
+def load_dataset(path: ElaboratedPath, reader_fn: Callable, **kwargs) -> pd.DataFrame:
+    dfs = []
+    for filename in path.paths:
+        data = reader_fn(filename, **validate_kwargs(reader_fn, kwargs))
+        if not isinstance(data, pd.DataFrame):
+            fn_name = f"{reader_fn.__module__}.{reader_fn.__qualname__}"
+            raise ValueError(
+                f"{fn_name} is used to load the data. "
+                f"However, {fn_name} returned {type(data)} instead of pandas DataFrame. "
+                f"Refer to the usage of {fn_name} to set the correct arguments."
+            )
+        dfs.append(data)
+    if len(dfs) == 0:
+        raise ValueError(
+            f"We were unable to load the specified dataset. Read more here: "
+            f"https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+
+    df = pd.concat(dfs)
+
+    if df is None:
+        raise ValueError(
+            f"We were unable to load the specified dataset. Read more here: "
+            f"https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+    if df.empty:
+        raise DatasetLoadException("Empty DataFrame, not producing a ADSDataset")
+    return df