oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/dataset/factory.py

@@ -0,0 +1,987 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from __future__ import print_function, absolute_import
+
+import os
+import re
+import warnings
+import oci
+import datetime
+import pandas as pd
+from fsspec.utils import infer_storage_options
+import inspect
+import fsspec
+
+from ads.common import utils
+from ads.common.utils import is_same_class
+from ads.dataset import logger
+from ads.dataset.classification_dataset import (
+    BinaryClassificationDataset,
+    MultiClassClassificationDataset,
+    BinaryTextClassificationDataset,
+    MultiClassTextClassificationDataset,
+)
+from ads.dataset.dataset import ADSDataset
+from ads.dataset.forecasting_dataset import ForecastingDataset
+from ads.dataset.helper import (
+    get_feature_type,
+    is_text_data,
+    generate_sample,
+    DatasetDefaults,
+    ElaboratedPath,
+    DatasetLoadException,
+)
+from ads.dataset.regression_dataset import RegressionDataset
+from ads.type_discovery.type_discovery_driver import TypeDiscoveryDriver
+from ads.type_discovery.typed_feature import (
+    ContinuousTypedFeature,
+    DateTimeTypedFeature,
+    CategoricalTypedFeature,
+    OrdinalTypedFeature,
+    GISTypedFeature,
+    DocumentTypedFeature,
+)
+from ads.type_discovery.typed_feature import TypedFeature
+from typing import Callable, Tuple
+from ocifs import OCIFileSystem
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.common.decorator.deprecate import deprecated
+
+default_snapshots_dir = None
+default_storage_options = None
+mindate = datetime.date(datetime.MINYEAR, 1, 1)
+
+
+warnings.warn(
+    (
+        "The `ads.dataset.factory` is deprecated in `oracle-ads 2.8.8` and will be removed in `oracle-ads 3.0`."
+        "Use Pandas to read from local files or object storage directly. "
+        "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/loading_data/connect.html."
+    ),
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
+class DatasetFactory:
+    @staticmethod
+    @deprecated(
+        "2.6.6",
+        details="Deprecated in favor of using Pandas. Pandas supports reading from object storage directly. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/loading_data/connect.html",
+    )
+    def open(
+        source,
+        target=None,
+        format="infer",
+        reader_fn: Callable = None,
+        name: str = None,
+        description="",
+        npartitions: int = None,
+        type_discovery=True,
+        html_table_index=None,
+        column_names="infer",
+        sample_max_rows=10000,
+        positive_class=None,
+        transformer_pipeline=None,
+        types={},
+        **kwargs,
+    ):
+        """
+        Returns an object of ADSDataset or ADSDatasetWithTarget  read from the given path
+
+        .. deprecated:: 2.6.6
+            "Deprecated in favor of using Pandas. Pandas supports reading from object storage directly.
+            Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/loading_data/connect.html",
+
+        Parameters
+        ----------
+        source: Union[str, pandas.DataFrame, h2o.DataFrame, pyspark.sql.dataframe.DataFrame]
+            If str, URI for the dataset. The dataset could be read from local or network file system, hdfs, s3, gcs and optionally pyspark in pyspark
+            conda env
+        target: str, optional
+            Name of the target in dataset.
+            If set an ADSDatasetWithTarget object is returned, otherwise an ADSDataset object is returned which can be
+            used to understand the dataset through visualizations
+        format: str, default: infer
+            Format of the dataset.
+            Supported formats: CSV, TSV, Parquet, libsvm, JSON, XLS/XLSX (Excel), HDF5, SQL, XML,
+            Apache server log files (clf, log), ARFF.
+            By default, the format would be inferred from the ending of the dataset file path.
+        reader_fn: Callable, default: None
+            The user may pass in their own custom reader function.
+            It must accept `(path, **kwarg)` and return a pandas DataFrame
+        name: str, optional default: ""
+        description: str, optional default: ""
+            Text describing the dataset
+        npartitions: int, deprecated
+            Number of partitions to split the data
+            By default this is set to the max number of cores supported by the backend compute accelerator
+        type_discovery: bool, default: True
+            If false, the data types of the dataframe are used as such.
+            By default, the dataframe columns are associated with the best suited data types. Associating the features
+            with the disovered datatypes would impact visualizations and model prediction.
+        html_table_index: int, optional
+            The index of the dataframe table in html content. This is used when the format of dataset is html
+        column_names: 'infer', list of str or None, default: 'infer'
+            Supported only for CSV and TSV.
+            List of column names to use.
+            By default, column names are inferred from the first line of the file.
+            If set to None, column names would be auto-generated instead of inferring from file.
+            If the file already contains a column header, specify header=0 to ignore the existing column names.
+        sample_max_rows: int, default: 10000, use -1 auto calculate sample size, use 0 (zero) for no sampling
+            Sample size of the dataframe to use for visualization and optimization.
+        positive_class: Any, optional
+            Label in target for binary classification problems which should be identified as positive for modeling.
+            By default, the first unique value is considered as the positive label.
+        types: dict, optional
+            Dictionary of <feature_name> : <data_type> to override the data type of features.
+        transformer_pipeline: datasets.pipeline.TransformerPipeline, optional
+            A pipeline of transformations done outside the sdk and need to be applied at the time of scoring
+        storage_options: dict, default: varies by source type
+            Parameters passed on to the backend filesystem class.
+        sep: str
+            Delimiting character for parsing the input file.
+        kwargs: additional keyword arguments that would be passed to underlying dataframe read API
+            based on the format of the dataset
+
+        Returns
+        -------
+        dataset : An instance of ADSDataset
+        (or)
+        dataset_with_target : An instance of ADSDatasetWithTarget
+
+        Examples
+        --------
+        >>> ds = DatasetFactory.open("/path/to/data.data", format='csv', delimiter=" ",
+        ...          na_values="n/a", skipinitialspace=True)
+
+        >>> ds = DatasetFactory.open("/path/to/data.csv", target="col_1", prefix="col_",
+        ...           skiprows=1, encoding="ISO-8859-1")
+
+        >>> ds = DatasetFactory.open("oci://bucket@namespace/path/to/data.tsv",
+        ...         column_names=["col1", "col2", "col3"], header=0)
+
+        >>> ds = DatasetFactory.open("oci://bucket@namespace/path/to/data.csv",
+        ...         storage_options={"config": "~/.oci/config",
+        ...         "profile": "USER_2"}, delimiter = ';')
+
+        >>> ds = DatasetFactory.open("/path/to/data.parquet", engine='pyarrow',
+        ...         types={"col1": "ordinal",
+        ...                "col2": "categorical",
+        ...                "col3" : "continuous",
+        ...                "col4" : "float64"})
+
+        >>> ds = DatasetFactory.open(df, target="class", sample_max_rows=5000,
+        ...          positive_class="yes")
+
+        >>> ds = DatasetFactory.open("s3://path/to/data.json.gz", format="json",
+        ...         compression="gzip", orient="records")
+        """
+        if npartitions:
+            warnings.warn(
+                "Variable `npartitions` is deprecated and will not be used",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        if (
+            "storage_options" not in kwargs
+            and type(source) is str
+            and len(source) > 6
+            and source[:6] == "oci://"
+        ):
+            kwargs["storage_options"] = {"config": {}}
+
+        if isinstance(source, str) or isinstance(source, list):
+            progress = utils.get_progress_bar(4)
+            progress.update("Opening data")
+            path = ElaboratedPath(source, format=format, **kwargs)
+            reader_fn = (
+                get_format_reader(path=path, **kwargs)
+                if reader_fn is None
+                else reader_fn
+            )
+            df = load_dataset(path=path, reader_fn=reader_fn, **kwargs)
+            name = path.name
+        elif isinstance(source, pd.DataFrame):
+            progress = utils.get_progress_bar(4)
+            progress.update("Partitioning data")
+            df = source
+            name = "User Provided DataFrame" if name is None else name
+        else:
+            raise TypeError(
+                f"The Source type: {type(source)} is not supported for DatasetFactory."
+            )
+        shape = df.shape
+        return DatasetFactory._build_dataset(
+            df=df,
+            shape=shape,
+            target=target,
+            sample_max_rows=sample_max_rows,
+            type_discovery=type_discovery,
+            types=types,
+            positive_class=positive_class,
+            name=name,
+            transformer_pipeline=transformer_pipeline,
+            description=description,
+            progress=progress,
+            **utils.inject_and_copy_kwargs(
+                kwargs,
+                **{"html_table_index": html_table_index, "column_names": column_names},
+            ),
+        )
+
+    @staticmethod
+    def open_to_pandas(
+        source: str, format: str = None, reader_fn: Callable = None, **kwargs
+    ) -> pd.DataFrame:
+        path = ElaboratedPath(source, format=format, **kwargs)
+        reader_fn = (
+            get_format_reader(path=path, **kwargs) if reader_fn is None else reader_fn
+        )
+        df = load_dataset(path=path, reader_fn=reader_fn, **kwargs)
+        return df
+
+    @staticmethod
+    def from_dataframe(df, target: str = None, **kwargs):
+        """
+        Returns an object of ADSDatasetWithTarget or ADSDataset given a pandas.DataFrame
+
+        Parameters
+        ----------
+        df: pandas.DataFrame
+        target: str
+        kwargs: dict
+            See DatasetFactory.open() for supported kwargs
+
+        Returns
+        -------
+        dataset: an object of ADSDataset target is not specified, otherwise an object of ADSDatasetWithTarget tagged
+                  according to the type of target
+
+        Examples
+        --------
+        >>> df = pd.DataFrame(data)
+        >>> ds = from_dataframe(df)
+        """
+        return DatasetFactory.open(df, target=target, **kwargs)
+
+    @staticmethod
+    @runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
+    @runtime_dependency(
+        module="ipywidgets",
+        object="HTML",
+        is_for_notebook_only=True,
+        install_from=OptionalDependency.NOTEBOOK,
+    )
+    def list_snapshots(snapshot_dir=None, name="", storage_options=None, **kwargs):
+        """
+        Displays the URIs for dataset snapshots under the given directory path.
+
+        Parameters
+        ----------
+        snapshot_dir: str
+            Return all dataset snapshots created using ADSDataset.snapshot() within this directory.
+            The path can contain protocols such as oci, s3.
+        name: str, optional
+            The list of snapshots in the directory gets filtered by the name. Accepts glob expressions.
+            default = `"ads_"`
+        storage_options: dict
+            Parameters passed on to the backend filesystem class.
+
+        Example
+        --------
+        >>> DatasetFactory.list_snapshots(snapshot_dir="oci://my_bucket/snapshots_dir",
+        ...             name="ads_iris_")
+
+        Returns a list of all snapshots (recursively) saved to obj storage bucket `"my_bucket"` with prefix
+        `"/snapshots_dir/ads_iris_**"` sorted by time created.
+        """
+        if snapshot_dir is None:
+            snapshot_dir = default_snapshots_dir
+            if snapshot_dir is None:
+                raise ValueError(
+                    "Specify snapshot_dir or use DatasetFactory.set_default_storage() to set default \
+                                      storage options"
+                )
+            else:
+                logger.info("Using default snapshots dir %s" % snapshot_dir)
+        if storage_options is None:
+            if default_storage_options is not None:
+                storage_options = default_storage_options
+                logger.info("Using default storage options")
+            else:
+                storage_options = dict()
+        assert isinstance(storage_options, dict), (
+            "The storage options parameter must be a dictionary. You can set "
+            "this gloabally by calling DatasetFactory.set_default_storage("
+            "storage_options={'config': 'location'}). "
+        )
+        url_options = infer_storage_options(snapshot_dir)
+        protocol = url_options.pop("protocol", None)
+
+        fs = OCIFileSystem(config=storage_options.get("config", None))
+        kwargs.update({"refresh": True})
+        obj_list = [
+            (k, v.get("timeCreated", mindate).strftime("%Y-%m-%d %H:%M:%S"))
+            for k, v in fs.glob(
+                os.path.join(snapshot_dir, name + "**"), detail=True, **kwargs
+            ).items()
+            if v["type"] == "file"
+        ]
+
+        files = []
+        for file, file_time in obj_list:
+            if protocol in ["oci"]:
+                r1 = re.compile(r"/part\.[0-9]{1,6}\.parquet$")
+                parquet_part = r1.search(file)
+                if parquet_part is not None:
+                    parquet_filename = file[: parquet_part.start()]
+                elif file.endswith("/_common_metadata"):
+                    parquet_filename = file[: -len("/_common_metadata")]
+                elif file.endswith("/_metadata"):
+                    parquet_filename = file[: -len("/_metadata")]
+                else:
+                    parquet_filename = file
+            else:
+                parquet_filename = file
+            parent_path = "%s://" % protocol
+            files.append((parent_path + parquet_filename, file_time))
+        files.sort(key=lambda x: x[1] or mindate, reverse=True)
+        list_df = pd.DataFrame(files, columns=["Name", "Created Time"])
+        list_df = list_df.drop_duplicates(subset=["Name"]).reset_index()
+        if len(list_df) == 0:
+            print(f"No snapshots found at: {os.path.join(snapshot_dir, name)}")
+
+        # display in HTML format if sdk is run in notebook mode
+        if utils.is_notebook():
+            from IPython.core.display import display
+
+            display(
+                HTML(
+                    list_df.style.set_table_attributes("class=table")
+                    .hide()
+                    .to_html()
+                )
+            )
+        return list_df
+
+    @staticmethod
+    def download(remote_path, local_path, storage=None, overwrite=False):
+        """
+        Download a remote file or directory to local storage.
+
+        Parameters
+        ---------
+        remote_path: str
+            Supports protocols like oci, s3, also supports glob expressions
+        local_path: str
+            Supports glob expressions
+        storage: dict
+            Parameters passed on to the  backend remote filesystem class.
+        overwrite: bool, default False
+            If True, the method will overwrite any existing files in the local_path
+
+        Examples
+        ---------
+        >>> DatasetFactory.download("oci://Bucket/prefix/to/data/*.csv",
+        ...         "/home/datascience/data/")
+        """
+        if storage is None:
+            if default_storage_options is not None:
+                storage = default_storage_options
+                logger.info("Using default storage options")
+            else:
+                storage = dict()
+
+        remote_files = fsspec.open_files(
+            remote_path, mode="rb", name_function=lambda i: "", **storage
+        )
+        if len(remote_files) < 1:
+            raise FileNotFoundError(remote_path)
+        display_error, error_msg = DatasetFactory._download_files(
+            remote_files=remote_files, local_path=local_path, overwrite=overwrite
+        )
+        if display_error:
+            logger.error(error_msg)
+        else:
+            logger.info(f"Download {remote_path} to {local_path}.")
+
+    @staticmethod
+    def _download_files(remote_files, local_path, overwrite=False):
+        display_error, error_msg = False, ""
+        for remote_file in remote_files:
+            bucket_idx = remote_file.path.find("/")
+            suffix = remote_file.path[bucket_idx + 1 :]
+            try:
+                with remote_file as f1:
+                    local_filepath = (
+                        os.path.join(local_path, suffix) if suffix else local_path
+                    )
+                    if os.path.exists(local_filepath) and not overwrite:
+                        raise FileExistsError(
+                            f"Trying to overwrite files in {local_filepath}. If you'd like to "
+                            f"overwrite these files, set force_overwrite to True."
+                        )
+                    os.makedirs(os.path.dirname(local_filepath), exist_ok=True)
+                    with open(local_filepath, "wb") as f2:
+                        f2.write(f1.read())
+            except oci.exceptions.ServiceError as e:
+                raise FileNotFoundError(f"Unable to open file: {remote_file.path}")
+        return display_error, error_msg
+
+    @staticmethod
+    def upload(local_file_or_dir, remote_file_or_dir, storage_options=None):
+        """
+        Upload local file or directory to remote storage
+
+        Parameters
+        ---------
+        local_file_or_dir: str
+            Supports glob expressions
+        remote_file_or_dir: str
+            Supports protocols like oci, s3, also supports glob expressions
+        storage_options: dict
+            Parameters passed on to the  backend remote filesystem class.
+        """
+        if not os.path.exists(local_file_or_dir):
+            raise ValueError("File/Directory does not exist: %s" % local_file_or_dir)
+        if storage_options is None and default_storage_options is not None:
+            storage_options = default_storage_options
+            logger.info("Using default storage options")
+
+        if os.path.isdir(local_file_or_dir):
+            for subdir, dirs, files in os.walk(local_file_or_dir):
+                for file in files:
+                    if os.path.abspath(subdir) == os.path.abspath(local_file_or_dir):
+                        path = file
+                    else:
+                        path = os.path.join(
+                            os.path.abspath(subdir).split("/", 2)[2], file
+                        )
+                    DatasetFactory._upload_file(
+                        os.path.join(subdir, file),
+                        os.path.join(remote_file_or_dir, path),
+                        storage_options=storage_options,
+                    )
+        else:
+            DatasetFactory._upload_file(
+                local_file_or_dir, remote_file_or_dir, storage_options=storage_options
+            )
+
+    @staticmethod
+    def set_default_storage(snapshots_dir=None, storage_options=None):
+        """
+        Set default storage directory and options.
+
+        Both snapshots_dir and storage_options can be overridden at the API scope.
+
+        Parameters
+        ----------
+        snapshots_dir: str
+            Path for the snapshots directory. Can contain protocols such as oci, s3
+        storage_options: dict, optional
+             Parameters passed on to the backend filesystem class.
+        """
+        global default_snapshots_dir
+        default_snapshots_dir = snapshots_dir
+        global default_storage_options
+        if storage_options is not None:
+            assert isinstance(storage_options, dict), (
+                f"The storage options parameter must be a dictionary. Instead "
+                f"we got the type: {type(storage_options)} "
+            )
+        default_storage_options = storage_options
+
+    @classmethod
+    def _upload_file(cls, local_file, remote_file, storage_options=None):
+        kwargs = {}
+        if storage_options is not None:
+            kwargs = {"storage_options": storage_options}
+        remote_file_handler = fsspec.open_files(
+            remote_file + "*", mode="wb", name_function=lambda i: "", **kwargs
+        )[0]
+        with remote_file_handler as f1:
+            with open(local_file, "rb") as f2:
+                for line in f2:
+                    f1.write(line)
+        print("Uploaded %s to %s" % (local_file, remote_file))
+
+    @classmethod
+    def _build_dataset(
+        cls,
+        df: pd.DataFrame,
+        shape: Tuple[int, int],
+        target: str = None,
+        progress=None,
+        **kwargs,
+    ):
+        n = shape[0]
+        if progress:
+            progress.update("Generating data sample")
+
+        sampled_df = generate_sample(
+            df,
+            n,
+            DatasetDefaults.sampling_confidence_level,
+            DatasetDefaults.sampling_confidence_interval,
+            **kwargs,
+        )
+
+        if target is None:
+            if progress:
+                progress.update("Building the dataset with no target.")
+            result = ADSDataset(df=df, sampled_df=sampled_df, shape=shape, **kwargs)
+            if progress:
+                progress.update("Done")
+            logger.info(
+                "Use `set_target()` to type the dataset for a particular learning task."
+            )
+            return result
+
+        if progress:
+            progress.update("Building dataset")
+
+        discover_target_type = kwargs["type_discovery"]
+        if target in kwargs["types"]:
+            sampled_df[target] = sampled_df[target].astype(kwargs["types"][target])
+            discover_target_type = False
+
+        # if type discovery is turned off, infer type from pandas dtype
+        target_type = DatasetFactory.infer_target_type(
+            target, sampled_df[target], discover_target_type
+        )
+
+        result = DatasetFactory._get_dataset(
+            df=df,
+            sampled_df=sampled_df,
+            target=target,
+            target_type=target_type,
+            shape=shape,
+            **kwargs,
+        )
+        if progress:
+            progress.update("Done")
+        logger.info(
+            "Use `suggest_recommendations()` to view and apply recommendations for dataset optimization."
+        )
+        return result
+
+    @classmethod
+    def infer_target_type(cls, target, target_series, discover_target_type=True):
+        # if type discovery is turned off, infer type from pandas dtype
+        if discover_target_type:
+            target_type = TypeDiscoveryDriver().discover(
+                target, target_series, is_target=True
+            )
+        else:
+            target_type = get_feature_type(target, target_series)
+        return target_type
+
+    @classmethod
+    def _get_dataset(
+        cls,
+        df: pd.DataFrame,
+        sampled_df: pd.DataFrame,
+        target: str,
+        target_type: TypedFeature,
+        shape: Tuple[int, int],
+        positive_class=None,
+        **init_kwargs,
+    ):
+        if len(df[target].dropna()) == 0:
+            logger.warning(
+                "It is not recommended to use an empty column as the target variable."
+            )
+            raise ValueError(
+                f"We do not support using empty columns as the chosen target"
+            )
+        if is_same_class(target_type, ContinuousTypedFeature):
+            return RegressionDataset(
+                df=df,
+                sampled_df=sampled_df,
+                target=target,
+                target_type=target_type,
+                shape=shape,
+                **init_kwargs,
+            )
+        elif is_same_class(
+            target_type, DateTimeTypedFeature
+        ) or df.index.dtype.name.startswith("datetime"):
+            return ForecastingDataset(
+                df=df,
+                sampled_df=sampled_df,
+                target=target,
+                target_type=target_type,
+                shape=shape,
+                **init_kwargs,
+            )
+
+        # Adding ordinal typed feature, but ultimately we should rethink how we want to model this type
+        elif is_same_class(target_type, CategoricalTypedFeature) or is_same_class(
+            target_type, OrdinalTypedFeature
+        ):
+            if target_type.meta_data["internal"]["unique"] == 2:
+                if is_text_data(sampled_df, target):
+                    return BinaryTextClassificationDataset(
+                        df=df,
+                        sampled_df=sampled_df,
+                        target=target,
+                        shape=shape,
+                        target_type=target_type,
+                        positive_class=positive_class,
+                        **init_kwargs,
+                    )
+
+                return BinaryClassificationDataset(
+                    df=df,
+                    sampled_df=sampled_df,
+                    target=target,
+                    shape=shape,
+                    target_type=target_type,
+                    positive_class=positive_class,
+                    **init_kwargs,
+                )
+            else:
+                if is_text_data(sampled_df, target):
+                    return MultiClassTextClassificationDataset(
+                        df=df,
+                        sampled_df=sampled_df,
+                        target=target,
+                        target_type=target_type,
+                        shape=shape,
+                        **init_kwargs,
+                    )
+                return MultiClassClassificationDataset(
+                    df=df,
+                    sampled_df=sampled_df,
+                    target=target,
+                    target_type=target_type,
+                    shape=shape,
+                    **init_kwargs,
+                )
+        elif (
+            is_same_class(target, DocumentTypedFeature)
+            or "text" in target_type["type"]
+            or "text" in target
+        ):
+            raise ValueError(
+                f"The column {target} cannot be used as the target column."
+            )
+        elif (
+            is_same_class(target_type, GISTypedFeature)
+            or "coord" in target_type["type"]
+            or "coord" in target
+        ):
+            raise ValueError(
+                f"The column {target} cannot be used as the target column."
+            )
+        # This is to catch constant columns that are boolean. Added as a fix for pd.isnull(), and datasets with a
+        #   binary target, but only data on one instance
+        elif target_type["low_level_type"] == "bool":
+            return BinaryClassificationDataset(
+                df=df,
+                sampled_df=sampled_df,
+                target=target,
+                shape=shape,
+                target_type=target_type,
+                positive_class=positive_class,
+                **init_kwargs,
+            )
+        raise ValueError(
+            f"Unable to identify problem type. Specify the data type of {target} using 'types'. "
+            f"For example, types = {{{target}: 'category'}}"
+        )
+
+
+class CustomFormatReaders:
+    @staticmethod
+    def read_tsv(path: str, **kwargs) -> pd.DataFrame:
+        return pd.read_csv(
+            path, **utils.inject_and_copy_kwargs(kwargs, **{"sep": "\t"})
+        )
+
+    @staticmethod
+    def read_json(path: str, **kwargs) -> pd.DataFrame:
+        try:
+            return pd.read_json(path, **kwargs)
+        except ValueError as e:
+            return pd.read_json(
+                path, **utils.inject_and_copy_kwargs(kwargs, **{"lines": True})
+            )
+
+    @staticmethod
+    def read_libsvm(path: str, **kwargs) -> pd.DataFrame:
+        from sklearn.datasets import load_svmlight_file
+        from joblib import Memory
+
+        mem = Memory("./mycache")
+
+        @mem.cache
+        def get_data(path):
+            X, y = load_svmlight_file(path)
+            df = pd.DataFrame(X.todense())
+            df["target"] = y
+            return df
+
+        return get_data(path)
+
+    @staticmethod
+    @runtime_dependency(
+        module="pandavro", object="read_avro", install_from=OptionalDependency.DATA
+    )
+    def read_avro(path: str, **kwargs) -> pd.DataFrame:
+        return read_avro(path, **kwargs)
+
+    DEFAULT_SQL_CHUNKSIZE = 12007
+    DEFAULT_SQL_ARRAYSIZE = 50000
+    DEFAULT_SQL_MIL = 128
+    DEFAULT_SQL_CTU = False
+
+    @classmethod
+    def read_sql(cls, path: str, table: str = None, **kwargs) -> pd.DataFrame:
+        """
+
+        :param path: str
+            This is the connection URL that gets passed to sqlalchemy's create_engine method
+        :param table: str
+            This is either the name of a table to select * from or a sql query to be run
+        :param kwargs:
+        :return: pd.DataFrame
+        """
+        if table is None:
+            raise ValueError(
+                "In order to read from a database you need to specify the table using the `table` "
+                "argument."
+            )
+        # check if it's oracle dialect
+        if str(path).lower().startswith("oracle"):
+            kwargs = utils.inject_and_copy_kwargs(
+                kwargs,
+                **{
+                    "arraysize": cls.DEFAULT_SQL_ARRAYSIZE,
+                    "max_identifier_length": cls.DEFAULT_SQL_MIL,
+                    "coerce_to_unicode": cls.DEFAULT_SQL_CTU,
+                },
+            )
+        engine = utils.get_sqlalchemy_engine(path, **kwargs)
+
+        table_name = table.strip()
+        with engine.connect() as connection:
+            # if it's a query expression:
+            if table_name.lower().startswith("select"):
+                sql_query = table_name
+            else:
+                sql_query = f"select * from {table_name}"
+
+            chunks = pd.read_sql_query(
+                sql_query,
+                con=connection,
+                **_validate_kwargs(
+                    pd.read_sql_query,
+                    utils.inject_and_copy_kwargs(
+                        kwargs, **{"chunksize": cls.DEFAULT_SQL_CHUNKSIZE}
+                    ),
+                ),
+            )
+            df = pd.DataFrame()
+            from tqdm import tqdm
+
+            with tqdm(chunks, unit=" rows") as t:
+                for chunk in chunks:
+                    df = pd.concat([df, chunk])
+                    t.update(len(chunk))
+
+            df = df.reset_index(drop=True)
+            if df.shape[0] == 0:
+                logger.warning(
+                    "The SQL expression returned zero rows. Therefore, no `ADSdataset` object was created."
+                )
+                raise Exception("The SQL expression returned no rows")
+        return df
+
+    @staticmethod
+    def read_log(path, **kwargs):
+        from ads.dataset.helper import parse_apache_log_str, parse_apache_log_datetime
+
+        df = pd.read_csv(
+            path,
+            # assume_missing=True,
+            sep=r'\s(?=(?:[^"]*"[^"]*")*[^"]*$)(?![^\[]*\])',
+            engine="python",
+            na_values="-",
+            header=None,
+            names=[
+                "host",
+                "identity",
+                "user",
+                "time",
+                "request",
+                "http_code",
+                "response_bytes",
+                "referer",
+                "user_agent",
+                "unknown",
+            ],
+            converters={
+                "time": parse_apache_log_datetime,
+                "request": parse_apache_log_str,
+                "status": int,
+                "size": int,
+                "referer": parse_apache_log_str,
+                "user_agent": parse_apache_log_str,
+            },
+            **kwargs,
+        )
+        return df
+
+    @staticmethod
+    def read_html(path, html_table_index: int = None, **kwargs):
+        if html_table_index is None:
+            return pd.concat(df for df in pd.read_html(path, **kwargs))
+        else:
+            return pd.read_html(path, **kwargs)[html_table_index]
+
+    @staticmethod
+    @runtime_dependency(module="scipy", install_from=OptionalDependency.VIZ)
+    def read_arff(path, **kwargs):
+        from scipy.io import arff
+        import requests
+        from io import BytesIO, TextIOWrapper
+
+        data = None
+        if os.path.isfile(path):
+            data, _ = arff.loadarff(path)
+        else:
+            with requests.get(path) as r:
+                if r.status_code == requests.codes.ok:
+                    f = TextIOWrapper(BytesIO(r.content))
+                    data, _ = arff.loadarff(f)
+        return pd.DataFrame(data)
+
+    @staticmethod
+    def read_xml(path: str, **kwargs) -> pd.DataFrame:
+        """
+        Load data from xml file.
+
+        Parameters
+        ----------
+        path: str
+            Path to XML file
+        storage_options: dict, optional
+            Storage options passed to Pandas to read the file.
+
+        Returns
+        -------
+        dataframe : pandas.DataFrame
+        """
+        import xml.etree.cElementTree as et
+
+        def get_children(df, node, parent, i):
+            for name in node.attrib.keys():
+                df.at[i, parent + name] = node.attrib[name]
+            for child in list(node):
+                if len(list(child)) > 0:
+                    get_children(df, child, parent + child.tag + "/", i)
+                else:
+                    df.at[i, parent + child.tag] = child.text
+
+        storage_options = kwargs.get("storage_options", {})
+
+        file_handles = fsspec.open_files(path, mode="rb", **storage_options)
+        ret_df = pd.DataFrame()
+        last_i = 0
+        for file_handle in file_handles:
+            with file_handle:
+                parsed_xml = et.parse(path)
+                for i, node in enumerate(parsed_xml.getroot()):
+                    get_children(ret_df, node, node.tag + "/", last_i + i)
+                last_i = i
+        return ret_df
+
+
+reader_fns = {
+    "csv": pd.read_csv,
+    "tsv": CustomFormatReaders.read_tsv,
+    "json": CustomFormatReaders.read_json,
+    "jsonl": CustomFormatReaders.read_json,
+    "excel": pd.read_excel,
+    "xls": pd.read_excel,
+    "xlsx": pd.read_excel,
+    "parquet": pd.read_parquet,
+    "libsvm": CustomFormatReaders.read_libsvm,
+    "hdf": pd.read_hdf,  # Todo: re.match(format, "hdf\d*") or format == "h5"
+    "hdf3": pd.read_hdf,
+    "hdf4": pd.read_hdf,
+    "h5": pd.read_hdf,
+    "avro": CustomFormatReaders.read_avro,
+    "avsc": CustomFormatReaders.read_avro,
+    "sql": CustomFormatReaders.read_sql,
+    "db": CustomFormatReaders.read_sql,
+    "log": CustomFormatReaders.read_log,
+    "clf": CustomFormatReaders.read_log,
+    "html": CustomFormatReaders.read_html,
+    "arff": CustomFormatReaders.read_arff,
+    "xml": CustomFormatReaders.read_xml,
+}
+
+
+def _validate_kwargs(func: Callable, kwargs):
+    valid_params = inspect.signature(func).parameters
+    if "kwargs" in valid_params:
+        return kwargs
+    else:
+        return {k: v for k, v in kwargs.items() if k in valid_params}
+
+
+def get_format_reader(path: ElaboratedPath, **kwargs) -> Callable:
+    format_key = path.format
+    try:
+        reader_fn = reader_fns[format_key]
+    except (KeyError, NameError):
+        raise ValueError(
+            f"We were unable to load the specified dataset. We have interpreted the format "
+            f"as {format_key}, if this is not correct, call again and set the `format` parameter = "
+            f"to the desired format. Read more here: https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+
+    return reader_fn
+
+
+def load_dataset(path: ElaboratedPath, reader_fn: Callable, **kwargs) -> pd.DataFrame:
+    dfs = []
+    for filename in path.paths:
+        data = reader_fn(filename, **_validate_kwargs(reader_fn, kwargs))
+        if not isinstance(data, pd.DataFrame):
+            fn_name = f"{reader_fn.__module__}.{reader_fn.__qualname__}"
+            raise ValueError(
+                f"{fn_name} is used to load the data. "
+                f"However, {fn_name} returned {type(data)} instead of pandas DataFrame. "
+                f"Refer to the usage of {fn_name} to set the correct arguments."
+            )
+        dfs.append(data)
+    if len(dfs) == 0:
+        raise ValueError(
+            f"We were unable to load the specified dataset. Read more here: "
+            f"https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+
+    df = pd.concat(dfs)
+
+    if df is None:
+        raise ValueError(
+            f"We were unable to load the specified dataset. Read more here: "
+            f"https://docs.cloud.oracle.com/en-us/iaas/tools/ads"
+            f"-sdk/latest/user_guide/loading_data/loading_data.html#specify-data-types-in-load-dataset"
+        )
+    if df.empty:
+        raise DatasetLoadException("Empty DataFrame, not producing a ADSDataset")
+    return df