oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/dataset/capabilities.md

@@ -1,51 +0,0 @@
-### Dataset Capabilities
-
-The Dataset is an abstraction atop a variety of sources
-   - local file system via Pandas dataframe
-   - remote file systems like s3, Oracle Storage
-   - advanced possibilities include using pyarrow for Apache Arrow datsets
-   - interoperate with Dask.dataframes
-
-The sdk's dataset provides the following capabilities:
-- Support loading dataset from pandas, remote sources, clipboard etc
-  ```
-  df = pd.load_csv(...)
-  ds = sdk.Dataset.fromPandas(df)
-  ```
-- Visualizing of dataset
-    - Show all features, their data types and counts, meta data for a dataset, for example if the dataset is time-series or not.
-    - Show (heatmap) of correlated features
-      ```
-      show_in_notebook(ds)
-      ```
-    - Show distribution of values for features, using a datatype-dependent
-      visualizer (text vs categorical vs numerical will all render differently)
-      ```
-      show_in_notebook(ds, features=["col1", "col2", "col3"])
-      ```
-
-    look at http://pandas.pydata.org/pandas-docs/stable/style.html for some pandas
-    examples, also seaborn https://stackoverflow.com/questions/39409866/correlation-heatmap
-
-- Data cleaning
-    - Support redaction of features
-    - Handle missing values
-    - Identify outliers and inconsistencies
-    - Imputation
-- Identify data type of features
-    - Define a data type hierarchy that includes special patterns like credit cards, zip codes, phone number, etc.
-- Feature selection
-  - features used in a model is part of AutoML, however, for modeling that's not using AutoML the dataset
-    should have feature selection capabilities using a plugin (initial should be information theoretic) - see
-    http://scikit-learn.org/stable/modules/classes.html#module-sklearn.feature_selection for examples
-- Sampling
-  - can be used to limit data used for modeling, can also be used to balance and unbalanced target variable.
-  - "auto mode" determines if up/down is required
-- Features can hold feature encoding hints
-   -  these hints are part of the dataset, not the underlying pandas dataframe
-- Discretization
-  - of target variable for regression to classification problem
-
-- Persistence & interchange
-  - support save as snapshot (possibly parquet format), a snapshot generates a shareable URI
-  - load from snapshot URI

ads/dataset/classification_dataset.py

@@ -1,339 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import pandas as pd
-import warnings
-
-from ads.common import utils, logger
-from ads.dataset import helper
-from ads.dataset.exception import ValidationError
-from ads.dataset.dataset_with_target import ADSDatasetWithTarget
-from sklearn.preprocessing import FunctionTransformer
-from ads.dataset.helper import deprecate_variable, deprecate_default_value
-
-
-class ClassificationDataset(ADSDatasetWithTarget):
-    """
-    Dataset for classification task
-    """
-
-    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
-        ADSDatasetWithTarget.__init__(
-            self, df=df, sampled_df=sampled_df, target=target, target_type=target_type, shape=shape, **kwargs
-        )
-
-    def auto_transform(
-        self,
-        fix_imbalance: bool = True,
-        correlation_threshold: float = 0.7,
-        frac: float = 1.0,
-        correlation_methods: str = "pearson",
-    ):
-        """
-        Return transformed dataset with several optimizations applied automatically.
-        The optimizations include:
-
-        - Dropping constant and primary key columns, which has no predictive quality,
-        - Imputation, to fill in missing values in noisy data:
-
-            - For continuous variables, fill with mean if less than 40% is missing, else drop,
-            - For categorical variables, fill with most frequent if less than 40% is missing, else drop,
-
-        - Dropping strongly co-correlated columns that tend to produce less generalizable models,
-        - Balancing dataset using up or down sampling.
-
-        Parameters
-        ----------
-        fix_imbalance : bool, defaults to True.
-            Fix imbalance between classes in dataset. Used only for classification datasets.
-        correlation_threshold: float, defaults to 0.7. It must be between 0 and 1, inclusive.
-            The correlation threshold where columns with correlation higher than the threshold will
-            be considered as strongly co-correlated and recommended to be taken care of.
-        frac: float, defaults to 1.0. Range -> (0, 1].
-            What fraction of the data should be used in the calculation?
-        correlation_methods: Union[list, str], defaults to 'pearson'.
-
-            - 'pearson': Use Pearson's Correlation between continuous features,
-            - 'cramers v': Use Cramer's V correlations between categorical features,
-            - 'correlation ratio': Use Correlation Ratio Correlation between categorical and continuous features,
-            - 'all': Is equivalent to ['pearson', 'cramers v', 'correlation ratio'].
-
-            Or a list containing any combination of these methods, for example, ['pearson', 'cramers v'].
-
-        Returns
-        -------
-        transformed_dataset : ADSDatasetWithTarget
-            The dataset after transformation
-
-        Examples
-        --------
-        >>> ds_clean = ds.auto_transform(correlation_threshold=0.6)
-        """
-        frac = deprecate_default_value(
-            frac,
-            None,
-            1,
-            f"<code>frac=None</code> is deprecated. Use <code>frac=1.0</code> instead.",
-            FutureWarning,
-        )
-        with utils.get_progress_bar(7) as progress:
-            df, sampled_df, transformer_pipeline = self._transform(
-                progress=progress,
-                fix_imbalance=fix_imbalance,
-                correlation_threshold=correlation_threshold,
-                frac=frac,
-                correlation_methods=correlation_methods,
-            )
-            return self._build_new_dataset(
-                df,
-                sampled_df=sampled_df,
-                transformers=transformer_pipeline.steps,
-                progress=progress,
-            )
-
-    def convert_to_text_classification(self, text_column: str):
-        """
-        Builds a new dataset with the given text column as the only feature besides target.
-
-        Parameters
-        ----------
-        text_column: str
-            Feature name to use for text classification task
-
-        Returns
-        -------
-        ds: TextClassificationDataset
-            Dataset with one text feature and a classification target
-
-        Examples
-        --------
-        >>> review_ds = DatasetFactory.open("review_data.csv")
-        >>> ds_text_class = review_ds.convert_to_text_classification('reviews')
-        """
-
-        def _select_features(df, feature_names, target):
-            if target in df.columns:
-                feature_names = feature_names + [target]
-            return df[feature_names]
-
-        transformer = (
-            f"convert_to_text_classification using feature {text_column}",
-            FunctionTransformer(
-                func=_select_features,
-                validate=False,
-                kw_args={
-                    "feature_names": [self.target.name, text_column],
-                    "target": self.target.name,
-                },
-            ).fit(self.sampled_df),
-        )
-        if utils.is_same_class(self, BinaryClassificationDataset):
-            new_ds = BinaryTextClassificationDataset(
-                self.df[[self.target.name, text_column]],
-                self.sampled_df[[self.target.name, text_column]],
-                self.target.name,
-                self.target.type,
-                (len(self.df), 2),
-                **self.init_kwargs,
-            )
-
-        else:
-            new_ds = MultiClassTextClassificationDataset(
-                self.df[[self.target.name, text_column]],
-                self.sampled_df[[self.target.name, text_column]],
-                self.target.name,
-                self.target.type,
-                (len(self.df), 2),
-                **self.init_kwargs,
-            )
-        new_ds.transformer_pipeline = self._update_transformer_pipeline(transformer)
-        return new_ds
-
-    def down_sample(self, sampler=None):
-        """
-        Fixes an imbalanced dataset by down-sampling.
-
-        Parameters
-        ----------
-        sampler: An instance of SamplerMixin
-            Should implement fit_resample(X,y) method. If None, does random down sampling.
-
-        Returns
-        -------
-        down_sampled_ds: ClassificationDataset
-            A down-sampled dataset.
-
-        Examples
-        --------
-        >>> ds = DatasetFactory.open("some_data.csv")
-        >>> ds_balanced_small = ds.down_sample()
-        """
-        return self._build_new_dataset(
-            helper.down_sample(self.df, self.target.name)
-            if sampler is None
-            else helper.sample(
-                sampler,
-                self.df.drop(self.target.name, axis=1),
-                self.df[self.target.name],
-            )
-        )
-
-    def up_sample(self, sampler="default"):
-        """
-        Fixes imbalanced dataset by up-sampling
-
-        Parameters
-        ----------
-        sampler: An instance of SamplerMixin
-            Should implement fit_resample(X,y) method.
-            If 'default', either SMOTE or random sampler will be used
-        fill_missing_type: a string
-            Can either be 'mean', 'mode' or 'median'.
-
-        Returns
-        -------
-        up_sampled_ds: ClassificationDataset
-            an up-sampled dataset
-
-        Examples
-        --------
-        >>> ds = DatasetFactory.open("some_data.csv")
-        >>> ds_balanced_large = ds.up_sample()
-        """
-        return self._build_new_dataset(
-            helper.up_sample(
-                self.df,
-                self.target.name,
-                sampler=sampler,
-                feature_types=self.feature_types,
-            )
-        )
-
-
-class BinaryClassificationDataset(ClassificationDataset):
-    """
-    Dataset for binary classification
-    """
-
-    def __init__(
-        self, df, sampled_df, target, target_type, shape, positive_class=None, **kwargs
-    ):
-        if positive_class is not None:
-            # map positive_class to True
-            update_arg = lambda x: x == positive_class
-
-            def mapper(df, column_name, arg):
-                df[column_name] = df[column_name].map(arg)
-                return df
-
-            df = mapper(df, target, update_arg)
-            sampled_df = mapper(sampled_df, target, update_arg)
-        ClassificationDataset.__init__(
-            self, df, sampled_df, target, target_type, shape, **kwargs
-        )
-
-    def set_positive_class(self, positive_class, missing_value=False):
-        """
-        Return new dataset with values in target column mapped to True or False
-        in accordance with the specified positive label.
-
-        Parameters
-        ----------
-        positive_class : same dtype as target
-            The target label which should be identified as positive outcome from model.
-        missing_value : bool
-            missing values will be converted to this
-
-        Returns
-        -------
-        dataset: same type as the caller
-
-        Raises
-        ------
-        ValidationError
-             if the positive_class is not present in target
-
-        Examples
-        --------
-        >>> ds = DatasetFactory.open("iris.csv")
-        >>> ds_with_target = ds.set_target('class')
-        >>> ds_with_pos_class = ds.set_positive_class('setosa')
-        """
-        if positive_class not in self.target.target_vals:
-            raise ValidationError(
-                "Positive label '%s' not in target values '%s'"
-                % (positive_class, self.target.target_vals)
-            )
-
-        return self.assign_column(
-            self.target.name,
-            lambda x: pd.isnull(x) and missing_value or x == positive_class,
-        )
-
-
-class MultiClassClassificationDataset(ClassificationDataset):
-    """
-    Dataset for multi-class classification
-    """
-
-    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
-        ClassificationDataset.__init__(
-            self, df, sampled_df, target, target_type, shape, **kwargs
-        )
-
-
-class BinaryTextClassificationDataset(BinaryClassificationDataset):
-    """
-    Dataset for binary text classification
-    """
-
-    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
-        BinaryClassificationDataset.__init__(
-            self, df, sampled_df, target, target_type, shape, **kwargs
-        )
-
-    def auto_transform(self):
-        """
-        Automatically chooses the most effective dataset transformation
-        """
-        logger.info("No optimizations.")
-        return self
-
-    def select_best_features(self, score_func=None, k=12):
-        """
-        Automatically chooses the best features and removes the rest
-        """
-        logger.info(
-            "There are an insufficient number of features to do feature selection."
-        )
-        return self
-
-
-class MultiClassTextClassificationDataset(MultiClassClassificationDataset):
-    """
-    Dataset for multi-class text classification
-    """
-
-    def __init__(self, df, sampled_df, target, target_type, shape, **kwargs):
-        MultiClassClassificationDataset.__init__(
-            self, df, sampled_df, target, target_type, shape, **kwargs
-        )
-
-    def auto_transform(self):
-        """
-        Automatically chooses the most effective dataset transformation
-        """
-        logger.info("No optimizations.")
-        return self
-
-    def select_best_features(self, score_func=None, k=12):
-        """
-        Automatically chooses the best features and removes the rest
-        """
-        logger.info(
-            "There are an insufficient number of features to do feature selection."
-        )
-        return self

ads/dataset/correlation.py

@@ -1,226 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import print_function, absolute_import
-
-import numpy as np
-import pandas as pd
-
-from ads.common import logger
-from ads.dataset.exception import ValidationError
-from collections import defaultdict, Counter
-from itertools import product, combinations
-from typing import Tuple
-
-
-def _cat_vs_cat(df: pd.core.frame.DataFrame, categorical_columns: list) -> pd.DataFrame:
-    """
-    calc the correlation of all pairs of categorical features and categorical features
-    """
-    if not categorical_columns:
-        return pd.DataFrame()
-    categorical_pairs = list(combinations(categorical_columns, 2))
-    corr_list = []
-    for col in categorical_pairs:
-        cat1_name = col[0]
-        cat2_name = col[1]
-        _check_if_same_type(df[col[0]], cat1_name)
-        _check_if_same_type(df[col[1]], cat2_name)
-        corr_list.append(
-            _cramers_v(np.array(df[col[0]].values), np.array(df[col[1]].values))
-        )
-    correlation_matrix = _list_to_dataframe(categorical_pairs, corr_list)
-    return correlation_matrix
-
-
-def _cat_vs_cts(
-    df: pd.core.frame.DataFrame, categorical_columns: list, continuous_columns: list
-) -> pd.DataFrame:
-    """
-    calc the correlation of all pairs of categorical features and continuous features
-    """
-    numerical_categorical_pairs = list(product(categorical_columns, continuous_columns))
-    corr_list = []
-    for col in numerical_categorical_pairs:
-        corr_list.append(
-            _correlation_ratio(np.array(df[col[0]].values), np.array(df[col[1]].values))
-        )
-    correlation_matrix = _list_to_dataframe(numerical_categorical_pairs, corr_list)
-    return correlation_matrix
-
-
-def _list_to_dataframe(name_list: list, corr_list: list) -> pd.DataFrame:
-    corr_dict = defaultdict(dict)
-    for idx, corr in zip(name_list, corr_list):
-        row_name = idx[0]
-        col_name = idx[1]
-        corr_dict[row_name][col_name] = corr_dict[col_name][row_name] = round(corr, 4)
-        corr_dict[row_name][row_name] = corr_dict[col_name][col_name] = 1.0000
-    correlation_matrix = pd.DataFrame.from_dict(corr_dict).sort_index()
-    correlation_matrix = correlation_matrix.loc[:, correlation_matrix.index]
-    return correlation_matrix
-
-
-def _correlation_ratio(cat: np.ndarray, cts: np.ndarray):
-    """
-    calc the correlation of a pair of a categorical feature and a continuous feature
-    using correlation ratio when input are two numpy arrays
-    """
-    keep_cts = ~pd.isnull(cts)
-    cat_no_nan = cat[keep_cts]
-    cts_no_nan = cts[keep_cts]
-
-    keep_cat = ~pd.isnull(cat_no_nan)
-    cat_no_none = cat_no_nan[keep_cat]
-    cts_no_none = cts_no_nan[keep_cat]
-
-    unq_cat, tags, group_count = np.unique(
-        list(cat_no_none), return_inverse=1, return_counts=1
-    )
-    group_mean = np.bincount(tags, cts_no_none) / group_count
-    overall_mean = np.nanmean(cts_no_none)
-    n = len(cts_no_none)
-
-    dispersion_within = np.dot(group_count, np.square(group_mean - overall_mean))
-    dispersion_population = cts_no_none.var() * n
-    ratio = dispersion_within / dispersion_population
-
-    return np.sqrt(ratio)
-
-
-def _count_occurrence(
-    cat1: np.ndarray, cat2: np.ndarray
-) -> Tuple[np.ndarray, int, int]:
-    """
-    calc the contingency table of two arrays
-    """
-    occurance_cnt = Counter([(x, y) for x, y in zip(cat1, cat2)])
-    nunique_cat1 = np.unique(cat1[~pd.isnull(cat1)])
-    nunique_cat2 = np.unique(cat2[~pd.isnull(cat2)])
-    r = len(nunique_cat1)
-    k = len(nunique_cat2)
-    contigency_table = np.zeros((r, k))
-    for row, num1 in enumerate(nunique_cat1):
-        for col, num2 in enumerate(nunique_cat2):
-            contigency_table[row, col] = occurance_cnt[(num1, num2)]
-
-    return contigency_table, r, k
-
-
-def _chi_squared(count_matrix: np.ndarray, n_obs: int) -> float:
-    """
-    Compute Chi-squared when given a contingency table
-    """
-    row_sums = np.tile(np.sum(count_matrix, axis=1), (count_matrix.shape[1], 1)).T
-    col_sums = np.tile(np.sum(count_matrix, axis=0), (count_matrix.shape[0], 1))
-    return np.sum(
-        np.square(count_matrix - row_sums * col_sums / n_obs)
-        / (row_sums * col_sums / n_obs)
-    )
-
-
-def _cramers_v(cat1: np.ndarray, cat2: np.ndarray) -> float:
-    """
-    calc the cramers v of two numpy arrays
-    """
-    n = len(cat1)
-    if n == 1:
-        return 0
-    contigency_table, r, k = _count_occurrence(cat1, cat2)
-
-    if r == 0:
-        return 0.0000
-
-    chi2 = _chi_squared(contigency_table, n)
-    phi2 = chi2 / n
-
-    phi2corr = max(0, phi2 - ((k - 1) * (r - 1)) / (n - 1))
-    rcorr = r - (np.square(r - 1)) / (n - 1)
-    kcorr = k - (np.square(k - 1)) / (n - 1)
-    denominator = min((kcorr - 1), (rcorr - 1))
-    if denominator == 0:
-        return np.nan
-    return np.sqrt(phi2corr / denominator)
-
-
-def _get_columns_by_type(
-    feature_types_df: pd.DataFrame, threshold: float = 0.8
-) -> Tuple[list, list, list]:
-    """
-    return the categorical columns, continuous columns and columns of other types
-    """
-    missing = feature_types_df.loc[:, "missing_percentage"] > threshold
-    not_missing = feature_types_df.loc[:, "missing_percentage"] <= threshold
-    missing_columns = list(feature_types_df.loc[missing, "feature_name"].values)
-    constant_columns = list(
-        feature_types_df.loc[
-            (feature_types_df.loc[:, "type"].isin(["constant"])), "feature_name"
-        ].values
-    )
-    categorical_columns = list(
-        feature_types_df.loc[
-            (feature_types_df.loc[:, "type"].isin(["categorical", "zipcode"]))
-            & not_missing,
-            "feature_name",
-        ].values
-    )
-    continuous_columns = list(
-        feature_types_df.loc[
-            (feature_types_df.loc[:, "type"].isin(["continuous", "ordinal"]))
-            & not_missing,
-            "feature_name",
-        ].values
-    )
-    other_columns = list(
-        set(feature_types_df.index.values)
-        - set(categorical_columns)
-        - set(continuous_columns)
-        - set(missing_columns)
-        - set(constant_columns)
-    )
-
-    if missing_columns:
-        logger.info(
-            f"The columns {missing_columns} are not included because more than {threshold}% of the values are missing. "
-        )
-        logger.info(
-            f"The columns {missing_columns} are not included because more than {threshold}% of the values are missing. "
-            f"Adjust this threshold using the `nan_threshold` parameter."
-        )
-    if constant_columns:
-        logger.info(
-            " The constant columns {} are not included.".format(constant_columns)
-        )
-    if other_columns:
-        logger.info(
-            f" The columns {other_columns} are not included because more than {threshold}% of the values are missing, or "
-            f"they are not one of the following types: "
-            f"`categorical`, `zipcode`, `continuous`"
-            f", or `ordinal`."
-        )
-
-    return categorical_columns, continuous_columns, other_columns
-
-
-def _validate_correlation_methods(correlation_methods):
-    if isinstance(correlation_methods, str):
-        correlation_methods = [correlation_methods]
-    for method in correlation_methods:
-        if method not in ["all", "pearson", "cramers v", "correlation ratio"]:
-            raise ValidationError(f"{method} is not supported.")
-    if "all" in [method for method in correlation_methods]:
-        correlation_methods = ["pearson", "cramers v", "correlation ratio"]
-    return correlation_methods
-
-
-def _check_if_same_type(series, col_name):
-    col = series.dropna().values
-    if len(col) > 0:
-        col_type = type(col[0])
-        if not all([isinstance(x, col_type) for x in col]):
-            raise TypeError(
-                f"More than one data type in the column `{col_name}`. Keep all the values in that column the same type."
-            )