fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/model/pandas/describe_timeseries_pandas.py

@@ -0,0 +1,222 @@
+from typing import Any, Dict, Tuple
+
+import numpy as np
+import pandas as pd
+from scipy.fft import _pocketfft
+from scipy.signal import find_peaks
+from statsmodels.tsa.stattools import adfuller
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import (
+    describe_numeric_1d,
+    describe_timeseries_1d,
+    series_handle_nulls,
+    series_hashable,
+)
+
+
+def stationarity_test(config: Settings, series: pd.Series) -> Tuple[bool, float]:
+    # make sure the data has no missing values
+    adfuller_test = adfuller(
+        series.dropna(),
+        autolag=config.vars.timeseries.autolag,
+        maxlag=config.vars.timeseries.maxlag,
+    )
+    p_value = adfuller_test[1]
+
+    significance_threshold = config.vars.timeseries.significance
+    return p_value < significance_threshold, p_value
+
+
+def fftfreq(n: int, d: float = 1.0) -> np.ndarray:
+    """
+    Return the Discrete Fourier Transform sample frequencies.
+
+    Args:
+        n : int
+            Window length.
+        d : scalar, optional
+            Sample spacing (inverse of the sampling rate). Defaults to 1.
+
+    Returns:
+        f : ndarray
+            Array of length `n` containing the sample frequencies.
+    """
+    val = 1.0 / (n * d)
+    results = np.empty(n, int)
+    N = (n - 1) // 2 + 1
+    p1 = np.arange(0, N, dtype=int)
+    results[:N] = p1
+    p2 = np.arange(-(n // 2), 0, dtype=int)
+    results[N:] = p2
+    return results * val
+
+
+def seasonality_test(series: pd.Series, mad_threshold: float = 6.0) -> Dict[str, Any]:
+    """Detect seasonality with FFT
+
+    Source: https://github.com/facebookresearch/Kats/blob/main/kats/detectors/seasonality.py
+
+    Args:
+        mad_threshold: Optional; float; constant for the outlier algorithm for peak
+            detector. The larger the value the less sensitive the outlier algorithm
+            is.
+
+    Returns:
+        FFT Plot with peaks, selected peaks, and outlier boundary line.
+    """
+
+    fft = get_fft(series)
+    _, _, peaks = get_fft_peaks(fft, mad_threshold)
+    seasonality_presence = len(peaks.index) > 0
+    selected_seasonalities = []
+    if seasonality_presence:
+        selected_seasonalities = peaks["freq"].transform(lambda x: 1 / x).tolist()
+
+    return {
+        "seasonality_presence": seasonality_presence,
+        "seasonalities": selected_seasonalities,
+    }
+
+
+def get_fft(series: pd.Series) -> pd.DataFrame:
+    """Computes FFT
+
+    Args:
+        series: pd.Series
+            time series
+
+    Returns:
+        DataFrame with columns 'freq' and 'ampl'.
+    """
+    data_fft = _pocketfft.fft(series.to_numpy())
+    data_psd = np.abs(data_fft) ** 2
+    fftfreq_ = fftfreq(len(data_psd), 1.0)
+    pos_freq_ix = fftfreq_ > 0
+
+    freq = fftfreq_[pos_freq_ix]
+    ampl = 10 * np.log10(data_psd[pos_freq_ix])
+
+    return pd.DataFrame({"freq": freq, "ampl": ampl})
+
+
+def get_fft_peaks(
+    fft: pd.DataFrame, mad_threshold: float = 6.0
+) -> Tuple[float, pd.DataFrame, pd.DataFrame]:
+    """Computes peaks in fft, selects the highest peaks (outliers) and
+        removes the harmonics (multiplies of the base harmonics found)
+
+    Args:
+        fft: FFT computed by get_fft
+        mad_threshold: Optional; constant for the outlier algorithm for peak detector.
+            The larger the value the less sensitive the outlier algorithm is.
+
+    Returns:
+        outlier threshold, peaks, selected peaks.
+    """
+    pos_fft = fft.loc[fft["ampl"] > 0]
+    median = pos_fft["ampl"].median()
+    pos_fft_above_med = pos_fft[pos_fft["ampl"] > median]
+    mad = abs(pos_fft_above_med["ampl"] - pos_fft_above_med["ampl"].mean()).mean()
+
+    threshold = median + mad * mad_threshold
+
+    peak_indices = find_peaks(fft["ampl"], threshold=0.1)
+    peaks = fft.loc[peak_indices[0], :]
+
+    orig_peaks = peaks.copy()
+
+    peaks = peaks.loc[peaks["ampl"] > threshold].copy()
+    peaks["Remove"] = [False] * len(peaks.index)
+    peaks.reset_index(inplace=True)
+
+    # Filter out harmonics
+    for idx1 in range(len(peaks)):
+        curr = peaks.loc[idx1, "freq"]
+        for idx2 in range(idx1 + 1, len(peaks)):
+            if peaks.loc[idx2, "Remove"] is True:
+                continue
+            fraction = (peaks.loc[idx2, "freq"] / curr) % 1
+            if fraction < 0.01 or fraction > 0.99:
+                peaks.loc[idx2, "Remove"] = True
+    peaks = peaks.loc[~peaks["Remove"]]
+    peaks.drop(inplace=True, columns="Remove")
+    return threshold, orig_peaks, peaks
+
+
+def identify_gaps(
+    gap: pd.Series, is_datetime: bool, gap_tolerance: int = 2
+) -> Tuple[pd.Series, list]:
+    zero = pd.Timedelta(0) if is_datetime else 0
+    diff = gap.diff()
+
+    non_zero_diff = diff[diff > zero]
+    min_gap_size = gap_tolerance * non_zero_diff.mean()
+
+    gap_stats = non_zero_diff[non_zero_diff > min_gap_size]
+    anchors = gap[diff > min_gap_size].index
+
+    gaps = []
+    for i in anchors:
+        gaps.append(gap.loc[gap.index[[i - 1, i]]].values)
+
+    return gap_stats, gaps
+
+
+def compute_gap_stats(series: pd.Series) -> pd.Series:
+    """Computes the intertevals in the series normalized by the period.
+
+    Args:
+        series (pd.Series): time series data to analysis.
+
+    Returns:
+        A series with the gaps intervals.
+    """
+
+    gap = series.dropna()
+    index_name = gap.index.name if gap.index.name else "index"
+    gap = gap.reset_index()[index_name]
+    gap.index.name = None
+
+    is_datetime = isinstance(series.index, pd.DatetimeIndex)
+    gap_stats, gaps = identify_gaps(gap, is_datetime)
+    has_gaps = len(gap_stats) > 0
+
+    stats = {
+        "min": gap_stats.min() if has_gaps else 0,
+        "max": gap_stats.max() if has_gaps else 0,
+        "mean": gap_stats.mean() if has_gaps else 0,
+        "std": gap_stats.std() if len(gap_stats) > 1 else 0,
+        "series": series,
+        "gaps": gaps,
+        "n_gaps": len(gaps),
+    }
+    return stats
+
+
+@describe_timeseries_1d.register
+@series_hashable
+@series_handle_nulls
+def pandas_describe_timeseries_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a timeseries.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+    config, series, stats = describe_numeric_1d(config, series, summary)
+
+    stats["seasonal"] = seasonality_test(series)["seasonality_presence"]
+    is_stationary, p_value = stationarity_test(config, series)
+    stats["stationary"] = is_stationary and not stats["seasonal"]
+    stats["addfuller"] = p_value
+    stats["series"] = series
+    stats["gap_stats"] = compute_gap_stats(series)
+
+    return config, series, stats

data_profiling/model/pandas/describe_url_pandas.py

@@ -0,0 +1,57 @@
+from typing import Tuple
+from urllib.parse import urlsplit
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import describe_url_1d
+
+
+def url_summary(series: pd.Series) -> dict:
+    """
+
+    Args:
+        series: series to summarize
+
+    Returns:
+
+    """
+    summary = {
+        "scheme_counts": series.map(lambda x: x.scheme).value_counts(),
+        "netloc_counts": series.map(lambda x: x.netloc).value_counts(),
+        "path_counts": series.map(lambda x: x.path).value_counts(),
+        "query_counts": series.map(lambda x: x.query).value_counts(),
+        "fragment_counts": series.map(lambda x: x.fragment).value_counts(),
+    }
+
+    return summary
+
+
+@describe_url_1d.register
+def pandas_describe_url_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a url series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    # Make sure we deal with strings (Issue #100)
+    if series.hasnans:
+        raise ValueError("May not contain NaNs")
+    if not hasattr(series, "str"):
+        raise ValueError("series should have .str accessor")
+
+    # Transform
+    series = series.apply(urlsplit)
+
+    # Update
+    summary.update(url_summary(series))
+
+    return config, series, summary

data_profiling/model/pandas/discretize_pandas.py

@@ -0,0 +1,81 @@
+from enum import Enum
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+
+class DiscretizationType(Enum):
+    UNIFORM = "uniform"
+    QUANTILE = "quantile"
+
+
+class Discretizer:
+    """
+    A class which enables the discretization of a pandas dataframe.
+    Perform this action when you want to convert a continuous variable
+    into a categorical variable.
+
+    Attributes:
+
+    method (DiscretizationType): this attribute controls how the buckets
+    of your discretization are formed. A uniform discretization type forms
+    the bins to be of equal width whereas a quantile discretization type
+    forms the bins to be of equal size.
+
+    n_bins (int): number of bins
+    reset_index (bool): instruction to reset the index of
+                        the dataframe after the discretization
+    """
+
+    def __init__(
+        self, method: DiscretizationType, n_bins: int = 10, reset_index: bool = False
+    ) -> None:
+        self.discretization_type = method
+        self.n_bins = n_bins
+        self.reset_index = reset_index
+
+    def discretize_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        """_summary_
+
+        Args:
+            dataframe (pd.DataFrame): pandas dataframe
+
+        Returns:
+            pd.DataFrame: discretized dataframe
+        """
+
+        discretized_df = dataframe.copy()
+        all_columns = dataframe.columns
+        num_columns = self._get_numerical_columns(dataframe)
+        for column in num_columns:
+            discretized_df.loc[:, column] = self._discretize_column(
+                discretized_df[column]
+            )
+
+        discretized_df = discretized_df[all_columns]
+        return (
+            discretized_df.reset_index(drop=True)
+            if self.reset_index
+            else discretized_df
+        )
+
+    def _discretize_column(self, column: pd.Series) -> pd.Series:
+        if self.discretization_type == DiscretizationType.QUANTILE:
+            return self._descritize_quantile(column)
+
+        elif self.discretization_type == DiscretizationType.UNIFORM:
+            return self._descritize_uniform(column)
+
+    def _descritize_quantile(self, column: pd.Series) -> pd.Series:
+        return pd.qcut(
+            column, q=self.n_bins, labels=False, retbins=False, duplicates="drop"
+        ).values
+
+    def _descritize_uniform(self, column: pd.Series) -> pd.Series:
+        return pd.cut(
+            column, bins=self.n_bins, labels=False, retbins=True, duplicates="drop"
+        )[0].values
+
+    def _get_numerical_columns(self, dataframe: pd.DataFrame) -> List[str]:
+        return dataframe.select_dtypes(include=np.number).columns.tolist()

data_profiling/model/pandas/duplicates_pandas.py

@@ -0,0 +1,56 @@
+from typing import Any, Dict, Optional, Sequence, Tuple
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.duplicates import get_duplicates
+
+
+@get_duplicates.register(Settings, pd.DataFrame, Sequence)
+def pandas_get_duplicates(
+    config: Settings, df: pd.DataFrame, supported_columns: Sequence
+) -> Tuple[Dict[str, Any], Optional[pd.DataFrame]]:
+    """Obtain the most occurring duplicate rows in the DataFrame.
+
+    Args:
+        config: report Settings object
+        df: the Pandas DataFrame.
+        supported_columns: the columns to consider
+
+    Returns:
+        A subset of the DataFrame, ordered by occurrence.
+    """
+    n_head = config.duplicates.head
+
+    metrics: Dict[str, Any] = {}
+    if n_head > 0:
+        if supported_columns and len(df) > 0:
+            duplicates_key = config.duplicates.key
+            if duplicates_key in df.columns:
+                raise ValueError(
+                    f"Duplicates key ({duplicates_key}) may not be part of the DataFrame. Either change the "
+                    f" column name in the DataFrame or change the 'duplicates.key' parameter."
+                )
+
+            duplicated_rows = df.duplicated(subset=supported_columns, keep=False)
+            duplicated_rows = (
+                df[duplicated_rows]
+                .rename_axis(index=lambda _: None)
+                .groupby(supported_columns, dropna=False, observed=True)
+                .size()
+                .reset_index(name=duplicates_key)
+            )
+
+            metrics["n_duplicates"] = len(duplicated_rows[duplicates_key])
+            metrics["p_duplicates"] = metrics["n_duplicates"] / len(df)
+
+            return (
+                metrics,
+                duplicated_rows.nlargest(n_head, duplicates_key),
+            )
+        else:
+            metrics["n_duplicates"] = 0
+            metrics["p_duplicates"] = 0.0
+            return metrics, None
+    else:
+        return metrics, None

data_profiling/model/pandas/imbalance_pandas.py

@@ -0,0 +1,35 @@
+from typing import Union
+
+import pandas as pd
+from numpy import log2
+from scipy.stats import entropy
+
+
+def column_imbalance_score(
+    value_counts: pd.Series, n_classes: int
+) -> Union[float, int]:
+    """column_imbalance_score
+
+    The class balance score for categorical and boolean variables uses entropy to calculate a  bounded score between 0 and 1.
+    A perfectly uniform distribution would return a score of 0, and a perfectly imbalanced distribution would return a score of 1.
+
+    When dealing with probabilities with finite values (e.g categorical), entropy is maximised the ‘flatter’ the distribution is. (Jaynes: Probability Theory, The Logic of Science)
+    To calculate the class imbalance, we calculate the entropy of that distribution and the maximum possible entropy for that number of classes.
+    To calculate the entropy of the 'distribution' we use value counts (e.g frequency of classes) and we can determine the maximum entropy as log2(number of classes).
+    We then divide the entropy by the maximum possible entropy to get a value between 0 and 1 which we then subtract from 1.
+
+    Args:
+        value_counts (pd.Series): frequency of each category
+        n_classes (int): number of classes
+
+    Returns:
+        Union[float, int]: float or integer bounded between 0 and 1 inclusively
+    """
+    # return 0 if there is only one class (when entropy =0) as it is balanced.
+    # note that this also prevents a zero division error with log2(n_classes)
+    if n_classes > 1:
+        # casting to numpy array to ensure correct dtype when a categorical integer
+        # variable is evaluated
+        value_counts = value_counts.to_numpy(dtype=float)
+        return 1 - (entropy(value_counts, base=2) / log2(n_classes))
+    return 0

data_profiling/model/pandas/missing_pandas.py

@@ -0,0 +1,42 @@
+import numpy as np
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.visualisation.missing import (
+    plot_missing_bar,
+    plot_missing_heatmap,
+    plot_missing_matrix,
+)
+
+
+def missing_bar(config: Settings, df: pd.DataFrame) -> str:
+    notnull_counts = len(df) - df.isnull().sum()
+    return plot_missing_bar(
+        config,
+        notnull_counts=notnull_counts,
+        nrows=len(df),
+        columns=list(df.columns),
+    )
+
+
+def missing_matrix(config: Settings, df: pd.DataFrame) -> str:
+    return plot_missing_matrix(
+        config,
+        columns=list(df.columns),
+        notnull=df.notnull().values,
+        nrows=len(df),
+    )
+
+
+def missing_heatmap(config: Settings, df: pd.DataFrame) -> str:
+    # Remove completely filled or completely empty variables.
+    columns = [i for i, n in enumerate(np.var(df.isnull(), axis="rows")) if n > 0]
+    df = df.iloc[:, columns]
+
+    # Create and mask the correlation matrix. Construct the base heatmap.
+    corr_mat = df.isnull().corr()
+    mask = np.zeros_like(corr_mat)
+    mask[np.triu_indices_from(mask)] = True
+    return plot_missing_heatmap(
+        config, corr_mat=corr_mat, mask=mask, columns=list(df.columns)
+    )

data_profiling/model/pandas/sample_pandas.py

@@ -0,0 +1,38 @@
+from typing import List
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.sample import Sample, get_sample
+
+
+@get_sample.register(Settings, pd.DataFrame)
+def pandas_get_sample(config: Settings, df: pd.DataFrame) -> List[Sample]:
+    """Obtains a sample from head and tail of the DataFrame
+
+    Args:
+        config: Settings object
+        df: the pandas DataFrame
+
+    Returns:
+        a list of Sample objects
+    """
+    samples: List[Sample] = []
+    if len(df) == 0:
+        return samples
+
+    n_head = config.samples.head
+    if n_head > 0:
+        samples.append(Sample(id="head", data=df.head(n=n_head), name="First rows"))
+
+    n_tail = config.samples.tail
+    if n_tail > 0:
+        samples.append(Sample(id="tail", data=df.tail(n=n_tail), name="Last rows"))
+
+    n_random = config.samples.random
+    if n_random > 0:
+        samples.append(
+            Sample(id="random", data=df.sample(n=n_random), name="Random sample")
+        )
+
+    return samples

data_profiling/model/pandas/summary_pandas.py

@@ -0,0 +1,101 @@
+"""Compute statistical description of datasets."""
+import multiprocessing
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Tuple
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+from visions import VisionsTypeset
+
+from data_profiling.config import Settings
+from data_profiling.model.typeset import ProfilingTypeSet
+from data_profiling.utils.compat import optional_option_context
+from data_profiling.utils.dataframe import sort_column_names
+
+BaseSummarizer: Any = "BaseSummarizer"  # type: ignore
+
+
+def _is_cast_type_defined(typeset: VisionsTypeset, series: str) -> bool:
+    return isinstance(typeset, ProfilingTypeSet) and series in typeset.type_schema
+
+
+def pandas_describe_1d(
+    config: Settings,
+    series: pd.Series,
+    summarizer: BaseSummarizer,
+    typeset: VisionsTypeset,
+) -> dict:
+    """Describe a series (infer the variable type, then calculate type-specific values).
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summarizer: Summarizer object
+        typeset: Typeset
+
+    Returns:
+        A Series containing calculated series description values.
+    """
+
+    # Make sure pd.NA is not in the series
+    with optional_option_context("future.no_silent_downcasting", True):
+        series = series.fillna(np.nan).infer_objects(copy=False)
+
+    has_cast_type = _is_cast_type_defined(typeset, series.name)  # type:ignore
+    cast_type = (
+        str(typeset.type_schema[series.name]) if has_cast_type else None
+    )  # type:ignore
+
+    if has_cast_type and not series.isna().all():
+        vtype = typeset.type_schema[series.name]  # type:ignore
+
+    elif config.infer_dtypes:
+        # Infer variable types
+        vtype = typeset.infer_type(series)
+        series = typeset.cast_to_inferred(series)
+    else:
+        # Detect variable types from pandas dataframe (df.dtypes).
+        # [new dtypes, changed using `astype` function are now considered]
+        vtype = typeset.detect_type(series)
+
+    typeset.type_schema[series.name] = vtype  # type:ignore
+    summary = summarizer.summarize(config, series, dtype=vtype)
+    # Cast type is only used on unsupported columns rendering pipeline
+    # to indicate the correct variable type when inference is not possible
+    summary["cast_type"] = cast_type
+
+    return summary
+
+
+def pandas_get_series_descriptions(
+    config: Settings,
+    df: pd.DataFrame,
+    summarizer: BaseSummarizer,
+    typeset: VisionsTypeset,
+    pbar: tqdm,
+) -> dict:
+    def describe_column(name: str, series: pd.Series) -> Tuple[str, dict]:
+        """Process a single series to get the column description."""
+        pbar.set_postfix_str(f"Describe variable: {name}")
+        description = pandas_describe_1d(config, series, summarizer, typeset)
+        pbar.update()
+        return name, description
+
+    pool_size = (
+        config.pool_size if config.pool_size > 0 else multiprocessing.cpu_count()
+    )
+
+    series_description = {}
+
+    with ThreadPoolExecutor(max_workers=pool_size) as executor:
+        future_to_col = {
+            executor.submit(describe_column, name, series): name  # type:ignore
+            for name, series in df.items()
+        }
+
+        for future in tqdm(future_to_col.keys(), total=len(future_to_col)):
+            name, description = future.result()
+            series_description[name] = description
+
+    return sort_column_names(series_description, config.sort)

data_profiling/model/pandas/table_pandas.py

@@ -0,0 +1,56 @@
+from collections import Counter
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.table import get_table_stats
+
+
+@get_table_stats.register
+def pandas_get_table_stats(
+    config: Settings, df: pd.DataFrame, variable_stats: dict
+) -> dict:
+    """General statistics for the DataFrame.
+
+    Args:
+        config: report Settings object
+        df: The DataFrame to describe.
+        variable_stats: Previously calculated statistic on the DataFrame.
+
+    Returns:
+        A dictionary that contains the table statistics.
+    """
+    n = len(df) if not df.empty else 0
+
+    memory_size = df.memory_usage(deep=config.memory_deep).sum()
+    record_size = float(memory_size) / n if n > 0 else 0
+
+    table_stats = {
+        "n": n,
+        "n_var": len(df.columns),
+        "memory_size": memory_size,
+        "record_size": record_size,
+        "n_cells_missing": 0,
+        "n_vars_with_missing": 0,
+        "n_vars_all_missing": 0,
+    }
+
+    for series_summary in variable_stats.values():
+        if "n_missing" in series_summary and series_summary["n_missing"] > 0:
+            table_stats["n_vars_with_missing"] += 1
+            table_stats["n_cells_missing"] += series_summary["n_missing"]
+            if series_summary["n_missing"] == n:
+                table_stats["n_vars_all_missing"] += 1
+
+    table_stats["p_cells_missing"] = (
+        table_stats["n_cells_missing"] / (table_stats["n"] * table_stats["n_var"])
+        if table_stats["n"] > 0 and table_stats["n_var"] > 0
+        else 0
+    )
+
+    # Variable type counts
+    table_stats.update(
+        {"types": dict(Counter([v["type"] for v in variable_stats.values()]))}
+    )
+
+    return table_stats