fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/model/handler.py

@@ -0,0 +1,81 @@
+"""
+    Auxiliary handler methods for data summary extraction
+"""
+from typing import Any, Callable, Dict, List, Sequence
+
+import networkx as nx
+from visions import VisionsTypeset
+
+
+def compose(functions: Sequence[Callable]) -> Callable:
+    """
+    Compose a sequence of functions.
+
+    :param functions: sequence of functions
+    :return: combined function applying all functions in order.
+    """
+
+    def composed_function(*args) -> List[Any]:
+        result = args  # Start with the input arguments
+        for func in functions:
+            result = func(*result) if isinstance(result, tuple) else func(result)
+        return result  # type: ignore
+
+    return composed_function  # type: ignore
+
+
+class Handler:
+    """A generic handler
+
+    Allows any custom mapping between data types and functions
+    """
+
+    def __init__(
+        self,
+        mapping: Dict[str, List[Callable]],
+        typeset: VisionsTypeset,
+        *args,
+        **kwargs
+    ):
+        self.mapping = mapping
+        self.typeset = typeset
+        self._complete_dag()
+
+    def _complete_dag(self) -> None:
+        for from_type, to_type in nx.topological_sort(
+            nx.line_graph(self.typeset.base_graph)
+        ):
+            self.mapping[str(to_type)] = (
+                self.mapping[str(from_type)] + self.mapping[str(to_type)]
+            )
+
+    def handle(self, dtype: str, *args, **kwargs) -> dict:
+        """
+        Returns:
+            object: a tuple containing the config, the dataset series and the summary extracted
+        """
+        funcs = self.mapping.get(dtype, [])
+        op = compose(funcs)
+        summary = op(*args)[-1]
+        return summary
+
+
+def get_render_map() -> Dict[str, Callable]:
+    import data_profiling.report.structure.variables as render_algorithms
+
+    render_map = {
+        "Boolean": render_algorithms.render_boolean,
+        "Numeric": render_algorithms.render_real,
+        "Complex": render_algorithms.render_complex,
+        "Text": render_algorithms.render_text,
+        "DateTime": render_algorithms.render_date,
+        "Categorical": render_algorithms.render_categorical,
+        "URL": render_algorithms.render_url,
+        "Path": render_algorithms.render_path,
+        "File": render_algorithms.render_file,
+        "Image": render_algorithms.render_image,
+        "Unsupported": render_algorithms.render_generic,
+        "TimeSeries": render_algorithms.render_timeseries,
+    }
+
+    return render_map

data_profiling/model/missing.py

@@ -0,0 +1,146 @@
+import importlib
+import warnings
+from typing import Any, Callable, Dict, Optional, Sized
+
+import pandas as pd
+
+from data_profiling.config import Settings
+
+
+class MissingDataBackend:
+    """Helper class to select and cache the appropriate missing-data backend (Pandas or Spark)."""
+
+    def __init__(self, df: Sized):
+        """Determine backend once and store it for all missing-data computations."""
+        if isinstance(df, pd.DataFrame):
+            self.backend_module = "data_profiling.model.pandas.missing_pandas"
+        else:
+            self.backend_module = "data_profiling.model.spark.missing_spark"
+
+        self.module = importlib.import_module(self.backend_module)
+
+    def get_method(self, method_name: str) -> Callable:
+        """Retrieve the appropriate missing-data function from the backend module."""
+        try:
+            return getattr(self.module, method_name)
+        except AttributeError as ex:
+            raise AttributeError(
+                f"Missing-data function '{method_name}' is not available in {self.backend_module}."
+            ) from ex
+
+
+class MissingData:
+    _method_name: str = ""
+
+    def compute(
+        self, config: Settings, df: Sized, backend: MissingDataBackend
+    ) -> Optional[Sized]:
+        """Computes correlation using the correct backend (Pandas or Spark)."""
+        try:
+            method = backend.get_method(self._method_name)
+        except AttributeError as ex:
+            raise NotImplementedError() from ex
+        else:
+            return method(config, df)
+
+
+class MissingBar(MissingData):
+    _method_name = "missing_bar"
+
+
+class MissingMatrix(MissingData):
+    _method_name = "missing_matrix"
+
+
+class MissingHeatmap(MissingData):
+    _method_name = "missing_heatmap"
+
+
+def get_missing_active(config: Settings, table_stats: dict) -> Dict[str, Any]:
+    """
+
+    Args:
+        config: report Settings object
+        table_stats: The overall statistics for the DataFrame.
+
+    Returns:
+
+    """
+
+    missing_map = {
+        "bar": {
+            "min_missing": 0,
+            "name": "Count",
+            "caption": "A simple visualization of nullity by column.",
+            "function": MissingBar(),
+        },
+        "matrix": {
+            "min_missing": 0,
+            "name": "Matrix",
+            "caption": "Nullity matrix is a data-dense display which lets you quickly visually pick out patterns in data completion.",
+            "function": MissingMatrix(),
+        },
+        "heatmap": {
+            "min_missing": 2,
+            "name": "Heatmap",
+            "caption": "The correlation heatmap measures nullity correlation: how strongly the presence or absence of one variable affects the presence of another.",
+            "function": MissingHeatmap(),
+        },
+    }
+
+    missing_map = {
+        name: settings
+        for name, settings in missing_map.items()
+        if (
+            config.missing_diagrams[name]
+            and table_stats["n_vars_with_missing"] >= settings["min_missing"]
+        )
+        and (
+            name != "heatmap"
+            or (
+                table_stats["n_vars_with_missing"] - table_stats["n_vars_all_missing"]
+                >= settings["min_missing"]
+            )
+        )
+    }
+
+    return missing_map
+
+
+def get_missing_diagram(
+    config: Settings, df: pd.DataFrame, settings: Dict[str, Any]
+) -> Optional[Dict[str, Any]]:
+    """Gets the rendered diagrams for missing values.
+
+    Args:
+        config: report Settings object
+        df: The DataFrame on which to calculate the missing values.
+        settings: missing diagram name, caption and function
+
+    Returns:
+        A dictionary containing the base64 encoded plots for each diagram that is active in the config (matrix, bar, heatmap).
+    """
+    backend = MissingDataBackend(df)
+
+    missing_func = settings.get("function")
+    if missing_func is None:
+        return None  # No function defined, skip execution
+
+    try:
+        result = missing_func.compute(config, df, backend)
+    except ValueError as e:
+        warnings.warn(
+            f"""There was an attempt to generate the {settings['name']} missing values diagrams, but this failed.
+        To hide this warning, disable the calculation
+        (using `df.profile_report(missing_diagrams={{"{settings['name']}": False}}`)
+        If this is problematic for your use case, please report this as an issue:
+        https://github.com/Data-Centric-AI-Community/data-profiling/issues
+        (include the error message: '{e}')"""
+        )
+        return None
+    else:
+        return {
+            "name": settings["name"],
+            "caption": settings["caption"],
+            "matrix": result,
+        }

data_profiling/model/pairwise.py

@@ -0,0 +1,33 @@
+from typing import Any, List, Tuple
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.visualisation.plot import scatter_pairwise
+
+
+def get_scatter_tasks(
+    config: Settings, continuous_variables: list
+) -> List[Tuple[Any, Any]]:
+    if not config.interactions.continuous:
+        return []
+
+    targets = config.interactions.targets
+    if len(targets) == 0:
+        targets = continuous_variables
+
+    tasks = [(x, y) for y in continuous_variables for x in targets]
+    return tasks
+
+
+def get_scatter_plot(
+    config: Settings, df: pd.DataFrame, x: Any, y: Any, continuous_variables: list
+) -> str:
+    if x in continuous_variables:
+        if y == x:
+            df_temp = df[[x]].dropna()
+        else:
+            df_temp = df[[x, y]].dropna()
+        return scatter_pairwise(config, df_temp[x], df_temp[y], x, y)
+    else:
+        return ""

data_profiling/model/pandas/__init__.py

@@ -0,0 +1,55 @@
+import importlib
+
+# List of modules in the 'pandas' model that should be imported explicitly
+PANDAS_MODULES = [
+    "correlations_pandas",
+    "describe_generic_pandas",
+    "describe_boolean_pandas",
+    "describe_categorical_pandas",
+    "describe_url_pandas",
+    "describe_file_pandas",
+    "describe_text_pandas",
+    "describe_timeseries_pandas",
+    "describe_numeric_pandas",
+    "describe_path_pandas",
+    "describe_image_pandas",
+    "describe_date_pandas",
+    "describe_counts_pandas",
+    "duplicates_pandas",
+    "sample_pandas",
+    "table_pandas",
+    "timeseries_index_pandas",
+    "summary_pandas",
+]
+
+# Dynamically import and expose functions from modules
+for module_name in PANDAS_MODULES:
+    module = importlib.import_module(f"data_profiling.model.pandas.{module_name}")
+    globals().update(
+        {
+            name: getattr(module, name)
+            for name in dir(module)
+            if not name.startswith("_")
+        }
+    )
+
+# Explicitly list exposed names for clarity
+__all__ = [
+    "pandas_describe_generic",
+    "pandas_describe_boolean_1d",
+    "pandas_describe_categorical_1d",
+    "pandas_describe_url_1d",
+    "pandas_describe_file_1d",
+    "pandas_describe_text_1d",
+    "pandas_describe_timeseries_1d",
+    "pandas_describe_numeric_1d",
+    "pandas_describe_path_1d",
+    "pandas_describe_image_1d",
+    "pandas_describe_date_1d",
+    "pandas_describe_counts",
+    "pandas_get_duplicates",
+    "pandas_get_sample",
+    "pandas_get_table_stats",
+    "pandas_get_time_index_description",
+    "pandas_get_series_descriptions",
+]

data_profiling/model/pandas/correlations_pandas.py

@@ -0,0 +1,207 @@
+"""Correlations between variables."""
+import itertools
+import warnings
+from typing import Callable, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from data_profiling.config import Settings
+from data_profiling.model.pandas.discretize_pandas import (
+    DiscretizationType,
+    Discretizer,
+)
+
+
+def spearman_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    df_aux = df.select_dtypes(include="number").copy()
+    return df_aux.corr(method="spearman")
+
+
+def pearson_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    df_aux = df.select_dtypes(include="number").copy()
+    return df_aux.corr(method="pearson")
+
+
+def kendall_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    df_aux = df.select_dtypes(include="number").copy()
+    return df_aux.corr(method="kendall")
+
+
+def _cramers_corrected_stat(confusion_matrix: pd.DataFrame, correction: bool) -> float:
+    """Calculate the Cramer's V corrected stat for two variables.
+
+    Args:
+        confusion_matrix: Crosstab between two variables.
+        correction: Should the correction be applied?
+
+    Returns:
+        The Cramer's V corrected stat for the two variables.
+    """
+    # handles empty crosstab
+    if confusion_matrix.empty:
+        return 0
+
+    chi2 = stats.chi2_contingency(confusion_matrix, correction=correction)[0]
+    n = confusion_matrix.sum().sum()
+    phi2 = chi2 / n
+    r = confusion_matrix.shape[0]
+    k = confusion_matrix.shape[1] if len(confusion_matrix.shape) > 1 else 1
+
+    # Deal with NaNs later on
+    with np.errstate(divide="ignore", invalid="ignore"):
+        phi2corr = max(0.0, phi2 - ((k - 1.0) * (r - 1.0)) / (n - 1.0))
+        rcorr = r - ((r - 1.0) ** 2.0) / (n - 1.0)
+        kcorr = k - ((k - 1.0) ** 2.0) / (n - 1.0)
+        rkcorr = min((kcorr - 1.0), (rcorr - 1.0))
+        if rkcorr == 0.0:
+            corr = 1.0
+        else:
+            corr = np.sqrt(phi2corr / rkcorr)
+    return corr
+
+
+def _pairwise_spearman(col_1: pd.Series, col_2: pd.Series) -> float:
+    return col_1.corr(col_2, method="spearman")
+
+
+def _pairwise_cramers(col_1: pd.Series, col_2: pd.Series) -> float:
+    return _cramers_corrected_stat(pd.crosstab(col_1, col_2), correction=True)
+
+
+def cramers_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    threshold = config.categorical_maximum_correlation_distinct
+
+    # `index` and `columns` must not be a set since Pandas 1.5,
+    # so convert it to a list. The order of the list is arbitrary.
+    categoricals = list(
+        {
+            key
+            for key, value in summary.items()
+            if value["type"] in {"Categorical", "Boolean"}
+            and 1 < value["n_distinct"] <= threshold
+        }
+    )
+
+    if len(categoricals) <= 1:
+        return None
+
+    categoricals = sorted(categoricals)
+    matrix = np.zeros((len(categoricals), len(categoricals)))
+    np.fill_diagonal(matrix, 1.0)
+    correlation_matrix = pd.DataFrame(
+        matrix,
+        index=categoricals,
+        columns=categoricals,
+    )
+
+    for name1, name2 in itertools.combinations(categoricals, 2):
+        confusion_matrix = pd.crosstab(df[name1], df[name2])
+        if confusion_matrix.empty:
+            correlation_matrix.loc[name2, name1] = np.nan
+        else:
+            correlation_matrix.loc[name2, name1] = _cramers_corrected_stat(
+                confusion_matrix, correction=True
+            )
+        correlation_matrix.loc[name1, name2] = correlation_matrix.loc[name2, name1]
+    return correlation_matrix
+
+
+def phik_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    df_cols_dict = {i: list(df.columns).index(i) for i in df.columns}
+
+    intcols = {
+        key
+        for key, value in summary.items()
+        # DateTime currently excluded
+        # In some use cases, it makes sense to convert it to interval
+        # See https://github.com/KaveIO/PhiK/issues/7
+        if value["type"] == "Numeric" and 1 < value["n_distinct"]
+    }
+
+    selcols = {
+        key
+        for key, value in summary.items()
+        if value["type"] != "Unsupported"
+        and 1 < value["n_distinct"] <= config.categorical_maximum_correlation_distinct
+    }
+    selcols = selcols.union(intcols)
+    selected_cols = sorted(selcols, key=lambda i: df_cols_dict[i])
+
+    if len(selected_cols) <= 1:
+        return None
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from phik import phik_matrix
+
+        correlation = phik_matrix(df[selected_cols], interval_cols=list(intcols))
+
+    return correlation
+
+
+def auto_compute(
+    config: Settings, df: pd.DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    threshold = config.categorical_maximum_correlation_distinct
+    numerical_columns = [
+        key
+        for key, value in summary.items()
+        if value["type"] in {"Numeric", "TimeSeries"} and value["n_distinct"] > 1
+    ]
+    categorical_columns = [
+        key
+        for key, value in summary.items()
+        if value["type"] in {"Categorical", "Boolean"}
+        and 1 < value["n_distinct"] <= threshold
+    ]
+
+    if len(numerical_columns + categorical_columns) <= 1:
+        return None
+
+    df_discretized = Discretizer(
+        DiscretizationType.UNIFORM, n_bins=config.correlations["auto"].n_bins
+    ).discretize_dataframe(df)
+    columns_tested = sorted(numerical_columns + categorical_columns)
+
+    correlation_matrix = pd.DataFrame(
+        np.ones((len(columns_tested), len(columns_tested))),
+        index=columns_tested,
+        columns=columns_tested,
+    )
+    for col_1_name, col_2_name in itertools.combinations(columns_tested, 2):
+
+        method = (
+            _pairwise_spearman
+            if any(elem in categorical_columns for elem in [col_1_name, col_2_name])
+            is False
+            else _pairwise_cramers
+        )
+
+        def f(col_name: str, method: Callable) -> pd.Series:
+            return (
+                df_discretized
+                if col_name in numerical_columns and method is _pairwise_cramers
+                else df
+            )
+
+        score = method(
+            f(col_1_name, method)[col_1_name], f(col_2_name, method)[col_2_name]
+        )
+        (
+            correlation_matrix.loc[col_1_name, col_2_name],
+            correlation_matrix.loc[col_2_name, col_1_name],
+        ) = (score, score)
+
+    return correlation_matrix

data_profiling/model/pandas/dataframe_pandas.py

@@ -0,0 +1,26 @@
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.utils.dataframe import rename_index
+
+
+def pandas_preprocess(config: Settings, df: pd.DataFrame) -> pd.DataFrame:
+    """Preprocess the dataframe
+
+    - Appends the index to the dataframe when it contains information
+    - Rename the "index" column to "df_index", if exists
+    - Convert the DataFrame's columns to str
+
+    Args:
+        config: report Settings object
+        df: the pandas DataFrame
+
+    Returns:
+        The preprocessed DataFrame
+    """
+    # Rename reserved column names
+    df = rename_index(df)
+
+    # Ensure that columns are strings
+    df.columns = df.columns.map(str)
+    return df

data_profiling/model/pandas/describe_boolean_pandas.py

@@ -0,0 +1,43 @@
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.pandas.imbalance_pandas import column_imbalance_score
+from data_profiling.model.summary_algorithms import (
+    describe_boolean_1d,
+    series_hashable,
+)
+
+
+@describe_boolean_1d.register
+@series_hashable
+def pandas_describe_boolean_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a boolean series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    value_counts: pd.Series = summary["value_counts_without_nan"]
+    if not value_counts.empty:
+        summary.update({"top": value_counts.index[0], "freq": value_counts.iloc[0]})
+        summary["imbalance"] = column_imbalance_score(value_counts, len(value_counts))
+    else:
+        summary.update(
+            {
+                "top": np.nan,
+                "freq": 0,
+                "imbalance": 0,
+            }
+        )
+
+    return config, series, summary