fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/model/correlations.py

@@ -0,0 +1,163 @@
+# mypy: ignore-errors
+
+"""Correlations between variables."""
+
+import warnings
+from typing import Dict, List, Optional, Sized, no_type_check
+
+import numpy as np
+import pandas as pd
+
+from data_profiling.config import Settings
+
+try:
+    from pandas.core.base import DataError
+except ImportError:
+    from pandas.errors import DataError
+
+
+class CorrelationBackend:
+    """Helper class to select and cache the appropriate correlation backend (Pandas or Spark)."""
+
+    @no_type_check
+    def __init__(self, df: Sized):
+        """Determine backend once and store it for all correlation computations."""
+        if isinstance(df, pd.DataFrame):
+            from data_profiling.model.pandas import (
+                correlations_pandas as correlation_backend,  # type: ignore
+            )
+        else:
+            from data_profiling.model.spark import (
+                correlations_spark as correlation_backend,  # type: ignore
+            )
+
+        self.backend = correlation_backend
+
+    def get_method(self, method_name: str):  # noqa: ANN201
+        """Retrieve the appropriate correlation method class from the backend."""
+        if hasattr(self.backend, method_name):
+            return getattr(self.backend, method_name)
+        raise AttributeError(
+            f"Correlation method '{method_name}' is not available in the backend."
+        )
+
+
+class Correlation:
+    _method_name: str = ""
+
+    def compute(
+        self, config: Settings, df: Sized, summary: dict, backend: CorrelationBackend
+    ) -> Optional[Sized]:
+        """Computes correlation using the correct backend (Pandas or Spark)."""
+        try:
+            method = backend.get_method(self._method_name)
+        except AttributeError as ex:
+            raise NotImplementedError() from ex
+        else:
+            return method(config, df, summary)
+
+
+class Auto(Correlation):
+    """Automatically selects the appropriate correlation method based on the DataFrame type."""
+
+    _method_name = "auto_compute"
+
+
+class Spearman(Correlation):
+    _method_name = "spearman_compute"
+
+
+class Pearson(Correlation):
+    _method_name = "pearson_compute"
+
+
+class Kendall(Correlation):
+    _method_name = "kendall_compute"
+
+
+class Cramers(Correlation):
+    _method_name = "cramers_compute"
+
+
+class PhiK(Correlation):
+    _method_name = "phik_compute"
+
+
+def warn_correlation(correlation_name: str, error: str) -> None:
+    warnings.warn(
+        f"""There was an attempt to calculate the {correlation_name} correlation, but this failed.
+To hide this warning, disable the calculation
+(using `df.profile_report(correlations={{\"{correlation_name}\": {{\"calculate\": False}}}})`
+If this is problematic for your use case, please report this as an issue:
+https://github.com/Data-Centric-AI-Community/data-profiling/issues
+(include the error message: '{error}')"""
+    )
+
+
+def calculate_correlation(
+    config: Settings, df: Sized, correlation_name: str, summary: dict
+) -> Optional[Sized]:
+    """Calculate the correlation coefficients between variables for the correlation types selected in the config
+    (auto, pearson, spearman, kendall, phi_k, cramers).
+
+    Args:
+        config: report Settings object
+        df: The DataFrame with variables.
+        correlation_name:
+        summary: summary dictionary
+
+    Returns:
+        The correlation matrices for the given correlation measures. Return None if correlation is empty.
+    """
+    backend = CorrelationBackend(df)
+
+    correlation_measures = {
+        "auto": Auto,
+        "pearson": Pearson,
+        "spearman": Spearman,
+        "kendall": Kendall,
+        "cramers": Cramers,
+        "phi_k": PhiK,
+    }
+
+    correlation = None
+    try:
+        correlation = correlation_measures[correlation_name]().compute(
+            config, df, summary, backend
+        )
+    except (ValueError, AssertionError, TypeError, DataError, IndexError) as e:
+        warn_correlation(correlation_name, str(e))
+
+    return correlation if correlation is not None and len(correlation) > 0 else None
+
+
+def perform_check_correlation(
+    correlation_matrix: pd.DataFrame, threshold: float
+) -> Dict[str, List[str]]:
+    """Check whether selected variables are highly correlated values in the correlation matrix.
+
+    Args:
+        correlation_matrix: The correlation matrix for the DataFrame.
+        threshold:.
+
+    Returns:
+        The variables that are highly correlated.
+    """
+
+    cols = correlation_matrix.columns
+    bool_index = abs(correlation_matrix.values) >= threshold
+    np.fill_diagonal(bool_index, False)
+    return {
+        col: cols[bool_index[i]].values.tolist()
+        for i, col in enumerate(cols)
+        if any(bool_index[i])
+    }
+
+
+def get_active_correlations(config: Settings) -> List[str]:
+    correlation_names = [
+        correlation_name
+        for correlation_name in config.correlations.keys()
+        if config.correlations[correlation_name].calculate
+    ]
+    return correlation_names

data_profiling/model/dataframe.py

@@ -0,0 +1,35 @@
+import importlib
+from typing import Any
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.pandas.dataframe_pandas import pandas_preprocess
+
+spec = importlib.util.find_spec("pyspark")
+if spec is None:
+    from typing import TypeVar
+
+    sparkDataFrame = TypeVar("sparkDataFrame")
+else:
+    from pyspark.sql import DataFrame as sparkDataFrame  # type: ignore
+
+    from data_profiling.model.spark.dataframe_spark import spark_preprocess
+
+
+def preprocess(config: Settings, df: Any) -> Any:
+    """
+    Search for invalid columns datatypes as well as ensures column names follow the expected rules
+    Args:
+        config: ydataprofiling Settings class
+        df: a pandas or spark dataframe
+
+    Returns: a pandas or spark dataframe
+    """
+    if isinstance(df, pd.DataFrame):
+        df = pandas_preprocess(config=config, df=df)
+    elif isinstance(df, sparkDataFrame):  # type: ignore
+        df = spark_preprocess(config=config, df=df)
+    else:
+        return NotImplementedError()
+    return df

data_profiling/model/describe.py

@@ -0,0 +1,210 @@
+"""Organize the calculation of statistics for each series in this DataFrame."""
+from datetime import datetime
+from typing import Any, Dict, Optional, Union
+
+import pandas as pd
+from tqdm.auto import tqdm
+from visions import VisionsTypeset
+
+from data_profiling.config import Settings
+from data_profiling.model import BaseAnalysis, BaseDescription
+from data_profiling.model.alerts import get_alerts
+from data_profiling.model.correlations import (
+    calculate_correlation,
+    get_active_correlations,
+)
+from data_profiling.model.dataframe import preprocess
+from data_profiling.model.description import TimeIndexAnalysis
+from data_profiling.model.duplicates import get_duplicates
+from data_profiling.model.missing import get_missing_active, get_missing_diagram
+from data_profiling.model.pairwise import get_scatter_plot, get_scatter_tasks
+from data_profiling.model.sample import get_custom_sample, get_sample
+from data_profiling.model.summarizer import BaseSummarizer
+from data_profiling.model.summary import get_series_descriptions
+from data_profiling.model.table import get_table_stats
+from data_profiling.model.timeseries_index import get_time_index_description
+from data_profiling.utils.progress_bar import progress
+from data_profiling.version import __version__
+
+
+def describe(
+    config: Settings,
+    df: Union[pd.DataFrame, "pyspark.sql.DataFrame"],  # type: ignore[name-defined] # noqa: F821
+    summarizer: BaseSummarizer,
+    typeset: VisionsTypeset,
+    sample: Optional[dict] = None,
+) -> BaseDescription:  # noqa: TC301
+    """Calculate the statistics for each series in this DataFrame.
+
+    Args:
+        config: report Settings object
+        df: DataFrame.
+        summarizer: summarizer object
+        typeset: visions typeset
+        sample: optional, dict with custom sample
+
+    Returns:
+        This function returns a dictionary containing:
+            - table: overall statistics.
+            - variables: descriptions per series.
+            - correlations: correlation matrices.
+            - missing: missing value diagrams.
+            - alerts: direct special attention to these patterns in your data.
+            - package: package details.
+    """
+    # ** Validate Input types **
+    if not isinstance(config, Settings):
+        raise TypeError(f"`config` must be of type `Settings`, got {type(config)}")
+
+    # Validate df input type
+
+    if not isinstance(df, pd.DataFrame):
+        try:
+            from pyspark.sql import DataFrame as SparkDataFrame  # type: ignore
+
+            if not isinstance(df, SparkDataFrame):  # noqa: TC301
+                raise TypeError(  # noqa: TC301
+                    f"`df` must be either a `pandas.DataFrame` or a `pyspark.sql.DataFrame`, but got {type(df)}."
+                )
+        except ImportError as ex:
+            raise TypeError(
+                f"`df must be either a `pandas.DataFrame` or a `pyspark.sql.DataFrame`, but got {type(df)}."
+                f"If using Spark, make sure PySpark is installed."
+            ) from ex
+
+    df = preprocess(config, df)
+
+    number_of_tasks = 5
+
+    with tqdm(
+        total=number_of_tasks,
+        desc="Summarize dataset",
+        disable=not config.progress_bar,
+        position=0,
+    ) as pbar:
+        date_start = datetime.utcnow()
+
+        # Variable-specific
+        pbar.total += len(df.columns)
+        series_description = get_series_descriptions(
+            config, df, summarizer, typeset, pbar
+        )
+
+        pbar.set_postfix_str("Get variable types")
+        pbar.total += 1
+        variables = {
+            column: description["type"]
+            for column, description in series_description.items()
+        }
+        supported_columns = [
+            column
+            for column, type_name in variables.items()
+            if type_name != "Unsupported"
+        ]
+        interval_columns = [
+            column
+            for column, type_name in variables.items()
+            if type_name in {"Numeric", "TimeSeries"}
+        ]
+        pbar.update()
+
+        # Table statistics
+        table_stats = progress(get_table_stats, pbar, "Get dataframe statistics")(
+            config, df, series_description
+        )
+
+        # Get correlations
+        if table_stats["n"] != 0:
+            correlation_names = get_active_correlations(config)
+            pbar.total += len(correlation_names)
+
+            correlations = {
+                correlation_name: progress(
+                    calculate_correlation,
+                    pbar,
+                    f"Calculate {correlation_name} correlation",
+                )(config, df, correlation_name, series_description)
+                for correlation_name in correlation_names
+            }
+
+            # make sure correlations is not None
+            correlations = {
+                key: value for key, value in correlations.items() if value is not None
+            }
+        else:
+            correlations = {}
+
+        # Scatter matrix
+        pbar.set_postfix_str("Get scatter matrix")
+        scatter_tasks = get_scatter_tasks(config, interval_columns)
+        pbar.total += len(scatter_tasks)
+        scatter_matrix: Dict[Any, Dict[Any, Any]] = {
+            x: {y: None} for x, y in scatter_tasks
+        }
+        for x, y in scatter_tasks:
+            scatter_matrix[x][y] = progress(
+                get_scatter_plot, pbar, f"scatter {x}, {y}"
+            )(config, df, x, y, interval_columns)
+
+        # missing diagrams
+        missing_map = get_missing_active(config, table_stats)
+        pbar.total += len(missing_map)
+        missing = {
+            name: progress(get_missing_diagram, pbar, f"Missing diagram {name}")(
+                config, df, settings
+            )
+            for name, settings in missing_map.items()
+        }
+        missing = {name: value for name, value in missing.items() if value is not None}
+
+        # Sample
+        pbar.set_postfix_str("Take sample")
+        if sample is None:
+            samples = get_sample(config, df)
+        else:
+            samples = get_custom_sample(sample)
+        pbar.update()
+
+        # Duplicates
+        metrics, duplicates = progress(get_duplicates, pbar, "Detecting duplicates")(
+            config, df, supported_columns
+        )
+        table_stats.update(metrics)
+
+        alerts = progress(get_alerts, pbar, "Get alerts")(
+            config, table_stats, series_description, correlations
+        )
+
+        if config.vars.timeseries.active:
+            tsindex_description = get_time_index_description(config, df, table_stats)
+
+        pbar.set_postfix_str("Get reproduction details")
+        package = {
+            "data_profiling_version": __version__,
+            "data_profiling_config": config.json(),
+        }
+        pbar.update()
+
+        pbar.set_postfix_str("Completed")
+
+        date_end = datetime.utcnow()
+
+    analysis = BaseAnalysis(config.title, date_start, date_end)
+    time_index_analysis = None
+    if config.vars.timeseries.active and tsindex_description:
+        time_index_analysis = TimeIndexAnalysis(**tsindex_description)
+
+    description = BaseDescription(
+        analysis=analysis,
+        time_index_analysis=time_index_analysis,
+        table=table_stats,
+        variables=series_description,
+        scatter=scatter_matrix,
+        correlations=correlations,
+        missing=missing,
+        alerts=alerts,
+        package=package,
+        sample=samples,
+        duplicates=duplicates,
+    )
+    return description

data_profiling/model/description.py

@@ -0,0 +1,108 @@
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Union
+
+from pandas import Timedelta
+
+
+@dataclass
+class BaseAnalysis:
+    """Description of base analysis module of report.
+    Overall info about report.
+
+    Attributes
+        title (str): Title of report.
+        date_start (Union[datetime, List[datetime]]): Start of generating description.
+        date_end (Union[datetime, List[datetime]]): End of generating description.
+    """
+
+    title: str
+    date_start: Union[datetime, List[datetime]]
+    date_end: Union[datetime, List[datetime]]
+
+    def __init__(self, title: str, date_start: datetime, date_end: datetime) -> None:
+        self.title = title
+        self.date_start = date_start
+        self.date_end = date_end
+
+    @property
+    def duration(self) -> Union[timedelta, List[timedelta]]:
+        if isinstance(self.date_start, datetime) and isinstance(
+            self.date_end, datetime
+        ):
+            return self.date_end - self.date_start
+        if isinstance(self.date_start, list) and isinstance(self.date_end, list):
+            return [
+                self.date_end[i] - self.date_start[i]
+                for i in range(len(self.date_start))
+            ]
+        else:
+            raise TypeError()
+
+
+@dataclass
+class TimeIndexAnalysis:
+    """Description of timeseries index analysis module of report.
+
+    Attributes:
+        n_series (Union[int, List[int]): Number of time series identified in the dataset.
+        length (Union[int, List[int]): Number of data points in the time series.
+        start (Any): Starting point of the time series.
+        end (Any): Ending point of the time series.
+        period (Union[float, List[float]): Average interval between data points in the time series.
+        frequency (Union[Optional[str], List[Optional[str]]): A string alias given to useful common time series frequencies, e.g. H - hours.
+    """
+
+    n_series: Union[int, List[int]]
+    length: Union[int, List[int]]
+    start: Any
+    end: Any
+    period: Union[float, List[float], Timedelta, List[Timedelta]]
+    frequency: Union[Optional[str], List[Optional[str]]]
+
+    def __init__(
+        self,
+        n_series: int,
+        length: int,
+        start: Any,
+        end: Any,
+        period: float,
+        frequency: Optional[str] = None,
+    ) -> None:
+        self.n_series = n_series
+        self.length = length
+        self.start = start
+        self.end = end
+        self.period = period
+        self.frequency = frequency
+
+
+@dataclass
+class BaseDescription:
+    """Description of DataFrame.
+
+    Attributes:
+        analysis (BaseAnalysis): Base info about report. Title, start time and end time of description generating.
+        time_index_analysis (Optional[TimeIndexAnalysis]): Description of timeseries index analysis module of report.
+        table (Any): DataFrame statistic. Base information about DataFrame.
+        variables (Dict[str, Any]): Description of variables (columns) of DataFrame. Key is column name, value is description dictionary.
+        scatter (Any): Pairwise scatter for all variables. Plot interactions between variables.
+        correlations (Dict[str, Any]): Prepare correlation matrix for DataFrame
+        missing (Dict[str, Any]): Describe missing values.
+        alerts (Any): Take alerts from all modules (variables, scatter, correlations), and group them.
+        package (Dict[str, Any]): Contains version of data-profiling and config.
+        sample (Any): Sample of data.
+        duplicates (Any): Description of duplicates.
+    """
+
+    analysis: BaseAnalysis
+    time_index_analysis: Optional[TimeIndexAnalysis]
+    table: Any
+    variables: Dict[str, Any]
+    scatter: Any
+    correlations: Dict[str, Any]
+    missing: Dict[str, Any]
+    alerts: Any
+    package: Dict[str, Any]
+    sample: Any
+    duplicates: Any

data_profiling/model/duplicates.py

@@ -0,0 +1,14 @@
+from typing import Any, Dict, Optional, Sequence, Tuple, TypeVar
+
+from multimethod import multimethod
+
+from data_profiling.config import Settings
+
+T = TypeVar("T")
+
+
+@multimethod
+def get_duplicates(
+    config: Settings, df: T, supported_columns: Sequence
+) -> Tuple[Dict[str, Any], Optional[T]]:
+    raise NotImplementedError()

data_profiling/model/expectation_algorithms.py

@@ -0,0 +1,112 @@
+from typing import Any, Tuple
+
+
+def generic_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    batch.expect_column_to_exist(name)
+
+    if summary["n_missing"] == 0:
+        batch.expect_column_values_to_not_be_null(name)
+
+    if summary["p_unique"] == 1.0:
+        batch.expect_column_values_to_be_unique(name)
+
+    return name, summary, batch
+
+
+def numeric_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    from great_expectations.profile.base import ProfilerTypeMapping
+
+    numeric_type_names = (
+        ProfilerTypeMapping.INT_TYPE_NAMES + ProfilerTypeMapping.FLOAT_TYPE_NAMES
+    )
+
+    batch.expect_column_values_to_be_in_type_list(
+        name,
+        numeric_type_names,
+        meta={
+            "notes": {
+                "format": "markdown",
+                "content": [
+                    "The column values should be stored in one of these types."
+                ],
+            }
+        },
+    )
+
+    if summary["monotonic_increase"]:
+        batch.expect_column_values_to_be_increasing(
+            name, strictly=summary["monotonic_increase_strict"]
+        )
+
+    if summary["monotonic_decrease"]:
+        batch.expect_column_values_to_be_decreasing(
+            name, strictly=summary["monotonic_decrease_strict"]
+        )
+
+    if any(k in summary for k in ["min", "max"]):
+        batch.expect_column_values_to_be_between(
+            name, min_value=summary.get("min"), max_value=summary.get("max")
+        )
+
+    return name, summary, batch
+
+
+def categorical_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    # Use for both categorical and special case (boolean)
+    absolute_threshold = 10
+    relative_threshold = 0.2
+    if (
+        summary["n_distinct"] < absolute_threshold
+        or summary["p_distinct"] < relative_threshold
+    ):
+        batch.expect_column_values_to_be_in_set(
+            name, set(summary["value_counts_without_nan"].keys())
+        )
+    return name, summary, batch
+
+
+def path_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    return name, summary, batch
+
+
+def datetime_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    if any(k in summary for k in ["min", "max"]):
+        batch.expect_column_values_to_be_between(
+            name,
+            min_value=summary.get("min"),
+            max_value=summary.get("max"),
+            parse_strings_as_datetimes=True,
+        )
+
+    return name, summary, batch
+
+
+def image_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    return name, summary, batch
+
+
+def url_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    return name, summary, batch
+
+
+def file_expectations(
+    name: str, summary: dict, batch: Any, *args
+) -> Tuple[str, dict, Any]:
+    # By definition within our type logic, a file exists (as it's a path that also exists)
+    batch.expect_file_to_exist(name)
+
+    return name, summary, batch