fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/model/pandas/timeseries_index_pandas.py

@@ -0,0 +1,33 @@
+"""Compute statistical description of datasets."""
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_numeric_dtype
+
+from data_profiling.config import Settings
+from data_profiling.model.timeseries_index import get_time_index_description
+
+
+@get_time_index_description.register
+def pandas_get_time_index_description(
+    config: Settings,
+    df: pd.DataFrame,
+    table_stats: dict,
+) -> dict:
+    if not (is_numeric_dtype(df.index) or isinstance(df.index, pd.DatetimeIndex)):
+        return {}
+
+    n_series = table_stats["types"].get("TimeSeries", 0)
+    length = table_stats["n"]
+    start = df.index.min()
+    end = df.index.max()
+    period = abs(np.diff(df.index)).mean()
+    if isinstance(df.index, pd.DatetimeIndex):
+        period = pd.Timedelta(period)
+
+    return {
+        "n_series": n_series,
+        "length": length,
+        "start": start,
+        "end": end,
+        "period": period,
+    }

data_profiling/model/pandas/utils_pandas.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+
+def weighted_median(data: np.ndarray, weights: np.ndarray) -> int:
+    """
+    Args:
+      data (list or numpy.array): data
+      weights (list or numpy.array): weights
+    """
+    if not isinstance(data, np.ndarray):
+        data = np.array(data)
+    if not isinstance(weights, np.ndarray):
+        weights = np.array(weights)
+
+    s_data, s_weights = map(np.sort, [data, weights])
+    midpoint = 0.5 * np.sum(s_weights)
+
+    if s_weights[-1] > midpoint:
+        w_median = data[weights == np.max(weights)][0]
+    else:
+        cs_weights = np.cumsum(s_weights)
+        idx = np.where(cs_weights <= midpoint)[0][-1]
+        if cs_weights[idx] == midpoint:
+            w_median = np.mean(s_data[idx : idx + 2])
+        else:
+            w_median = s_data[idx + 1]
+    return w_median

data_profiling/model/sample.py

@@ -0,0 +1,37 @@
+from typing import List, Optional, TypeVar
+
+from multimethod import multimethod
+from pydantic.v1 import BaseModel
+
+from data_profiling.config import Settings
+
+T = TypeVar("T")  # type: ignore
+
+
+class Sample(BaseModel):
+    id: str
+    data: T  # type: ignore
+    name: str
+    caption: Optional[str] = None
+
+
+@multimethod
+def get_sample(config: Settings, df: T) -> List[Sample]:
+    raise NotImplementedError()
+
+
+def get_custom_sample(sample: dict) -> List[Sample]:
+    if "name" not in sample:
+        sample["name"] = None
+    if "caption" not in sample:
+        sample["caption"] = None
+
+    samples = [
+        Sample(
+            id="custom",
+            data=sample["data"],
+            name=sample["name"],
+            caption=sample["caption"],
+        )
+    ]
+    return samples

data_profiling/model/spark/__init__.py

@@ -0,0 +1,48 @@
+import importlib
+
+# Dynamically import all modules inside the Spark folder
+SPARK_MODULES = [
+    "correlations_spark",
+    "dataframe_spark",
+    "describe_boolean_spark",
+    "describe_categorical_spark",
+    "describe_counts_spark",
+    "describe_date_spark",
+    "describe_generic_spark",
+    "describe_numeric_spark",
+    "describe_supported_spark",
+    "duplicates_spark",
+    "missing_spark",
+    "sample_spark",
+    "summary_spark",
+    "table_spark",
+    "timeseries_index_spark",
+    "describe_text_spark",
+]
+
+# Load modules dynamically
+for module_name in SPARK_MODULES:
+    module = importlib.import_module(f"data_profiling.model.spark.{module_name}")
+    globals().update(
+        {
+            name: getattr(module, name)
+            for name in dir(module)
+            if not name.startswith("_")
+        }
+    )  # type: ignore
+
+# Explicitly list all available functions
+__all__ = [
+    "describe_generic_spark",
+    "describe_boolean_1d_spark",
+    "describe_categorical_1d_spark",
+    "describe_text_1d_spark",
+    "describe_numeric_1d_spark",
+    "describe_date_1d_spark",
+    "describe_counts_spark",
+    "get_duplicates_spark",
+    "get_sample_spark",
+    "get_table_stats_spark",
+    "get_time_index_description_spark",
+    "get_series_descriptions_spark",
+]

data_profiling/model/spark/correlations_spark.py

@@ -0,0 +1,152 @@
+"""Correlations between variables."""
+from typing import Optional
+
+import pandas as pd
+import pyspark
+from packaging import version
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.stat import Correlation
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import PandasUDFType, lit, pandas_udf
+from pyspark.sql.types import ArrayType, DoubleType, StructField, StructType
+
+from data_profiling.config import Settings
+
+CORRELATION_PEARSON = "pearson"
+CORRELATION_SPEARMAN = "spearman"
+
+
+def spearman_compute(
+    config: Settings, df: DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    # Get the numerical cols for index and column names
+    # Spark only computes Spearman natively for the above dtypes
+    matrix, num_cols = _compute_corr_natively(
+        df, summary, corr_type=CORRELATION_SPEARMAN
+    )
+    return pd.DataFrame(matrix, index=num_cols, columns=num_cols)
+
+
+def pearson_compute(
+    config: Settings, df: DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+
+    # Get the numerical cols for index and column names
+    # Spark only computes Pearson natively for the above dtypes
+    matrix, num_cols = _compute_corr_natively(
+        df, summary, corr_type=CORRELATION_PEARSON
+    )
+    return pd.DataFrame(matrix, index=num_cols, columns=num_cols)
+
+
+def _compute_corr_natively(df: DataFrame, summary: dict, corr_type: str) -> ArrayType:
+    """
+    This function exists as pearson and spearman correlation computations have the
+    exact same workflow. The syntax is Correlation.corr(dataframe, method="pearson" OR "spearman"),
+    and Correlation is from pyspark.ml.stat
+    """
+    variables = {
+        column: {
+            "type": description["type"],
+            "all_missing": description.get("n", 0) - description.get("n_missing", 0)
+            == 0,
+        }
+        for column, description in summary.items()
+    }
+    interval_columns = [
+        column
+        for column, mapping in variables.items()
+        if mapping["type"] == "Numeric" and not mapping["all_missing"]
+    ]
+
+    if not interval_columns:
+        return [], interval_columns
+
+    df = df.select(*interval_columns)
+
+    # convert to vector column first
+    vector_col = "corr_features"
+
+    assembler_args = {"inputCols": df.columns, "outputCol": vector_col}
+
+    # As handleInvalid was only implemented in spark 2.4.0, we use it only if pyspark version >= 2.4.0
+    if version.parse(pyspark.__version__) >= version.parse("2.4.0"):
+        assembler_args["handleInvalid"] = "skip"
+
+    assembler = VectorAssembler(**assembler_args)
+    df_vector = assembler.transform(df).select(vector_col)
+
+    if df_vector.count() <= 1:
+        return [], interval_columns
+
+    # get correlation matrix
+    matrix = (
+        Correlation.corr(df_vector, vector_col, method=corr_type).head()[0].toArray()
+    )
+    return matrix, interval_columns
+
+
+def kendall_compute(
+    config: Settings, df: DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    raise NotImplementedError()
+
+
+def cramers_compute(
+    config: Settings, df: DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+    raise NotImplementedError()
+
+
+def phi_k_compute(
+    config: Settings, df: DataFrame, summary: dict
+) -> Optional[pd.DataFrame]:
+
+    threshold = config.categorical_maximum_correlation_distinct
+    intcols = {
+        key
+        for key, value in summary.items()
+        # DateTime currently excluded
+        # In some use cases, it makes sense to convert it to interval
+        # See https://github.com/KaveIO/PhiK/issues/7
+        if value["type"] == "Numeric" and 1 < value["n_distinct"]
+    }
+
+    supportedcols = {
+        key
+        for key, value in summary.items()
+        if value["type"] != "Unsupported" and 1 < value["n_distinct"] <= threshold
+    }
+    selcols = list(supportedcols.union(intcols))
+
+    if len(selcols) <= 1:
+        return None
+
+    # pandas mapped udf works only with a groupby, we force the groupby to operate on all columns at once
+    # by giving one value to all columns
+    groupby_df = df.select(selcols).withColumn("groupby", lit(1))
+
+    # generate output schema for pandas_udf
+    output_schema_components = []
+    for column in selcols:
+        output_schema_components.append(StructField(column, DoubleType(), True))
+    output_schema = StructType(output_schema_components)
+
+    # create the pandas grouped map function to do vectorized kendall within spark itself
+    @pandas_udf(output_schema, PandasUDFType.GROUPED_MAP)
+    def phik(pdf: pd.DataFrame) -> pd.DataFrame:
+        correlation = phik.phik_matrix(df=pdf, interval_cols=list(intcols))
+        return correlation
+
+    # return the appropriate dataframe (similar to pandas_df.corr results)
+    if len(groupby_df.head(1)) > 0:
+        # perform correlation in spark, and get the results back in pandas
+        df = pd.DataFrame(
+            groupby_df.groupby("groupby").apply(phik).toPandas().values,
+            columns=selcols,
+            index=selcols,
+        )
+    else:
+        df = pd.DataFrame()
+
+    return df

data_profiling/model/spark/dataframe_spark.py

@@ -0,0 +1,34 @@
+import warnings
+
+from pyspark.sql import DataFrame
+from pyspark.sql.types import MapType
+
+from data_profiling.config import Settings
+
+
+def spark_preprocess(config: Settings, df: DataFrame) -> DataFrame:
+    """Preprocess the Spark DataFrame by removing MapType columns.
+    - Excludes columns of type MapType
+    - Raises a warning if any MapType columns are removed
+
+    Args:
+        config: Report settings object
+        df: The Spark DataFrame
+
+    Returns:
+        The preprocessed DataFrame without MapType columns.
+    """
+    # Identify MapType columns
+    columns_to_remove = [
+        field.name for field in df.schema.fields if isinstance(field.dataType, MapType)
+    ]
+
+    if columns_to_remove:
+        warnings.warn(
+            f"""Spark DataFrame profiling does not handle MapTypes. Column(s) {', '.join(columns_to_remove)} will be ignored.
+            To fix this, consider converting your MapType into a StructType or extracting key-value pairs using `explode()`.
+            """
+        )
+        df = df.drop(*columns_to_remove)
+
+    return df  # Return unchanged DataFrame if no MapType columns exist

data_profiling/model/spark/describe_boolean_spark.py

@@ -0,0 +1,27 @@
+from typing import Tuple
+
+from pyspark.sql import DataFrame
+
+from data_profiling.config import Settings
+
+
+def describe_boolean_1d_spark(
+    config: Settings, df: DataFrame, summary: dict
+) -> Tuple[Settings, DataFrame, dict]:
+    """Describe a boolean series.
+
+    Args:
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    value_counts = summary["value_counts"]
+
+    # get the most common boolean value and its frequency
+    top = value_counts.first()
+    summary.update({"top": top[0], "freq": top[1]})
+
+    return config, df, summary

data_profiling/model/spark/describe_categorical_spark.py

@@ -0,0 +1,28 @@
+from typing import Tuple
+
+from pyspark.sql import DataFrame
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import describe_categorical_1d
+
+
+@describe_categorical_1d.register
+def describe_categorical_1d_spark(
+    config: Settings, df: DataFrame, summary: dict
+) -> Tuple[Settings, DataFrame, dict]:
+    """Describe a categorical series.
+
+    Args:
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    # FIXME: cat description
+    redact = config.vars.cat.redact
+    if not redact:
+        summary["first_rows"] = df.limit(5).toPandas().squeeze("columns")
+
+    return config, df, summary

data_profiling/model/spark/describe_counts_spark.py

@@ -0,0 +1,105 @@
+"""
+    Pyspark counts
+"""
+from typing import Tuple
+
+import pandas as pd
+from pyspark.sql import DataFrame
+from pyspark.sql import functions as F
+from pyspark.sql import types as T
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import describe_counts
+
+
+@describe_counts.register
+def describe_counts_spark(
+    config: Settings, series: DataFrame, summary: dict
+) -> Tuple[Settings, DataFrame, dict]:
+    """Counts the values in a series (with and without NaN, distinct).
+
+    Args:
+        config: Profiling settings.
+        series: Spark DataFrame column for which we want to calculate the values.
+        summary: Dictionary to store the summary results.
+
+    Returns:
+        Updated settings, input series, and summary dictionary.
+    """
+    # Cast Decimal Type s
+    if isinstance(series.schema.fields[0].dataType, T.DecimalType):
+        series = series.select(
+            F.col(series.columns[0]).cast(T.DoubleType()).alias(series.columns[0])
+        )
+
+    # Count occurrences of each value
+    value_counts = series.groupBy(series.columns[0]).count()
+
+    # Sort by count descending, persist the result
+    value_counts = value_counts.orderBy(F.desc("count")).persist()
+
+    # Sort by column value ascending (for frequency tables)
+    value_counts_index_sorted = value_counts.orderBy(F.asc(series.columns[0]))
+
+    # Count missing values
+    if series.dtypes[0][1] in ("int", "float", "bigint", "double"):
+        n_missing = (
+            # Need to add the isnan() check because Pandas isnull check will count NaN as null, but Spark does not
+            value_counts.filter(
+                F.col(series.columns[0]).isNull() | F.isnan(F.col(series.columns[0]))
+            )
+            .select("count")
+            .first()
+        )
+    else:
+        n_missing = (
+            # Need to add the isnan() check because Pandas isnull check will count NaN as null, but Spark does not
+            value_counts.filter(F.col(series.columns[0]).isNull())
+            .select("count")
+            .first()
+        )
+
+    n_missing = n_missing["count"] if n_missing else 0
+
+    # Convert top 200 values to Pandas for frequency table display
+    top_200_sorted = (
+        value_counts_index_sorted.limit(200)
+        .toPandas()
+        .set_index(series.columns[0], drop=True)
+        .squeeze(axis="columns")
+    )
+
+    summary["n_missing"] = n_missing
+    summary["value_counts"] = value_counts.persist()
+    summary["value_counts_index_sorted"] = top_200_sorted
+
+    column = series.columns[0]
+
+    if series.dtypes[0][1] in ("int", "float", "bigint", "double"):
+        value_counts_no_nan = (
+            value_counts.filter(F.col(column).isNotNull())  # Exclude NaNs
+            .filter(~F.isnan(F.col(column)))  # Remove implicit NaNs (if numeric column)
+            .groupBy(column)  # Group by unique values
+            .agg(F.sum("count").alias("count"))  # Sum of count
+            .orderBy(F.desc("count"))  # Sort in descending order
+        )
+    else:
+        value_counts_no_nan = (
+            value_counts.filter(F.col(column).isNotNull())  # Exclude NULLs
+            .groupBy(column)  # Group by unique timestamp values
+            .agg(F.sum("count").alias("count"))  # Sum of count
+            .orderBy(F.desc("count"))  # Sort by most frequent timestamps
+        )
+
+    # Convert to Pandas Series, forcing proper structure
+    if value_counts_no_nan.count() > 0:
+        pdf = value_counts_no_nan.toPandas().set_index(column)["count"]
+        summary["value_counts_without_nan"] = pd.Series(
+            pdf
+        )  # Ensures it's always a Series
+    else:
+        summary["value_counts_without_nan"] = pd.Series(
+            dtype=int
+        )  # Ensures an empty Series
+
+    return config, series, summary

data_profiling/model/spark/describe_date_spark.py

@@ -0,0 +1,51 @@
+from typing import Tuple
+
+import pyspark.sql.functions as F
+from numpy import array
+from pyspark.sql import DataFrame
+
+from data_profiling.config import Settings
+
+
+def date_stats_spark(df: DataFrame, summary: dict) -> dict:
+    column = df.columns[0]
+
+    expr = [
+        F.min(F.col(column)).alias("min"),
+        F.max(F.col(column)).alias("max"),
+    ]
+
+    return df.agg(*expr).first().asDict()
+
+
+def describe_date_1d_spark(
+    config: Settings, df: DataFrame, summary: dict
+) -> Tuple[Settings, DataFrame, dict]:
+    """Describe a date series.
+
+    Args:
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+    col_name = df.columns[0]
+    stats = date_stats_spark(df, summary)
+
+    summary.update({"min": stats["min"], "max": stats["max"]})
+
+    summary["range"] = summary["max"] - summary["min"]
+
+    # Convert date to numeric so we can compute the histogram
+    df = df.withColumn(col_name, F.unix_timestamp(df[col_name]))
+
+    # Get the number of bins
+    bins = config.plot.histogram.bins
+    bins_arg = "auto" if bins == 0 else min(bins, summary["n_distinct"])
+
+    # Run the histogram
+    bin_edges, hist = df.select(col_name).rdd.flatMap(lambda x: x).histogram(bins_arg)
+
+    summary.update({"histogram": (array(hist), array(bin_edges))})
+    return config, df, summary

data_profiling/model/spark/describe_generic_spark.py

@@ -0,0 +1,30 @@
+from typing import Tuple
+
+from pyspark.sql import DataFrame
+
+from data_profiling.config import Settings
+
+
+def describe_generic_spark(
+    config: Settings, df: DataFrame, summary: dict
+) -> Tuple[Settings, DataFrame, dict]:
+    """Describe generic series.
+    Args:
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    # number of observations in the Series
+    length = df.count()
+
+    summary["n"] = length
+    summary["p_missing"] = summary["n_missing"] / length
+    summary["count"] = length - summary["n_missing"]
+
+    # FIXME: This is not correct, but used to fulfil render expectations
+    # @chanedwin
+    summary["memory_size"] = 0
+
+    return config, df, summary