fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/model/pandas/describe_image_pandas.py

@@ -0,0 +1,255 @@
+from functools import partial
+from pathlib import Path
+from typing import Optional, Tuple, Union
+
+import filetype
+import imagehash
+import pandas as pd
+from PIL import ExifTags, Image
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import (
+    describe_image_1d,
+    named_aggregate_summary,
+)
+
+
+def open_image(path: Path) -> Optional[Image.Image]:
+    """
+
+    Args:
+        path:
+
+    Returns:
+
+    """
+    try:
+        return Image.open(path)
+    except (OSError, AttributeError):
+        return None
+
+
+def is_image_truncated(image: Image) -> bool:
+    """Returns True if the path refers to a truncated image
+
+    Args:
+        image:
+
+    Returns:
+        True if the image is truncated
+    """
+    try:
+        image.load()
+    except (OSError, AttributeError):
+        return True
+    else:
+        return False
+
+
+def get_image_shape(image: Image) -> Optional[Tuple[int, int]]:
+    """
+
+    Args:
+        image:
+
+    Returns:
+
+    """
+    try:
+        return image.size
+    except (OSError, AttributeError):
+        return None
+
+
+def hash_image(image: Image) -> Optional[str]:
+    """
+
+    Args:
+        image:
+
+    Returns:
+
+    """
+    try:
+        return str(imagehash.phash(image))
+    except (OSError, AttributeError):
+        return None
+
+
+def decode_byte_exif(exif_val: Union[str, bytes]) -> str:
+    """Decode byte encodings
+
+    Args:
+        exif_val:
+
+    Returns:
+
+    """
+    if isinstance(exif_val, str):
+        return exif_val
+    else:
+        return exif_val.decode()
+
+
+def extract_exif(image: Image) -> dict:
+    """
+
+    Args:
+        image:
+
+    Returns:
+
+    """
+    try:
+        exif_data = image._getexif()
+        if exif_data is not None:
+            exif = {
+                ExifTags.TAGS[k]: decode_byte_exif(v)
+                for k, v in exif_data.items()
+                if k in ExifTags.TAGS
+            }
+        else:
+            exif = {}
+    except (AttributeError, OSError):
+        # Not all file types (e.g. .gif) have exif information.
+        exif = {}
+
+    return exif
+
+
+def path_is_image(p: Path) -> bool:
+    guess = filetype.guess(str(p))
+    return guess is not None and guess.mime.startswith("image/")
+
+
+def count_duplicate_hashes(image_descriptions: dict) -> int:
+    """
+
+    Args:
+        image_descriptions:
+
+    Returns:
+
+    """
+    counts = pd.Series(
+        [x["hash"] for x in image_descriptions if "hash" in x]
+    ).value_counts()
+    return counts.sum() - len(counts)
+
+
+def extract_exif_series(image_exifs: list) -> dict:
+    """
+
+    Args:
+        image_exifs:
+
+    Returns:
+
+    """
+    exif_keys = []
+    exif_values: dict = {}
+
+    for image_exif in image_exifs:
+        # Extract key
+        exif_keys.extend(list(image_exif.keys()))
+
+        # Extract values per key
+        for exif_key, exif_val in image_exif.items():
+            if exif_key not in exif_values:
+                exif_values[exif_key] = []
+
+            exif_values[exif_key].append(exif_val)
+
+    series = {"exif_keys": pd.Series(exif_keys, dtype=object).value_counts().to_dict()}
+
+    for k, v in exif_values.items():
+        series[k] = pd.Series(v).value_counts()
+
+    return series
+
+
+def extract_image_information(
+    path: Path, exif: bool = False, hash: bool = False
+) -> dict:
+    """Extracts all image information per file, as opening files is slow
+
+    Args:
+        path: Path to the image
+        exif: extract exif information
+        hash: calculate hash (for duplicate detection)
+
+    Returns:
+        A dict containing image information
+    """
+    information: dict = {}
+    image = open_image(path)
+    information["opened"] = image is not None
+    if image is not None:
+        information["truncated"] = is_image_truncated(image)
+        if not information["truncated"]:
+            information["size"] = image.size
+            if exif:
+                information["exif"] = extract_exif(image)
+            if hash:
+                information["hash"] = hash_image(image)
+
+    return information
+
+
+def image_summary(series: pd.Series, exif: bool = False, hash: bool = False) -> dict:
+    """
+
+    Args:
+        series: series to summarize
+        exif: extract exif information
+        hash: calculate hash (for duplicate detection)
+
+    Returns:
+
+    """
+
+    image_information = series.apply(
+        partial(extract_image_information, exif=exif, hash=hash)
+    )
+    summary = {
+        "n_truncated": sum(
+            1 for x in image_information if "truncated" in x and x["truncated"]
+        ),
+        "image_dimensions": pd.Series(
+            [x["size"] for x in image_information if "size" in x],
+            name="image_dimensions",
+        ),
+    }
+
+    image_widths = summary["image_dimensions"].map(lambda x: x[0])
+    summary.update(named_aggregate_summary(image_widths, "width"))
+    image_heights = summary["image_dimensions"].map(lambda x: x[1])
+    summary.update(named_aggregate_summary(image_heights, "height"))
+    image_areas = image_widths * image_heights
+    summary.update(named_aggregate_summary(image_areas, "area"))
+
+    if hash:
+        summary["n_duplicate_hash"] = count_duplicate_hashes(image_information)
+
+    if exif:
+        exif_series = extract_exif_series(
+            [x["exif"] for x in image_information if "exif" in x]
+        )
+        summary["exif_keys_counts"] = exif_series["exif_keys"]
+        summary["exif_data"] = exif_series
+
+    return summary
+
+
+@describe_image_1d.register
+def pandas_describe_image_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    if series.hasnans:
+        raise ValueError("May not contain NaNs")
+    if not hasattr(series, "str"):
+        raise ValueError("series should have .str accessor")
+
+    summary.update(image_summary(series, config.vars.image.exif))
+
+    return config, series, summary

data_profiling/model/pandas/describe_numeric_pandas.py

@@ -0,0 +1,175 @@
+from typing import Any, Dict, Tuple
+
+import numpy as np
+import pandas as pd
+
+from data_profiling.utils.compat import pandas_version_info
+
+if pandas_version_info() >= (1, 5):
+    from pandas.core.arrays.integer import IntegerDtype
+else:
+    from pandas.core.arrays.integer import _IntegerDtype as IntegerDtype
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import (
+    chi_square,
+    describe_numeric_1d,
+    histogram_compute,
+    series_handle_nulls,
+    series_hashable,
+)
+
+
+def mad(arr: np.ndarray) -> np.ndarray:
+    """Median Absolute Deviation: a "Robust" version of standard deviation.
+    Indices variability of the sample.
+    https://en.wikipedia.org/wiki/Median_absolute_deviation
+    """
+    return np.median(np.abs(arr - np.median(arr)))
+
+
+def numeric_stats_pandas(series: pd.Series) -> Dict[str, Any]:
+    return {
+        "mean": series.mean(),
+        "std": series.std(),
+        "variance": series.var(),
+        "min": series.min(),
+        "max": series.max(),
+        # Unbiased kurtosis obtained using Fisher's definition (kurtosis of normal == 0.0). Normalized by N-1.
+        "kurtosis": series.kurt(),
+        # Unbiased skew normalized by N-1
+        "skewness": series.skew(),
+        "sum": series.sum(),
+    }
+
+
+def numeric_stats_numpy(
+    present_values: np.ndarray, series: pd.Series, series_description: Dict[str, Any]
+) -> Dict[str, Any]:
+    vc = series_description["value_counts_without_nan"]
+    index_values = vc.index.values
+
+    # FIXME: can be performance optimized by using weights in std, var, kurt and skew...
+    if len(index_values):
+        return {
+            "mean": np.average(index_values, weights=vc.values),
+            "std": np.std(present_values, ddof=1),
+            "variance": np.var(present_values, ddof=1),
+            "min": np.min(index_values),
+            "max": np.max(index_values),
+            # Unbiased kurtosis obtained using Fisher's definition (kurtosis of normal == 0.0). Normalized by N-1.
+            "kurtosis": series.kurt(),
+            # Unbiased skew normalized by N-1
+            "skewness": series.skew(),
+            "sum": np.dot(index_values, vc.values),
+        }
+    else:  # Empty numerical series
+        return {
+            "mean": np.nan,
+            "std": 0.0,
+            "variance": 0.0,
+            "min": np.nan,
+            "max": np.nan,
+            "kurtosis": 0.0,
+            "skewness": 0.0,
+            "sum": 0,
+        }
+
+
+@describe_numeric_1d.register
+@series_hashable
+@series_handle_nulls
+def pandas_describe_numeric_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a numeric series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    chi_squared_threshold = config.vars.num.chi_squared_threshold
+    quantiles = config.vars.num.quantiles
+
+    value_counts = summary["value_counts_without_nan"]
+
+    negative_index = value_counts.index < 0
+    summary["n_negative"] = value_counts.loc[negative_index].sum()
+    summary["p_negative"] = summary["n_negative"] / summary["n"]
+
+    infinity_values = [np.inf, -np.inf]
+    infinity_index = value_counts.index.isin(infinity_values)
+    summary["n_infinite"] = value_counts.loc[infinity_index].sum()
+
+    summary["n_zeros"] = 0
+    if 0 in value_counts.index:
+        summary["n_zeros"] = value_counts.loc[0]
+
+    stats = summary
+
+    if isinstance(series.dtype, IntegerDtype):
+        stats.update(numeric_stats_pandas(series))
+        present_values = series.astype(str(series.dtype).lower())
+        finite_values = present_values
+    else:
+        present_values = series.values
+        finite_values = present_values[np.isfinite(present_values)]
+        stats.update(numeric_stats_numpy(present_values, series, summary))
+
+    stats.update(
+        {
+            "mad": mad(present_values),
+        }
+    )
+
+    if chi_squared_threshold > 0.0:
+        stats["chi_squared"] = chi_square(finite_values)
+
+    stats["range"] = stats["max"] - stats["min"]
+    stats.update(
+        {
+            f"{percentile:.0%}": value
+            for percentile, value in series.quantile(quantiles).to_dict().items()
+        }
+    )
+    stats["iqr"] = stats["75%"] - stats["25%"]
+    stats["cv"] = stats["std"] / stats["mean"] if stats["mean"] else np.nan
+    stats["p_zeros"] = stats["n_zeros"] / summary["n"]
+    stats["p_infinite"] = summary["n_infinite"] / summary["n"]
+
+    stats["monotonic_increase"] = series.is_monotonic_increasing
+    stats["monotonic_decrease"] = series.is_monotonic_decreasing
+
+    stats["monotonic_increase_strict"] = (
+        stats["monotonic_increase"] and series.is_unique
+    )
+    stats["monotonic_decrease_strict"] = (
+        stats["monotonic_decrease"] and series.is_unique
+    )
+    if summary["monotonic_increase_strict"]:
+        stats["monotonic"] = 2
+    elif summary["monotonic_decrease_strict"]:
+        stats["monotonic"] = -2
+    elif summary["monotonic_increase"]:
+        stats["monotonic"] = 1
+    elif summary["monotonic_decrease"]:
+        stats["monotonic"] = -1
+    else:
+        stats["monotonic"] = 0
+
+    if len(value_counts[~infinity_index].index.values) > 0:
+        stats.update(
+            histogram_compute(
+                config,
+                value_counts[~infinity_index].index.values,
+                summary["n_distinct"],
+                weights=value_counts[~infinity_index].values,
+            )
+        )
+
+    return config, series, stats

data_profiling/model/pandas/describe_path_pandas.py

@@ -0,0 +1,63 @@
+import os
+from typing import Tuple
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import describe_path_1d
+
+
+def path_summary(series: pd.Series) -> dict:
+    """
+
+    Args:
+        series: series to summarize
+
+    Returns:
+
+    """
+
+    # TODO: optimize using value counts
+    summary = {
+        "common_prefix": os.path.commonprefix(series.values.tolist())
+        or "No common prefix",
+        "stem_counts": series.map(lambda x: os.path.splitext(x)[0]).value_counts(),
+        "suffix_counts": series.map(lambda x: os.path.splitext(x)[1]).value_counts(),
+        "name_counts": series.map(lambda x: os.path.basename(x)).value_counts(),
+        "parent_counts": series.map(lambda x: os.path.dirname(x)).value_counts(),
+        "anchor_counts": series.map(lambda x: os.path.splitdrive(x)[0]).value_counts(),
+    }
+
+    summary["n_stem_unique"] = len(summary["stem_counts"])
+    summary["n_suffix_unique"] = len(summary["suffix_counts"])
+    summary["n_name_unique"] = len(summary["name_counts"])
+    summary["n_parent_unique"] = len(summary["parent_counts"])
+    summary["n_anchor_unique"] = len(summary["anchor_counts"])
+
+    return summary
+
+
+@describe_path_1d.register
+def pandas_describe_path_1d(
+    config: Settings, series: pd.Series, summary: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a path series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    # Make sure we deal with strings (Issue #100)
+    if series.hasnans:
+        raise ValueError("May not contain NaNs")
+    if not hasattr(series, "str"):
+        raise ValueError("series should have .str accessor")
+
+    summary.update(path_summary(series))
+
+    return config, series, summary

data_profiling/model/pandas/describe_supported_pandas.py

@@ -0,0 +1,41 @@
+from typing import Tuple
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.summary_algorithms import describe_supported, series_hashable
+
+
+@describe_supported.register
+@series_hashable
+def pandas_describe_supported(
+    config: Settings, series: pd.Series, series_description: dict
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe a supported series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        series_description: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    # number of non-NaN observations in the Series
+    count = series_description["count"]
+
+    value_counts = series_description["value_counts_without_nan"]
+    distinct_count = len(value_counts)
+    unique_count = value_counts.where(value_counts == 1).count()
+
+    stats = {
+        "n_distinct": distinct_count,
+        "p_distinct": distinct_count / count if count > 0 else 0,
+        "is_unique": unique_count == count and count > 0,
+        "n_unique": unique_count,
+        "p_unique": unique_count / count if count > 0 else 0,
+    }
+    stats.update(series_description)
+
+    return config, series, stats

data_profiling/model/pandas/describe_text_pandas.py

@@ -0,0 +1,62 @@
+from typing import Tuple
+
+import pandas as pd
+
+from data_profiling.config import Settings
+from data_profiling.model.pandas.describe_categorical_pandas import (
+    length_summary_vc,
+    unicode_summary_vc,
+    word_summary_vc,
+)
+from data_profiling.model.summary_algorithms import (
+    histogram_compute,
+    series_handle_nulls,
+    series_hashable,
+)
+
+
+@series_hashable
+@series_handle_nulls
+def pandas_describe_text_1d(
+    config: Settings,
+    series: pd.Series,
+    summary: dict,
+) -> Tuple[Settings, pd.Series, dict]:
+    """Describe string series.
+
+    Args:
+        config: report Settings object
+        series: The Series to describe.
+        summary: The dict containing the series description so far.
+
+    Returns:
+        A dict containing calculated series description values.
+    """
+
+    series = series.astype(str)
+
+    # Only run if at least 1 non-missing value
+    value_counts = summary["value_counts_without_nan"]
+    value_counts.index = value_counts.index.astype(str)
+
+    summary.update({"first_rows": series.head(5)})
+
+    if config.vars.text.length:
+        summary.update(length_summary_vc(value_counts))
+        summary.update(
+            histogram_compute(
+                config,
+                summary["length_histogram"].index.values,
+                len(summary["length_histogram"]),
+                name="histogram_length",
+                weights=summary["length_histogram"].values,
+            )
+        )
+
+    if config.vars.text.characters:
+        summary.update(unicode_summary_vc(value_counts))
+
+    if config.vars.text.words:
+        summary.update(word_summary_vc(value_counts, config.vars.cat.stop_words))
+
+    return config, series, summary