fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/utils/cache.py

@@ -0,0 +1,59 @@
+"""Dataset cache utility functions"""
+import zipfile
+from pathlib import Path
+
+from requests import get as get_file
+
+from data_profiling.utils.paths import get_data_path
+
+
+def cache_file(file_name: str, url: str) -> Path:
+    """Check if file_name already is in the data path, otherwise download it from url.
+
+    Args:
+        file_name: the file name
+        url: the URL of the dataset
+
+    Returns:
+        The relative path to the dataset
+    """
+
+    data_path = get_data_path()
+    data_path.mkdir(exist_ok=True)
+
+    file_path = data_path / file_name
+
+    # If not exists, download and create file
+    if not file_path.exists():
+        response = get_file(url, allow_redirects=True)
+        response.raise_for_status()
+
+        file_path.write_bytes(response.content)
+
+    return file_path
+
+
+def cache_zipped_file(file_name: str, url: str) -> Path:
+    """Check if file_name already is in the data path, otherwise download it from url.
+
+    Args:
+        file_name: the file name
+        url: the URL of the dataset
+
+    Returns:
+        The relative path to the dataset
+    """
+
+    data_path = get_data_path()
+    file_path = data_path / file_name
+
+    # If not exists, download and create file
+    if not file_path.exists():
+        tmp_path = cache_file("tmp.zip", url)
+
+        with zipfile.ZipFile(tmp_path, "r") as zip_file:
+            zip_file.extract(file_path.name, data_path)
+
+        tmp_path.unlink()
+
+    return file_path

data_profiling/utils/common.py

@@ -0,0 +1,142 @@
+"""Common util functions (e.g. missing in Python)."""
+
+import collections.abc
+import contextlib
+import os
+import platform
+import subprocess
+import zipfile
+from datetime import datetime, timedelta
+
+# Image type detection
+from pathlib import Path
+from typing import Mapping
+
+import pandas as pd
+import requests
+
+from data_profiling.version import __version__
+
+
+def update(d: dict, u: Mapping) -> dict:
+    """Recursively update a dict.
+
+    Args:
+        d: Dictionary to update.
+        u: Dictionary with values to use.
+
+    Returns:
+        The merged dictionary.
+    """
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def _copy(self, target):
+    """Monkeypatch for pathlib
+
+    Args:
+        self:
+        target:
+
+    Returns:
+
+    """
+    import shutil
+
+    assert self.is_file()
+    shutil.copy(str(self), target)
+
+
+Path.copy = _copy  # type: ignore
+
+
+def extract_zip(outfile, effective_path):
+    try:
+        with zipfile.ZipFile(outfile) as z:
+            z.extractall(effective_path)
+    except zipfile.BadZipFile as e:
+        raise ValueError("Bad zip file") from e
+
+
+def convert_timestamp_to_datetime(timestamp: int) -> datetime:
+    if timestamp >= 0:
+        return datetime.fromtimestamp(timestamp)
+    else:
+        return datetime(1970, 1, 1) + timedelta(seconds=int(timestamp))
+
+
+def analytics_features(
+    dataframe: str, datatype: str, report_type: str, ncols: int, nrows: int, dbx: str
+) -> None:
+    endpoint = "https://packages.ydata.ai/data-profiling?"
+    package_version = __version__
+
+    if (
+        bool(os.getenv("YDATA_PROFILING_NO_ANALYTICS")) is not True
+        and package_version != "0.0.dev0"
+    ):
+        try:
+            subprocess.check_output("nvidia-smi")
+            gpu_present = True
+        except Exception:
+            gpu_present = False
+
+        python_version = ".".join(platform.python_version().split(".")[:2])
+
+        with contextlib.suppress(Exception):
+            request_message = (
+                f"{endpoint}version={package_version}"
+                f"&python_version={python_version}"
+                f"&report_type={report_type}"
+                f"&dataframe={dataframe}"
+                f"&ncols={ncols}"
+                f"&nrows={nrows}"
+                f"&datatype={datatype}"
+                f"&os={platform.system()}"
+                f"&gpu={str(gpu_present)}"
+                f"&dbx={dbx}"
+            )
+
+            requests.get(request_message)
+
+
+def is_running_in_databricks():
+    mask = "DATABRICKS_RUNTIME_VERSION" in os.environ
+    if "DATABRICKS_RUNTIME_VERSION" in os.environ:
+        return os.environ["DATABRICKS_RUNTIME_VERSION"]
+    else:
+        return str(mask)
+
+
+def calculate_nrows(df):
+    """
+    Calculates the approx. number of rows spark dataframes
+
+    Returns: int, approximate number of rows
+    """
+    if isinstance(df, pd.DataFrame):
+        if df is not None:
+            nrows = len(df)
+        else:
+            nrows = 0
+    else:
+        try:
+            n_partitions = df.rdd.getNumPartitions()
+
+            nrows = (
+                df.rdd.mapPartitionsWithIndex(
+                    lambda idx, partition: [sum(1 for _ in partition)]
+                    if idx == 0
+                    else [0]
+                ).collect()[0]
+                * n_partitions
+            )
+        except Exception:
+            nrows = 0
+
+    return nrows

data_profiling/utils/compat.py

@@ -0,0 +1,31 @@
+"""Utility functions for (version) compatibility"""
+
+from contextlib import contextmanager
+from functools import lru_cache
+from typing import Generator, Tuple
+
+import pandas as pd
+
+
+@lru_cache(maxsize=1)
+def pandas_version_info() -> Tuple[int, ...]:
+    """
+    Get the Pandas version as a tuple of integers,
+    akin to `sys.version_info` for the Python version.
+    """
+    return tuple(int(s) for s in pd.__version__.split("."))
+
+
+@contextmanager
+def optional_option_context(
+    option_key: str, value: object
+) -> Generator[None, None, None]:
+    """
+    A context manager that sets an option only if it is available in the
+    current pandas version; otherwise, it is a no-op.
+    """
+    try:
+        with pd.option_context(option_key, value):
+            yield
+    except pd.errors.OptionError:
+        yield

data_profiling/utils/dataframe.py

@@ -0,0 +1,238 @@
+"""Utils for pandas DataFrames."""
+import hashlib
+import re
+import unicodedata
+import warnings
+from pathlib import Path
+from typing import Any, Optional
+
+import pandas as pd
+from pandas.core.util.hashing import hash_pandas_object
+
+
+def warn_read(extension: str) -> None:
+    """Warn the user when an extension is not supported.
+
+    Args:
+        extension: The extension that is not supported.
+    """
+    warnings.warn(
+        f"""There was an attempt to read a file with extension {extension}, we assume it to be in CSV format.
+To prevent this warning from showing up, please rename the file to any of the extensions supported by pandas
+(docs: https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html)
+If you think this extension should be supported, please report this as an issue:
+https://github.com/Data-Centric-AI-Community/data-profiling/issues"""
+    )
+
+
+def is_supported_compression(file_extension: str) -> bool:
+    """Determine if the given file extension indicates a compression format that pandas can handle automatically.
+
+    Args:
+        file_extension (str): the file extension to test
+
+    Returns:
+        bool: True if the extension indicates a compression format that pandas handles automatically and False otherwise
+
+    Notes:
+        Pandas can handle on the fly decompression from the following extensions: ‘.bz2’, ‘.gz’, ‘.zip’, or ‘.xz’
+        (otherwise no decompression). If using ‘.zip’, the ZIP file must contain exactly one data file to be read in.
+    """
+    return file_extension.lower() in [".bz2", ".gz", ".xz", ".zip"]
+
+
+def remove_suffix(text: str, suffix: str) -> str:
+    """Removes the given suffix from the given string.
+
+    Args:
+        text (str): the string to remove the suffix from
+        suffix (str): the suffix to remove from the string
+
+    Returns:
+        str: the string with the suffix removed, if the string ends with the suffix, otherwise the unmodified string
+
+    Notes:
+        In python 3.9+, there is a built-in string method called removesuffix() that can serve this purpose.
+    """
+    return text[: -len(suffix)] if suffix and text.endswith(suffix) else text
+
+
+def uncompressed_extension(file_name: Path) -> str:
+    """Returns the uncompressed extension of the given file name.
+
+    Args:
+        file_name (Path): the file name to get the uncompressed extension of
+
+    Returns:
+        str: the uncompressed extension, or the original extension if pandas doesn't handle it automatically
+    """
+    extension = file_name.suffix.lower()
+    return (
+        Path(remove_suffix(str(file_name).lower(), extension)).suffix
+        if is_supported_compression(extension)
+        else extension
+    )
+
+
+def read_pandas(file_name: Path) -> pd.DataFrame:
+    """Read DataFrame based on the file extension. This function is used when the file is in a standard format.
+    Various file types are supported (.csv, .json, .jsonl, .data, .tsv, .xls, .xlsx, .xpt, .sas7bdat, .parquet)
+
+    Args:
+        file_name: the file to read
+
+    Returns:
+        DataFrame
+
+    Notes:
+        This function is based on pandas IO tools:
+        https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html
+        https://pandas.pydata.org/pandas-docs/stable/reference/io.html
+
+        This function is not intended to be flexible or complete. The main use case is to be able to read files without
+        user input, which is currently used in the editor integration. For more advanced use cases, the user should load
+        the DataFrame in code.
+    """
+    extension = uncompressed_extension(file_name)
+    if extension == ".json":
+        df = pd.read_json(str(file_name))
+    elif extension == ".jsonl":
+        df = pd.read_json(str(file_name), lines=True)
+    elif extension == ".dta":
+        df = pd.read_stata(str(file_name))
+    elif extension == ".tsv":
+        df = pd.read_csv(str(file_name), sep="\t")
+    elif extension in [".xls", ".xlsx"]:
+        df = pd.read_excel(str(file_name))
+    elif extension in [".hdf", ".h5"]:
+        df = pd.read_hdf(str(file_name))
+    elif extension in [".sas7bdat", ".xpt"]:
+        df = pd.read_sas(str(file_name))
+    elif extension == ".parquet":
+        df = pd.read_parquet(str(file_name))
+    elif extension in [".pkl", ".pickle"]:
+        df = pd.read_pickle(str(file_name))
+    elif extension == ".tar":
+        raise ValueError(
+            "tar compression is not supported directly by pandas, please use the 'tarfile' module"
+        )
+    else:
+        if extension != ".csv":
+            warn_read(extension)
+
+        df = pd.read_csv(str(file_name))
+    return df
+
+
+def rename_index(df: pd.DataFrame) -> pd.DataFrame:
+    """If the DataFrame contains a column or index named `index`, this will produce errors. We rename the {index,column}
+    to be `df_index`.
+
+    Args:
+        df: DataFrame to process.
+
+    Returns:
+        The DataFrame with {index,column} `index` replaced by `df_index`, unchanged if the DataFrame does not contain such a string.
+    """
+    df.rename(columns={"index": "df_index"}, inplace=True)
+
+    if "index" in df.index.names:
+        df.index.names = [x if x != "index" else "df_index" for x in df.index.names]
+    return df
+
+
+def expand_mixed(df: pd.DataFrame, types: Any = None) -> pd.DataFrame:
+    """Expand non-nested lists, dicts and tuples in a DataFrame into columns with a prefix.
+
+    Args:
+        types: list of types to expand (Default: list, dict, tuple)
+        df: DataFrame
+
+    Returns:
+        DataFrame with the dict values as series, identifier by the combination of the series and dict keys.
+
+    Notes:
+        TODO: allow for list expansion by duplicating rows.
+        TODO: allow for expansion of nested data structures.
+    """
+    if types is None:
+        types = [list, dict, tuple]
+
+    for column_name in df.columns:
+        # All
+        non_nested_enumeration = (
+            df[column_name]
+            .dropna()
+            .map(lambda x: type(x) in types and not any(type(y) in types for y in x))
+        )
+
+        if non_nested_enumeration.all():
+            # Expand and prefix
+            expanded = pd.DataFrame(df[column_name].dropna().tolist())
+            expanded = expanded.add_prefix(column_name + "_")
+
+            # Add recursion
+            expanded = expand_mixed(expanded)
+
+            # Drop the expanded
+            df.drop(columns=[column_name], inplace=True)
+
+            df = pd.concat([df, expanded], axis=1)
+    return df
+
+
+# Change this if `hash_dataframe`'s implementation changes.
+HASH_PREFIX = "2@"
+
+
+def hash_dataframe(df: pd.DataFrame) -> str:
+    """Hash a DataFrame (implementation might change in the future)
+
+    Args:
+        df: the DataFrame
+
+    Returns:
+        The DataFrame's hash
+    """
+    # hash_pandas_object returns a series of uint64s. Using their
+    # binary representation would be more efficient, but it's not
+    # necessarily portable across architectures. Using the human-readable
+    # string values should be good enough.
+    hash_values = "\n".join(hash_pandas_object(df).values.astype(str))
+    digest = hashlib.sha256(hash_values.encode("utf-8")).hexdigest()
+    return f"{HASH_PREFIX}{digest}"
+
+
+def slugify(value: str, allow_unicode: bool = False) -> str:
+    """
+    Taken from https://github.com/django/django/blob/master/django/utils/text.py
+    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
+    dashes to single dashes. Remove characters that aren't alphanumerics,
+    underscores, or hyphens. Convert to lowercase. Also strip leading and
+    trailing whitespace, dashes, and underscores.
+    """
+    value = str(value)
+    if allow_unicode:
+        value = unicodedata.normalize("NFKC", value)
+    else:
+        value = (
+            unicodedata.normalize("NFKD", value)
+            .encode("ascii", "ignore")
+            .decode("ascii")
+        )
+    value = re.sub(r"[^\w\s-]", "", value.lower())
+    return re.sub(r"[-\s]+", "-", value).strip("-_")
+
+
+def sort_column_names(dct: dict, sort: Optional[str]) -> dict:
+    if sort is None:
+        return dct
+
+    sort = sort.lower()
+    if sort.startswith("asc"):
+        dct = dict(sorted(dct.items(), key=lambda x: x[0].casefold()))
+    elif sort.startswith("desc"):
+        dct = dict(sorted(dct.items(), key=lambda x: x[0].casefold(), reverse=True))
+    else:
+        raise ValueError('"sort" should be "ascending", "descending" or None.')
+    return dct

data_profiling/utils/logger.py

@@ -0,0 +1,53 @@
+"""
+    Logger function for data-profiling reports
+"""
+
+import logging
+
+import pandas as pd
+
+from data_profiling.utils.common import (
+    analytics_features,
+    calculate_nrows,
+    is_running_in_databricks,
+)
+
+
+class ProfilingLogger(logging.Logger):
+    def __init__(self, name: str, level: int = logging.INFO):
+        super().__init__(name, level)
+
+    def info_def_report(self, df, timeseries: bool) -> None:  # noqa: ANN001
+        try:
+            ncols = len(df.columns)
+        except AttributeError:
+            ncols = 0
+
+        nrows = calculate_nrows(df)
+
+        if isinstance(df, pd.DataFrame):
+            dataframe = "pandas"
+            report_type = "regular"
+        elif df is None:
+            dataframe = "pandas"
+            report_type = "compare"
+        else:
+            dataframe = "spark"
+            report_type = "regular"
+
+        dbx = is_running_in_databricks()
+        datatype = "timeseries" if timeseries else "tabular"
+
+        analytics_features(
+            dataframe=dataframe,
+            datatype=datatype,
+            report_type=report_type,
+            nrows=nrows,
+            ncols=ncols,
+            dbx=dbx,
+        )
+
+        super().info(
+            f"[PROFILING] Calculating profile with the following characteristics "
+            f"- {dataframe} | {datatype} | {report_type}."
+        )

data_profiling/utils/notebook.py

@@ -0,0 +1,8 @@
+"""Utility functions that come in handy when working with Jupyter notebooks"""
+
+
+def full_width() -> None:
+    """Resize the notebook to use it's full width"""
+    from IPython.display import HTML, display
+
+    display(HTML("<style>.container { width:100% !important; }</style>"))

data_profiling/utils/paths.py

@@ -0,0 +1,45 @@
+"""Paths that are useful throughout the project."""
+from pathlib import Path
+
+
+def get_project_root() -> Path:
+    """Returns the path to the project root folder.
+
+    Returns:
+        The path to the project root folder.
+    """
+    return Path(__file__).parent.parent.parent.parent
+
+
+def get_config(file_name: str) -> Path:
+    """Returns the path a config file.
+
+    Returns:
+        The path to a config file.
+    """
+    return Path(__file__).parent.parent / file_name
+
+
+def get_data_path() -> Path:
+    """Returns the path to the dataset cache ([root] / data)
+
+    Returns:
+        The path to the dataset cache
+    """
+    return get_project_root() / "data"
+
+
+def get_html_template_path() -> Path:
+    """Returns the path to the HTML templates
+
+    Returns:
+        The path to the HTML templates
+    """
+    return (
+        Path(__file__).parent.parent
+        / "report"
+        / "presentation"
+        / "flavours"
+        / "html"
+        / "templates"
+    )

data_profiling/utils/progress_bar.py

@@ -0,0 +1,15 @@
+from functools import wraps
+from typing import Any, Callable
+
+from tqdm import tqdm
+
+
+def progress(fn: Callable, bar: tqdm, message: str) -> Callable:
+    @wraps(fn)
+    def inner(*args, **kwargs) -> Any:
+        bar.set_postfix_str(message)
+        ret = fn(*args, **kwargs)
+        bar.update()
+        return ret
+
+    return inner

data_profiling/utils/styles.py

@@ -0,0 +1,22 @@
+def get_alert_styles() -> dict:
+    return {
+        "constant": "warning",
+        "constant_length": "primary",
+        "duplicates": "secondary",
+        "empty": "info",
+        "high_cardinality": "danger",
+        "high_correlation": "secondary",
+        "infinite": "info",
+        "imbalance": "primary",
+        "missing": "info",
+        "non_stationary": "secondary",
+        "rejected": "danger",
+        "seasonal": "secondary",
+        "skewed": "info",
+        "truncated": "info",
+        "type_date": "warning",
+        "uniform": "danger",
+        "unique": "danger",
+        "unsupported": "warning",
+        "zeros": "info",
+    }

data_profiling/utils/versions.py

@@ -0,0 +1,19 @@
+try:
+    from importlib.metadata import version
+except ImportError:
+    import pkg_resources
+
+    def version(pkg: str) -> str:  # type: ignore
+        return pkg_resources.get_distribution(pkg).version
+
+
+def pandas_version() -> list:
+    return list(map(int, version("pandas").split(".")))
+
+
+def pandas_major_version() -> int:
+    return pandas_version()[0]
+
+
+def is_pandas_1() -> bool:
+    return pandas_major_version() == 1

data_profiling/version.py

@@ -0,0 +1 @@
+__version__ = '4.19.0'

data_profiling/visualisation/__init__.py

@@ -0,0 +1 @@
+"""Code for generating plots"""