fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/report/structure/variables/render_timeseries.py

@@ -0,0 +1,371 @@
+from data_profiling.config import Settings
+from data_profiling.report.formatters import (
+    fmt,
+    fmt_bytesize,
+    fmt_monotonic,
+    fmt_numeric,
+    fmt_percent,
+    fmt_timespan_timedelta,
+)
+from data_profiling.report.presentation.core import (
+    Container,
+    FrequencyTable,
+    Image,
+    Table,
+    VariableInfo,
+)
+from data_profiling.report.structure.variables.render_common import render_common
+from data_profiling.report.utils import image_or_empty
+from data_profiling.visualisation.plot import (
+    histogram,
+    mini_ts_plot,
+    plot_acf_pacf,
+    plot_timeseries_gap_analysis,
+)
+
+
+def _render_gap_tab(config: Settings, summary: dict) -> Container:
+    gap_stats = [
+        {
+            "name": "number of gaps",
+            "value": fmt_numeric(
+                summary["gap_stats"]["n_gaps"], precision=config.report.precision
+            ),
+        },
+        {
+            "name": "min",
+            "value": fmt_timespan_timedelta(
+                summary["gap_stats"]["min"], precision=config.report.precision
+            ),
+        },
+        {
+            "name": "max",
+            "value": fmt_timespan_timedelta(
+                summary["gap_stats"]["max"], precision=config.report.precision
+            ),
+        },
+        {
+            "name": "mean",
+            "value": fmt_timespan_timedelta(
+                summary["gap_stats"]["mean"], precision=config.report.precision
+            ),
+        },
+        {
+            "name": "std",
+            "value": fmt_timespan_timedelta(
+                summary["gap_stats"]["std"], precision=config.report.precision
+            ),
+        },
+    ]
+
+    gap_table = Table(
+        gap_stats,
+        name="Gap statistics",
+        style=config.html.style,
+    )
+
+    gap_plot = Image(
+        plot_timeseries_gap_analysis(
+            config, summary["gap_stats"]["series"], summary["gap_stats"]["gaps"]
+        ),
+        image_format=config.plot.image_format,
+        alt="Gap plot",
+        name="",
+        anchor_id=f"{summary['varid']}_gap_plot",
+    )
+    return Container(
+        [gap_table, gap_plot],
+        image_format=config.plot.image_format,
+        sequence_type="grid",
+        name="Gap analysis",
+        anchor_id=f"{summary['varid']}_gap_analysis",
+    )
+
+
+def render_timeseries(config: Settings, summary: dict) -> dict:
+    varid = summary["varid"]
+    template_variables = render_common(config, summary)
+    image_format = config.plot.image_format
+    name = "Numeric time series"
+
+    # Top
+    info = VariableInfo(
+        summary["varid"],
+        summary["varname"],
+        name,
+        summary["alerts"],
+        summary["description"],
+        style=config.html.style,
+    )
+
+    table1 = Table(
+        [
+            {
+                "name": "Distinct",
+                "value": fmt(summary["n_distinct"]),
+                "alert": "n_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Distinct (%)",
+                "value": fmt_percent(summary["p_distinct"]),
+                "alert": "p_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing",
+                "value": fmt(summary["n_missing"]),
+                "alert": "n_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing (%)",
+                "value": fmt_percent(summary["p_missing"]),
+                "alert": "p_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Infinite",
+                "value": fmt(summary["n_infinite"]),
+                "alert": "n_infinite" in summary["alert_fields"],
+            },
+            {
+                "name": "Infinite (%)",
+                "value": fmt_percent(summary["p_infinite"]),
+                "alert": "p_infinite" in summary["alert_fields"],
+            },
+        ],
+        style=config.html.style,
+    )
+
+    table2 = Table(
+        [
+            {
+                "name": "Mean",
+                "value": fmt_numeric(
+                    summary["mean"], precision=config.report.precision
+                ),
+                "alert": False,
+            },
+            {
+                "name": "Minimum",
+                "value": fmt_numeric(summary["min"], precision=config.report.precision),
+                "alert": False,
+            },
+            {
+                "name": "Maximum",
+                "value": fmt_numeric(summary["max"], precision=config.report.precision),
+                "alert": False,
+            },
+            {
+                "name": "Zeros",
+                "value": fmt(summary["n_zeros"]),
+                "alert": "n_zeros" in summary["alert_fields"],
+            },
+            {
+                "name": "Zeros (%)",
+                "value": fmt_percent(summary["p_zeros"]),
+                "alert": "p_zeros" in summary["alert_fields"],
+            },
+            {
+                "name": "Memory size",
+                "value": fmt_bytesize(summary["memory_size"]),
+                "alert": False,
+            },
+        ],
+        style=config.html.style,
+    )
+
+    mini_plot = Image(
+        mini_ts_plot(config, summary["series"]),
+        image_format=image_format,
+        alt="Mini TS plot",
+    )
+
+    template_variables["top"] = Container(
+        [info, table1, table2, mini_plot], sequence_type="grid"
+    )
+
+    quantile_statistics = Table(
+        [
+            {
+                "name": "Minimum",
+                "value": fmt_numeric(summary["min"], precision=config.report.precision),
+            },
+            {
+                "name": "5-th percentile",
+                "value": fmt_numeric(summary["5%"], precision=config.report.precision),
+            },
+            {
+                "name": "Q1",
+                "value": fmt_numeric(summary["25%"], precision=config.report.precision),
+            },
+            {
+                "name": "median",
+                "value": fmt_numeric(summary["50%"], precision=config.report.precision),
+            },
+            {
+                "name": "Q3",
+                "value": fmt_numeric(summary["75%"], precision=config.report.precision),
+            },
+            {
+                "name": "95-th percentile",
+                "value": fmt_numeric(summary["95%"], precision=config.report.precision),
+            },
+            {
+                "name": "Maximum",
+                "value": fmt_numeric(summary["max"], precision=config.report.precision),
+            },
+            {
+                "name": "Range",
+                "value": fmt_numeric(
+                    summary["range"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Interquartile range (IQR)",
+                "value": fmt_numeric(summary["iqr"], precision=config.report.precision),
+            },
+        ],
+        name="Quantile statistics",
+        style=config.html.style,
+    )
+
+    descriptive_statistics = Table(
+        [
+            {
+                "name": "Standard deviation",
+                "value": fmt_numeric(summary["std"], precision=config.report.precision),
+            },
+            {
+                "name": "Coefficient of variation (CV)",
+                "value": fmt_numeric(summary["cv"], precision=config.report.precision),
+            },
+            {
+                "name": "Kurtosis",
+                "value": fmt_numeric(
+                    summary["kurtosis"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Mean",
+                "value": fmt_numeric(
+                    summary["mean"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Median Absolute Deviation (MAD)",
+                "value": fmt_numeric(summary["mad"], precision=config.report.precision),
+            },
+            {
+                "name": "Skewness",
+                "value": fmt_numeric(
+                    summary["skewness"], precision=config.report.precision
+                ),
+                "class": "alert" if "skewness" in summary["alert_fields"] else "",
+            },
+            {
+                "name": "Sum",
+                "value": fmt_numeric(summary["sum"], precision=config.report.precision),
+            },
+            {
+                "name": "Variance",
+                "value": fmt_numeric(
+                    summary["variance"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Monotonicity",
+                "value": fmt_monotonic(summary["monotonic"]),
+            },
+            {
+                "name": "Augmented Dickey-Fuller test p-value",
+                "value": fmt_numeric(summary["addfuller"]),
+            },
+        ],
+        name="Descriptive statistics",
+        style=config.html.style,
+    )
+
+    statistics = Container(
+        [quantile_statistics, descriptive_statistics],
+        anchor_id=f"{varid}statistics",
+        name="Statistics",
+        sequence_type="grid",
+    )
+
+    summary_histogram = summary.get("histogram", [])
+
+    hist_data = None
+    hist_caption = None
+
+    if summary_histogram:
+        if isinstance(summary_histogram, list):
+            hist_data = histogram(
+                config,
+                [x[0] for x in summary["histogram"]],
+                [x[1] for x in summary["histogram"]],
+            )
+            hist_caption = f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][0][1]) - 1})"
+        else:
+            hist_data = histogram(config, *summary["histogram"])
+            hist_caption = f"<strong>Histogram with fixed size bins</strong> (bins={len(summary['histogram'][1]) - 1})"
+
+    hist = image_or_empty(
+        hist_data,
+        image_format=image_format,
+        alt="Histogram",
+        caption=hist_caption,
+        name="Histogram",
+        anchor_id=f"{varid}histogram",
+    )
+
+    fq = FrequencyTable(
+        template_variables["freq_table_rows"],
+        name="Common values",
+        anchor_id=f"{varid}common_values",
+        redact=False,
+    )
+
+    evs = Container(
+        [
+            FrequencyTable(
+                template_variables["firstn_expanded"],
+                name=f"Minimum {config.n_extreme_obs} values",
+                anchor_id=f"{varid}firstn",
+                redact=False,
+            ),
+            FrequencyTable(
+                template_variables["lastn_expanded"],
+                name=f"Maximum {config.n_extreme_obs} values",
+                anchor_id=f"{varid}lastn",
+                redact=False,
+            ),
+        ],
+        sequence_type="tabs",
+        name="Extreme values",
+        anchor_id=f"{varid}extreme_values",
+    )
+
+    acf_pacf = Image(
+        plot_acf_pacf(config, summary["series"]),
+        image_format=image_format,
+        alt="Autocorrelation",
+        caption="<strong>ACF and PACF</strong>",
+        name="Autocorrelation",
+        anchor_id=f"{varid}acf_pacf",
+    )
+
+    ts_plot = Image(
+        mini_ts_plot(config, summary["series"], figsize=(7, 3)),
+        image_format=image_format,
+        alt="Time-series plot",
+        name="Time-series",
+        anchor_id=f"{varid}_ts_plot",
+    )
+
+    ts_gap = _render_gap_tab(config, summary)
+
+    template_variables["bottom"] = Container(
+        [statistics, hist, ts_plot, ts_gap, fq, evs, acf_pacf],
+        sequence_type="tabs",
+        anchor_id=f"{varid}bottom",
+    )
+
+    return template_variables

data_profiling/report/structure/variables/render_url.py

@@ -0,0 +1,132 @@
+from data_profiling.config import Settings
+from data_profiling.report.formatters import fmt, fmt_bytesize, fmt_percent
+from data_profiling.report.presentation.core import (
+    Container,
+    FrequencyTable,
+    FrequencyTableSmall,
+    Table,
+    VariableInfo,
+)
+from data_profiling.report.presentation.frequency_table_utils import freq_table
+from data_profiling.report.structure.variables.render_common import render_common
+
+
+def render_url(config: Settings, summary: dict) -> dict:
+    varid = summary["varid"]
+    n_freq_table_max = config.n_freq_table_max
+
+    n_obs_cat = config.vars.cat.n_obs
+    redact = config.vars.cat.redact
+
+    template_variables = render_common(config, summary)
+
+    keys = ["scheme", "netloc", "path", "query", "fragment"]
+    for url_part in keys:
+        template_variables[f"freqtable_{url_part}"] = freq_table(
+            freqtable=summary[f"{url_part}_counts"],
+            n=summary["n"],
+            max_number_to_print=n_freq_table_max,
+        )
+
+    full_frequency_table = FrequencyTable(
+        template_variables["freq_table_rows"],
+        name="Full",
+        anchor_id=f"{varid}full_frequency",
+        redact=redact,
+    )
+    scheme_frequency_table = FrequencyTable(
+        template_variables["freqtable_scheme"],
+        name="Scheme",
+        anchor_id=f"{varid}scheme_frequency",
+        redact=redact,
+    )
+    netloc_frequency_table = FrequencyTable(
+        template_variables["freqtable_netloc"],
+        name="Netloc",
+        anchor_id=f"{varid}netloc_frequency",
+        redact=redact,
+    )
+    path_frequency_table = FrequencyTable(
+        template_variables["freqtable_path"],
+        name="Path",
+        anchor_id=f"{varid}path_frequency",
+        redact=redact,
+    )
+    query_frequency_table = FrequencyTable(
+        template_variables["freqtable_query"],
+        name="Query",
+        anchor_id=f"{varid}query_frequency",
+        redact=redact,
+    )
+    fragment_frequency_table = FrequencyTable(
+        template_variables["freqtable_fragment"],
+        name="Fragment",
+        anchor_id=f"{varid}fragment_frequency",
+        redact=redact,
+    )
+
+    items = [
+        full_frequency_table,
+        scheme_frequency_table,
+        netloc_frequency_table,
+        path_frequency_table,
+        query_frequency_table,
+        fragment_frequency_table,
+    ]
+    template_variables["bottom"] = Container(
+        items, sequence_type="tabs", name="url stats", anchor_id=f"{varid}urlstats"
+    )
+
+    # Element composition
+    info = VariableInfo(
+        summary["varid"],
+        summary["varname"],
+        "URL",
+        summary["alerts"],
+        summary["description"],
+        style=config.html.style,
+    )
+
+    table = Table(
+        [
+            {
+                "name": "Distinct",
+                "value": fmt(summary["n_distinct"]),
+                "alert": "n_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Distinct (%)",
+                "value": fmt_percent(summary["p_distinct"]),
+                "alert": "p_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing",
+                "value": fmt(summary["n_missing"]),
+                "alert": "n_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing (%)",
+                "value": fmt_percent(summary["p_missing"]),
+                "alert": "p_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Memory size",
+                "value": fmt_bytesize(summary["memory_size"]),
+                "alert": False,
+            },
+        ],
+        style=config.html.style,
+    )
+
+    fqm = FrequencyTableSmall(
+        freq_table(
+            freqtable=summary["value_counts_without_nan"],
+            n=summary["n"],
+            max_number_to_print=n_obs_cat,
+        ),
+        redact=redact,
+    )
+
+    template_variables["top"] = Container([info, table, fqm], sequence_type="grid")
+
+    return template_variables

data_profiling/report/utils.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from data_profiling.config import ImageType
+from data_profiling.report.presentation.core import Container, Image
+
+
+def image_or_empty(
+    image: Optional[str],
+    *,
+    alt: str,
+    image_format: ImageType,
+    caption: Optional[str] = None,
+    name: Optional[str] = None,
+    anchor_id: Optional[str] = None,
+) -> Container | Image:
+    """
+    Add helper to render an image or an empty container when necessary.
+    """
+    if image is None:
+        return Container(
+            items=[],
+            name=name or f"{alt}_empty",
+            anchor_id=anchor_id,
+            sequence_type="grid",
+        )
+
+    return Image(
+        image=image,
+        image_format=image_format,
+        alt=alt,
+        caption=caption,
+        name=name,
+        anchor_id=anchor_id,
+    )

data_profiling/serialize_report.py

@@ -0,0 +1,143 @@
+import warnings
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+if TYPE_CHECKING:
+    from data_profiling.profile_report import ProfileReport
+
+from data_profiling.config import Settings
+from data_profiling.model import BaseDescription
+from data_profiling.report.presentation.core import Root
+from data_profiling.version import __version__
+
+
+class SerializeReport:
+    """Extend the report to be able to dump and load reports."""
+
+    df = None
+    config = None
+    _df_hash: Optional[str] = None
+    _report = None
+    _description_set = None
+
+    @property
+    def df_hash(self) -> Optional[str]:
+        return None
+
+    def dumps(self) -> bytes:
+        """
+        Serialize ProfileReport and return bytes for reproducing ProfileReport or Caching.
+
+        Returns:
+            Bytes which contains hash of DataFrame, config, _description_set and _report
+        """
+        import pickle
+
+        # Note: _description_set and _report may are None if they haven't been computed
+        return pickle.dumps(
+            [
+                self.df_hash,
+                self.config,
+                self._description_set,
+                self._report,
+            ]
+        )
+
+    def loads(self, data: bytes) -> Union["ProfileReport", "SerializeReport"]:
+        """
+        Deserialize the serialized report
+
+        Args:
+            data: The bytes of a serialize ProfileReport object.
+
+        Raises:
+            ValueError: if ignore_config is set to False and the configs do not match.
+
+        Returns:
+            self
+        """
+        import pickle
+
+        try:
+            (
+                df_hash,
+                loaded_config,
+                loaded_description_set,
+                loaded_report,
+            ) = pickle.loads(data)
+        except Exception as e:
+            raise ValueError("Failed to load data") from e
+
+        if not all(
+            (
+                df_hash is None or isinstance(df_hash, str),
+                isinstance(loaded_config, Settings),
+                loaded_description_set is None
+                or isinstance(loaded_description_set, BaseDescription),
+                loaded_report is None or isinstance(loaded_report, Root),
+            )
+        ):
+            raise ValueError(
+                "Failed to load data: file may be damaged or from an incompatible version"
+            )
+        if (df_hash == self.df_hash) or (self.df is None):
+            # load to an empty ProfileReport
+            # Set description_set, report, sample if they are None，or raise an warning.
+            if self._description_set is None:
+                self._description_set = loaded_description_set
+            else:
+                warnings.warn(
+                    "The description set of current ProfileReport is not None. It won't be loaded."
+                )
+            if self._report is None:
+                self._report = loaded_report
+            else:
+                warnings.warn(
+                    "The report of current ProfileReport is not None. It won't be loaded."
+                )
+
+            # overwrite config
+            self.config = loaded_config
+
+            # warn if version not equal
+            if (
+                loaded_description_set is not None
+                and loaded_description_set.package["data_profiling_version"]
+                != __version__
+            ):
+                warnings.warn(
+                    f"The package version specified in the loaded data is not equal to the version installed. "
+                    f"Currently running on data-profiling {__version__} , while loaded data is generated by data_profiling, {loaded_description_set.package['data_profiling_version']}."
+                )
+
+            # set df_hash
+            self._df_hash = df_hash
+
+        else:
+            raise ValueError("DataFrame does not match with the current ProfileReport.")
+        return self
+
+    def dump(self, output_file: Union[Path, str]) -> None:
+        """
+        Dump ProfileReport to file
+        """
+        if not isinstance(output_file, Path):
+            output_file = Path(str(output_file))
+
+        output_file = output_file.with_suffix(".pp")
+        output_file.write_bytes(self.dumps())
+
+    def load(
+        self, load_file: Union[Path, str]
+    ) -> Union["ProfileReport", "SerializeReport"]:
+        """
+        Load ProfileReport from file
+
+        Raises:
+             ValueError: if the DataFrame or Config do not match with the current ProfileReport
+        """
+        if not isinstance(load_file, Path):
+            load_file = Path(str(load_file))
+
+        self.loads(load_file.read_bytes())
+        return self

data_profiling/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility functions for the complete package."""

data_profiling/utils/backend.py

@@ -0,0 +1,9 @@
+"""
+    File with a function to check the backend being used
+"""
+import importlib
+
+
+def is_pyspark_installed() -> bool:
+    """Check if PySpark is installed without importing it."""
+    return importlib.util.find_spec("pyspark") is not None