fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/report/structure/variables/render_path.py

@@ -0,0 +1,134 @@
+from data_profiling.config import Settings
+from data_profiling.report.formatters import fmt, fmt_numeric
+from data_profiling.report.presentation.core import Container, FrequencyTable, Table
+from data_profiling.report.presentation.frequency_table_utils import freq_table
+from data_profiling.report.structure.variables.render_categorical import (
+    render_categorical,
+)
+
+
+def render_path(config: Settings, summary: dict) -> dict:
+    varid = summary["varid"]
+    n_freq_table_max = config.n_freq_table_max
+    redact = config.vars.cat.redact
+
+    template_variables = render_categorical(config, summary)
+
+    keys = ["name", "parent", "suffix", "stem", "anchor"]
+    for path_part in keys:
+        template_variables[f"freqtable_{path_part}"] = freq_table(
+            freqtable=summary[f"{path_part}_counts"],
+            n=summary["n"],
+            max_number_to_print=n_freq_table_max,
+        )
+
+    # Top
+    template_variables["top"].content["items"][0].content["var_type"] = "Path"
+
+    # Bottom
+    path_overview_tab = Container(
+        [
+            Table(
+                [
+                    {
+                        "name": "Common prefix",
+                        "value": fmt(summary["common_prefix"]),
+                        "alert": False,
+                    },
+                    {
+                        "name": "Unique stems",
+                        "value": fmt_numeric(
+                            summary["n_stem_unique"], precision=config.report.precision
+                        ),
+                        "alert": False,
+                    },
+                    {
+                        "name": "Unique names",
+                        "value": fmt_numeric(
+                            summary["n_name_unique"], precision=config.report.precision
+                        ),
+                        "alert": False,
+                    },
+                    {
+                        "name": "Unique extensions",
+                        "value": fmt_numeric(
+                            summary["n_suffix_unique"],
+                            precision=config.report.precision,
+                        ),
+                        "alert": False,
+                    },
+                    {
+                        "name": "Unique directories",
+                        "value": fmt_numeric(
+                            summary["n_parent_unique"],
+                            precision=config.report.precision,
+                        ),
+                        "alert": False,
+                    },
+                    {
+                        "name": "Unique anchors",
+                        "value": fmt_numeric(
+                            summary["n_anchor_unique"],
+                            precision=config.report.precision,
+                        ),
+                        "alert": False,
+                    },
+                ],
+                style=config.html.style,
+            )
+        ],
+        anchor_id=f"{varid}tbl",
+        name="Overview",
+        sequence_type="list",
+    )
+
+    path_items = [
+        path_overview_tab,
+        FrequencyTable(
+            template_variables["freq_table_rows"],
+            name="Full",
+            anchor_id=f"{varid}full_frequency",
+            redact=redact,
+        ),
+        FrequencyTable(
+            template_variables["freqtable_stem"],
+            name="Stem",
+            anchor_id=f"{varid}stem_frequency",
+            redact=redact,
+        ),
+        FrequencyTable(
+            template_variables["freqtable_name"],
+            name="Name",
+            anchor_id=f"{varid}name_frequency",
+            redact=redact,
+        ),
+        FrequencyTable(
+            template_variables["freqtable_suffix"],
+            name="Extension",
+            anchor_id=f"{varid}suffix_frequency",
+            redact=redact,
+        ),
+        FrequencyTable(
+            template_variables["freqtable_parent"],
+            name="Parent",
+            anchor_id=f"{varid}parent_frequency",
+            redact=redact,
+        ),
+        FrequencyTable(
+            template_variables["freqtable_anchor"],
+            name="Anchor",
+            anchor_id=f"{varid}anchor_frequency",
+            redact=redact,
+        ),
+    ]
+
+    path_tab = Container(
+        path_items,
+        name="Path",
+        sequence_type="tabs",
+        anchor_id=f"{varid}path",
+    )
+
+    template_variables["bottom"].content["items"].append(path_tab)
+
+    return template_variables

data_profiling/report/structure/variables/render_real.py

@@ -0,0 +1,314 @@
+from data_profiling.config import Settings
+from data_profiling.report.formatters import (
+    fmt,
+    fmt_bytesize,
+    fmt_monotonic,
+    fmt_numeric,
+    fmt_percent,
+)
+from data_profiling.report.presentation.core import (
+    Container,
+    FrequencyTable,
+    Table,
+    VariableInfo,
+)
+from data_profiling.report.structure.variables.render_common import render_common
+from data_profiling.report.utils import image_or_empty
+from data_profiling.visualisation.plot import histogram, mini_histogram
+
+
+def render_real(config: Settings, summary: dict) -> dict:
+    varid = summary["varid"]
+    template_variables = render_common(config, summary)
+    image_format = config.plot.image_format
+
+    name = "Real number (ℝ)"
+
+    # Top
+    info = VariableInfo(
+        summary["varid"],
+        summary["varname"],
+        name,
+        summary["alerts"],
+        summary["description"],
+        style=config.html.style,
+    )
+
+    table1 = Table(
+        [
+            {
+                "name": "Distinct",
+                "value": fmt(summary["n_distinct"]),
+                "alert": "n_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Distinct (%)",
+                "value": fmt_percent(summary["p_distinct"]),
+                "alert": "p_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing",
+                "value": fmt(summary["n_missing"]),
+                "alert": "n_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing (%)",
+                "value": fmt_percent(summary["p_missing"]),
+                "alert": "p_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Infinite",
+                "value": fmt(summary["n_infinite"]),
+                "alert": "n_infinite" in summary["alert_fields"],
+            },
+            {
+                "name": "Infinite (%)",
+                "value": fmt_percent(summary["p_infinite"]),
+                "alert": "p_infinite" in summary["alert_fields"],
+            },
+            {
+                "name": "Mean",
+                "value": fmt_numeric(
+                    summary["mean"], precision=config.report.precision
+                ),
+                "alert": False,
+            },
+        ],
+        style=config.html.style,
+    )
+
+    table2 = Table(
+        [
+            {
+                "name": "Minimum",
+                "value": fmt_numeric(summary["min"], precision=config.report.precision),
+                "alert": False,
+            },
+            {
+                "name": "Maximum",
+                "value": fmt_numeric(summary["max"], precision=config.report.precision),
+                "alert": False,
+            },
+            {
+                "name": "Zeros",
+                "value": fmt(summary["n_zeros"]),
+                "alert": "n_zeros" in summary["alert_fields"],
+            },
+            {
+                "name": "Zeros (%)",
+                "value": fmt_percent(summary["p_zeros"]),
+                "alert": "p_zeros" in summary["alert_fields"],
+            },
+            {
+                "name": "Negative",
+                "value": fmt(summary["n_negative"]),
+                "alert": False,
+            },
+            {
+                "name": "Negative (%)",
+                "value": fmt_percent(summary["p_negative"]),
+                "alert": False,
+            },
+            {
+                "name": "Memory size",
+                "value": fmt_bytesize(summary["memory_size"]),
+                "alert": False,
+            },
+        ],
+        style=config.html.style,
+    )
+
+    summary_histogram = summary.get("histogram", [])
+
+    mini_hist_data = None
+
+    if summary_histogram:
+        if isinstance(summary_histogram, list):
+            mini_hist_data = mini_histogram(
+                config,
+                [x[0] for x in summary_histogram],
+                [x[1] for x in summary_histogram],
+            )
+        else:
+            mini_hist_data = mini_histogram(config, *summary_histogram)
+
+    mini_histo = image_or_empty(
+        mini_hist_data,
+        alt="Mini histogram",
+        image_format=image_format,
+        name="Mini Histogram",
+        anchor_id=f"{varid}minihistogram",
+    )
+
+    template_variables["top"] = Container(
+        [info, table1, table2, mini_histo], sequence_type="grid"
+    )
+
+    quantile_statistics = Table(
+        [
+            {
+                "name": "Minimum",
+                "value": fmt_numeric(summary["min"], precision=config.report.precision),
+            },
+            {
+                "name": "5-th percentile",
+                "value": fmt_numeric(summary["5%"], precision=config.report.precision),
+            },
+            {
+                "name": "Q1",
+                "value": fmt_numeric(summary["25%"], precision=config.report.precision),
+            },
+            {
+                "name": "median",
+                "value": fmt_numeric(summary["50%"], precision=config.report.precision),
+            },
+            {
+                "name": "Q3",
+                "value": fmt_numeric(summary["75%"], precision=config.report.precision),
+            },
+            {
+                "name": "95-th percentile",
+                "value": fmt_numeric(summary["95%"], precision=config.report.precision),
+            },
+            {
+                "name": "Maximum",
+                "value": fmt_numeric(summary["max"], precision=config.report.precision),
+            },
+            {
+                "name": "Range",
+                "value": fmt_numeric(
+                    summary["range"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Interquartile range (IQR)",
+                "value": fmt_numeric(summary["iqr"], precision=config.report.precision),
+            },
+        ],
+        name="Quantile statistics",
+        style=config.html.style,
+    )
+
+    descriptive_statistics = Table(
+        [
+            {
+                "name": "Standard deviation",
+                "value": fmt_numeric(summary["std"], precision=config.report.precision),
+            },
+            {
+                "name": "Coefficient of variation (CV)",
+                "value": fmt_numeric(summary["cv"], precision=config.report.precision),
+            },
+            {
+                "name": "Kurtosis",
+                "value": fmt_numeric(
+                    summary["kurtosis"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Mean",
+                "value": fmt_numeric(
+                    summary["mean"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Median Absolute Deviation (MAD)",
+                "value": fmt_numeric(summary["mad"], precision=config.report.precision),
+            },
+            {
+                "name": "Skewness",
+                "value": fmt_numeric(
+                    summary["skewness"], precision=config.report.precision
+                ),
+                "class": "alert" if "skewness" in summary["alert_fields"] else "",
+            },
+            {
+                "name": "Sum",
+                "value": fmt_numeric(summary["sum"], precision=config.report.precision),
+            },
+            {
+                "name": "Variance",
+                "value": fmt_numeric(
+                    summary["variance"], precision=config.report.precision
+                ),
+            },
+            {
+                "name": "Monotonicity",
+                "value": fmt_monotonic(summary["monotonic"]),
+            },
+        ],
+        name="Descriptive statistics",
+        style=config.html.style,
+    )
+
+    statistics = Container(
+        [quantile_statistics, descriptive_statistics],
+        anchor_id=f"{varid}statistics",
+        name="Statistics",
+        sequence_type="grid",
+    )
+
+    hist_data = None
+    hist_caption = None
+
+    if summary_histogram:
+        if isinstance(summary_histogram, list):
+            hist_data = histogram(
+                config,
+                [x[0] for x in summary_histogram],
+                [x[1] for x in summary_histogram],
+            )
+            bins = len(summary_histogram[0][1])
+            hist_caption = (
+                f"<strong>Histogram with fixed size bins</strong> (bins={bins - 1})"
+            )
+        else:
+            hist_data = histogram(config, *summary_histogram)
+            hist_caption = (
+                f"<strong>Histogram with fixed size bins</strong> "
+                f"(bins={len(summary_histogram[1]) - 1})"
+            )
+
+    hist = image_or_empty(
+        hist_data,
+        alt="Histogram",
+        image_format=image_format,
+        caption=hist_caption,
+        name="Histogram",
+        anchor_id=f"{varid}histogram",
+    )
+
+    fq = FrequencyTable(
+        template_variables["freq_table_rows"],
+        name="Common values",
+        anchor_id=f"{varid}common_values",
+        redact=False,
+    )
+
+    evs = Container(
+        [
+            FrequencyTable(
+                template_variables["firstn_expanded"],
+                name=f"Minimum {config.n_extreme_obs} values",
+                anchor_id=f"{varid}firstn",
+                redact=False,
+            ),
+            FrequencyTable(
+                template_variables["lastn_expanded"],
+                name=f"Maximum {config.n_extreme_obs} values",
+                anchor_id=f"{varid}lastn",
+                redact=False,
+            ),
+        ],
+        sequence_type="tabs",
+        name="Extreme values",
+        anchor_id=f"{varid}extreme_values",
+    )
+
+    template_variables["bottom"] = Container(
+        [statistics, hist, fq, evs],
+        sequence_type="tabs",
+        anchor_id=f"{varid}bottom",
+    )
+
+    return template_variables

data_profiling/report/structure/variables/render_text.py

@@ -0,0 +1,189 @@
+from typing import Any, Dict, List
+
+from data_profiling.config import Settings
+from data_profiling.report.formatters import fmt, fmt_bytesize, fmt_percent
+from data_profiling.report.presentation.core import (
+    Container,
+    FrequencyTable,
+    Image,
+    Table,
+)
+from data_profiling.report.presentation.core.variable_info import VariableInfo
+from data_profiling.report.structure.variables.render_categorical import (
+    _get_n,
+    freq_table,
+    render_categorical,
+    render_categorical_frequency,
+    render_categorical_length,
+    render_categorical_unicode,
+)
+from data_profiling.report.structure.variables.render_common import render_common
+from data_profiling.visualisation.plot import plot_word_cloud
+
+
+def render_text(config: Settings, summary: Dict[str, Any]) -> Dict[str, Any]:
+    if config.vars.text.redact:
+        render = render_categorical(config, summary)
+        return render
+
+    varid = summary["varid"]
+    words = config.vars.text.words
+    characters = config.vars.text.characters
+    length = config.vars.text.length
+
+    template_variables = render_common(config, summary)
+
+    top_items: List[Any] = []
+    var_info = VariableInfo(
+        anchor_id=varid,
+        var_name=summary["varname"],
+        var_type=summary["type"],
+        alerts=summary["alerts"],
+        description=summary["description"],
+        style=config.html.style,
+    )
+    top_items.append(var_info)
+
+    table = Table(
+        [
+            {
+                "name": "Distinct",
+                "value": fmt(summary["n_distinct"]),
+                "alert": "n_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Distinct (%)",
+                "value": fmt_percent(summary["p_distinct"]),
+                "alert": "p_distinct" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing",
+                "value": fmt(summary["n_missing"]),
+                "alert": "n_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Missing (%)",
+                "value": fmt_percent(summary["p_missing"]),
+                "alert": "p_missing" in summary["alert_fields"],
+            },
+            {
+                "name": "Memory size",
+                "value": fmt_bytesize(summary["memory_size"]),
+                "alert": False,
+            },
+        ],
+        style=config.html.style,
+    )
+    top_items.append(table)
+
+    if words and "word_counts" in summary:
+        mini_wordcloud = Image(
+            plot_word_cloud(config, summary["word_counts"]),
+            image_format=config.plot.image_format,
+            alt="Mini wordcloud",
+        )
+        top_items.append(mini_wordcloud)
+    template_variables["top"] = Container(top_items, sequence_type="grid")
+
+    # ============================================================================================
+
+    bottom_items = []
+    overview_items = []
+    # length isn't being computed for categorical in spark
+    if length and "max_length" in summary:
+        length_table, length_histo = render_categorical_length(config, summary, varid)
+        overview_items.append(length_table)
+
+    # characters isn't being computed for categorical in spark
+    unitab = None
+    if characters and "category_alias_counts" in summary:
+        overview_table_char, unitab = render_categorical_unicode(config, summary, varid)
+        overview_items.append(overview_table_char)
+
+    unique_stats = render_categorical_frequency(config, summary, varid)
+    overview_items.append(unique_stats)
+
+    if not config.vars.text.redact:
+        rows = ("1st row", "2nd row", "3rd row", "4th row", "5th row")
+
+        if isinstance(summary["first_rows"], list):
+            sample = Table(
+                [
+                    {
+                        "name": name,
+                        "value": fmt(value),
+                        "alert": False,
+                    }
+                    for name, *value in zip(rows, *summary["first_rows"])
+                ],
+                name="Sample",
+                style=config.html.style,
+            )
+        else:
+            sample = Table(
+                [
+                    {
+                        "name": name,
+                        "value": fmt(value),
+                        "alert": False,
+                    }
+                    for name, value in zip(rows, summary["first_rows"])
+                ],
+                name="Sample",
+                style=config.html.style,
+            )
+        overview_items.append(sample)
+    overview = Container(
+        overview_items,
+        name="Overview",
+        anchor_id=f"{varid}overview",
+        sequence_type="batch_grid",
+        batch_size=len(overview_items),
+        titles=False,
+    )
+    bottom_items.append(overview)
+
+    if words and "word_counts" in summary:
+        woc = freq_table(
+            freqtable=summary["word_counts"],
+            n=_get_n(summary["word_counts"]),
+            max_number_to_print=10,
+        )
+
+        fqwo = FrequencyTable(
+            woc,
+            name="Common words",
+            anchor_id=f"{varid}cwo",
+            redact=config.vars.text.redact,
+        )
+
+        image = Image(
+            plot_word_cloud(config, summary["word_counts"]),
+            image_format=config.plot.image_format,
+            alt="Wordcloud",
+        )
+
+        bottom_items.append(
+            Container(
+                [fqwo, image],
+                name="Words",
+                anchor_id=f"{varid}word",
+                sequence_type="grid",
+            )
+        )
+
+    if unitab is not None:
+        bottom_items.append(
+            Container(
+                [unitab],
+                name="Characters",
+                anchor_id=f"{varid}characters",
+                sequence_type="grid",
+            )
+        )
+
+    template_variables["bottom"] = Container(
+        bottom_items, sequence_type="tabs", anchor_id=f"{varid}bottom"
+    )
+
+    return template_variables