tabalyst 0.7.5__tar.gz → 0.7.6__tar.gz

{tabalyst-0.7.5/src/tabalyst.egg-info → tabalyst-0.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.7.5
+Version: 0.7.6
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MPL-2.0
@@ -114,8 +114,9 @@ customers.json
 executions.json
 ```
 
-Add `--details` to generate one self-contained HTML page per column, and `-o`
-to choose another HTML filename for one source:
+Add `--details` to write one HTML page per column and the chart pages, such as
+the combinations of missing values (it also runs the second pass), and `-o` to
+choose another HTML filename for one source:
 
 ```console
 tabalyst report customers.csv --details
@@ -164,7 +165,8 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 The report is built on Tabalyst Scan; values of sensitive columns can be masked
 or hidden. Ambiguous dates stay
 ambiguous; the report shows the evidence of the column without applying it. The
-HTML report is self-contained and works without a CDN or network connection.
+HTML report is self-contained and works without a CDN or network connection;
+only the chart pages of `--details` load Apache ECharts from a CDN to draw.
 The JSON profile contains the same canonical analysis result for scripts and
 future Tabalyst tools. See
 [how to read the report](https://docs.tabalyst.com/report/read-report/).

{tabalyst-0.7.5 → tabalyst-0.7.6}/README.md

@@ -74,8 +74,9 @@ customers.json
 executions.json
 ```
 
-Add `--details` to generate one self-contained HTML page per column, and `-o`
-to choose another HTML filename for one source:
+Add `--details` to write one HTML page per column and the chart pages, such as
+the combinations of missing values (it also runs the second pass), and `-o` to
+choose another HTML filename for one source:
 
 ```console
 tabalyst report customers.csv --details
@@ -124,7 +125,8 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 The report is built on Tabalyst Scan; values of sensitive columns can be masked
 or hidden. Ambiguous dates stay
 ambiguous; the report shows the evidence of the column without applying it. The
-HTML report is self-contained and works without a CDN or network connection.
+HTML report is self-contained and works without a CDN or network connection;
+only the chart pages of `--details` load Apache ECharts from a CDN to draw.
 The JSON profile contains the same canonical analysis result for scripts and
 future Tabalyst tools. See
 [how to read the report](https://docs.tabalyst.com/report/read-report/).

{tabalyst-0.7.5 → tabalyst-0.7.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.7.5"
+version = "0.7.6"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"

{tabalyst-0.7.5 → tabalyst-0.7.6}/src/tabalyst/cli/report.py

@@ -210,7 +210,10 @@ def report_command(
         bool,
         typer.Option(
             "--details/--no-details",
-            help="Generate standalone HTML pages for every column (off by default).",
+            help=(
+                "Also write a page for every column and the chart pages, which "
+                "load Apache ECharts from a CDN. Forces the second pass, as --deep."
+            ),
         ),
     ] = False,
     quiet: Annotated[

{tabalyst-0.7.5 → tabalyst-0.7.6}/src/tabalyst/config.py

@@ -431,18 +431,22 @@ class DriftSettings(BaseModel):
 
 
 class MissingSettings(BaseModel):
-    """How the values missing together are counted (design 20.31).
+    """How the values missing together are counted (design 20.31 and 20.44).
 
     A dataset of more than ``max_columns`` columns is left out (the pairs grow with
     the square of the columns); ``max_listed`` partners at most are listed for a
-    column, the ones missing most often with it. Both are part of the key of a
-    result.
+    column, the ones missing most often with it. The combinations of a dataset
+    (``missing_patterns``) are tracked up to ``max_patterns``, each counted exactly
+    from its first record, and ``max_patterns_listed`` are listed, the most frequent
+    first. Each is part of the key of a result.
     """
 
     model_config = ConfigDict(extra="forbid", frozen=True)
 
     max_columns: int = Field(default=100, ge=2, le=1_000)
     max_listed: int = Field(default=10, ge=1, le=1_000)
+    max_patterns: int = Field(default=1_000, ge=1, le=100_000)
+    max_patterns_listed: int = Field(default=200, ge=1, le=10_000)
 
 
 class ArraySettings(BaseModel):

{tabalyst-0.7.5 → tabalyst-0.7.6}/src/tabalyst/models.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-"""Serializable report document (format ``tabalyst.report``, revision 4).
+"""Serializable report document (format ``tabalyst.report``, ``FORMAT_REVISION``).
 
 The document is the Scan base at the same paths, plus a ``report`` block with
 the generation data, the presentation settings and the projections by dataset
@@ -40,7 +40,7 @@ from tabalyst.summary_models import ColumnSummary, TableSummary
 
 FORMAT = "tabalyst.report"
 FORMAT_VERSION = "0.2.0"
-FORMAT_REVISION = 5
+FORMAT_REVISION = 6
 
 
 MeasureStatus = Literal["complete", "limited", "not_applicable", "disabled", "failed"]
@@ -359,7 +359,7 @@ class ReportProfile(ResultModel):
 
     format: Literal["tabalyst.report"] = FORMAT
     format_version: Literal["0.2.0"] = FORMAT_VERSION
-    format_revision: Literal[5] = FORMAT_REVISION
+    format_revision: Literal[6] = FORMAT_REVISION
     engine: EngineInfo
     status: Literal["complete", "partial"]
     run: RunInfo

{tabalyst-0.7.5 → tabalyst-0.7.6}/src/tabalyst/report_chain.py

@@ -134,7 +134,7 @@ def plan_chain(
         skip_reading = SOURCE_NOT_READ
     if skip_reading == SOURCE_UNAVAILABLE and mode == "forced":
         raise SourceNotReadable(
-            "--deep needs to read the source of the scan document, which was not "
+            "--deep and --details need to read the source of the scan document, which was not "
             "found beside it. The report was not written. Restore the source "
             "next to the document or report the source itself."
         )

tabalyst-0.7.6/src/tabalyst/report_charts.py

@@ -0,0 +1,227 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""The chart pages of a Report (``docs/dev/report/chart-pages.md``): what each page
+draws, read from the report JSON alone, and the library the pages load.
+
+Pure: no file, no source. ``reporting.render_chart_page`` prints what this module
+builds and ``static/charts.js`` draws it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from urllib.parse import quote
+
+from tabalyst.models import ReportProfile
+from tabalyst.report_insights import block_line, missing_blocks
+from tabalyst.report_views import InsightLine
+from tabalyst.scanner.secondpass.models import MissingPatternsResult
+
+
+@dataclass(frozen=True, slots=True)
+class Library:
+    """A script the detail pages load from a CDN, pinned to one published file:
+    ``integrity`` is the SRI hash of that file, so it cannot be swapped."""
+
+    name: str
+    version: str
+    url: str
+    integrity: str
+
+
+# Moving to an embedded copy changes this constant and the templates, nothing else
+# (chart-pages.md 5). The hash was computed from the published file.
+ECHARTS = Library(
+    name="Apache ECharts",
+    version="6.1.0",
+    url="https://cdn.jsdelivr.net/npm/echarts@6.1.0/dist/echarts.min.js",
+    integrity="sha384-C2iskrW/uPW46KzOjrvJIQo4YkV8lkD+QS0CrDN18IIPIpT/g2USu8bTP3nvmIAD",
+)
+
+# The patterns the plot draws; the table lists every pattern the result lists.
+DRAWN = 20
+# The file of each topic, stable once released.
+CHART_FILES = {"missing": "charts-missing.html"}
+TOPIC_TITLES = {"missing": "Missing values"}
+# Why a dataset has no section on a chart page, in words.
+LEFT_OUT_REASONS = {
+    "no_missing_values": "fewer than two columns lack a value",
+    "too_many_columns": "more columns lack a value than deep.missing.max_columns allows",
+    "withheld_columns": "a column that lacks values is withheld by the exposure setting",
+    "analysis_failed": "the second pass did not count the combinations",
+}
+_HIDDEN = {"mask", "hide"}
+
+
+@dataclass(frozen=True, slots=True)
+class ColumnRow:
+    """A column that lacks values: the set size of the plot, with the link to its
+    column page."""
+
+    field: str
+    display: str
+    position: int
+    missing: int
+    share: str
+    href: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class PatternRow:
+    """A listed pattern: the exact set of columns some records lack a value in."""
+
+    names: tuple[str, ...]
+    size: int
+    count: int
+    share: str
+
+
+@dataclass(frozen=True, slots=True)
+class MissingSection:
+    """The combinations of missing values of one dataset, as its section of
+    ``charts-missing.html`` prints and draws them. ``data`` is what ``charts.js``
+    reads: the columns in document order, their counts, and each listed pattern as
+    its count and the positions of its columns."""
+
+    dataset: str
+    result: MissingPatternsResult
+    lacking: int
+    columns: list[ColumnRow]
+    patterns: list[PatternRow]
+    lines: list[InsightLine]
+    data: dict[str, object] = field(default_factory=dict)
+
+    @property
+    def drawn(self) -> int:
+        """The patterns the plot draws at first."""
+        return min(DRAWN, len(self.patterns))
+
+
+@dataclass(frozen=True, slots=True)
+class LeftOut:
+    """A dataset without a section on a chart page, and why (a key of ``LEFT_OUT_REASONS``)."""
+
+    topic: str
+    dataset: str
+    reason: str
+
+    @property
+    def words(self) -> str:
+        """The reason in words."""
+        return LEFT_OUT_REASONS[self.reason]
+
+
+def _share(part: int, whole: int) -> str:
+    return f"{part / whole:.1%}" if whole else "0%"
+
+
+def _reason(profile: ReportProfile, dataset: str) -> str:
+    """Why a dataset has no combinations, from the plan of the second pass."""
+    deep = profile.analyses.deep if profile.analyses is not None else None
+    if deep is None:
+        return "analysis_failed"
+    for candidate in deep.plan.candidates:
+        if candidate.analyzer == "missing_patterns" and candidate.dataset == dataset:
+            if "too_many_columns" in candidate.applicability.reasons:
+                return "too_many_columns"
+            return "analysis_failed"
+    return "no_missing_values"
+
+
+def missing_sections(
+    profile: ReportProfile,
+    column_links: Mapping[tuple[str, str], str] | None = None,
+) -> tuple[list[MissingSection], list[LeftOut]]:
+    """The sections of ``charts-missing.html``, one per dataset with combinations
+    counted, in the order of the report, and the datasets left out with their reason.
+
+    ``column_links`` maps a dataset and a field id to the href of its column page,
+    relative to the folder of the detail pages. A result that names a column the
+    report withholds is left out whole: its counts would describe that column.
+    """
+    links = column_links or {}
+    found: dict[str, MissingPatternsResult] = {}
+    deep = profile.analyses.deep if profile.analyses is not None else None
+    for item in deep.analyses if deep is not None else []:
+        if item.analyzer != "missing_patterns" or item.status not in ("complete", "limited"):
+            continue
+        if isinstance(item.result, MissingPatternsResult):
+            found[item.dataset] = item.result
+    sections: list[MissingSection] = []
+    left: list[LeftOut] = []
+    for dataset in profile.datasets:
+        exposure = {field.id: field.exposure for field in dataset.fields}
+        result = found.get(dataset.id)
+        if result is None:
+            left.append(LeftOut("missing", dataset.id, _reason(profile, dataset.id)))
+            continue
+        if any(exposure.get(column.field) in _HIDDEN for column in result.columns):
+            left.append(LeftOut("missing", dataset.id, "withheld_columns"))
+            continue
+        sections.append(_section(dataset.id, result, links))
+    return sections, left
+
+
+def _section(
+    dataset: str, result: MissingPatternsResult, links: Mapping[tuple[str, str], str]
+) -> MissingSection:
+    position = {column.field: index for index, column in enumerate(result.columns)}
+    names = {column.field: column.display for column in result.columns}
+    columns = sorted(
+        (
+            ColumnRow(
+                field=column.field,
+                display=column.display,
+                position=position[column.field],
+                missing=column.missing,
+                share=_share(column.missing, result.records),
+                href=links.get((dataset, column.field)),
+            )
+            for column in result.columns
+        ),
+        key=lambda row: (-row.missing, position[row.field]),
+    )
+    patterns = [
+        PatternRow(
+            names=tuple(names[name] for name in pattern.columns),
+            size=len(pattern.columns),
+            count=pattern.count,
+            share=_share(pattern.count, result.records),
+        )
+        for pattern in result.listed
+    ]
+    data: dict[str, object] = {
+        "records": result.records,
+        "columns": [column.display for column in result.columns],
+        "missing": [column.missing for column in result.columns],
+        "patterns": [
+            [pattern.count, [position[name] for name in pattern.columns]]
+            for pattern in result.listed
+        ],
+        "drawn": DRAWN,
+    }
+    return MissingSection(
+        dataset=dataset,
+        result=result,
+        lacking=result.records - result.complete_records,
+        columns=columns,
+        patterns=patterns,
+        lines=[block_line(block) for block in missing_blocks(result)],
+        data=data,
+    )
+
+
+def column_charts(
+    sections: list[MissingSection], dataset: str, field_id: str
+) -> list[tuple[str, str]]:
+    """The chart pages that name a column, as ``(title, href)`` from a page of the
+    same folder: ``charts-missing.html#<dataset>`` for a column that lacks values."""
+    return [
+        (TOPIC_TITLES["missing"], f"{CHART_FILES['missing']}#{quote(section.dataset, safe='')}")
+        for section in sections
+        if section.dataset == dataset
+        and any(column.field == field_id for column in section.result.columns)
+    ]

{tabalyst-0.7.5 → tabalyst-0.7.6}/src/tabalyst/report_deep.py

@@ -56,6 +56,7 @@ from tabalyst.scanner.secondpass.models import (
     MeasureResult,
     MissingIdCopiesResult,
     MissingMatrixResult,
+    MissingPatternsResult,
     NumberResolution,
     PartsResult,
     PatternCounts,
@@ -219,6 +220,8 @@ def result_summary(result: AnalysisOutput) -> tuple[str, str]:
         return _drift_summary(result), EXACT
     if isinstance(result, MissingMatrixResult):
         return _matrix_summary(result), EXACT
+    if isinstance(result, MissingPatternsResult):
+        return _patterns_summary(result), _exact_unless_partial(result.complete)
     if isinstance(result, DateOrderResult):
         return _order_summary(result), EXACT
     if isinstance(result, ArrayProfileResult):
@@ -405,6 +408,20 @@ def _matrix_summary(result: MissingMatrixResult) -> str:
     return text
 
 
+def _patterns_summary(result: MissingPatternsResult) -> str:
+    lacking = result.records - result.complete_records
+    seen = f"{result.distinct:,}" if result.complete else f"at least {result.distinct:,}"
+    text = (
+        f"{lacking:,} records lack a value in at least one of {len(result.columns):,} columns, "
+        f"in {seen} combinations"
+    )
+    if result.listed:
+        top = result.listed[0]
+        names = {column.field: column.display for column in result.columns}
+        text += f"; the most frequent, {', '.join(names[field] for field in top.columns)} ({top.count:,})"
+    return text
+
+
 def _order_summary(result: DateOrderResult) -> str:
     text = f"{result.compared:,} records compared, {result.broken:,} with the first date after the second"
     skipped = sum(result.excluded.model_dump().values())