mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/nbrender/outputs_data.py

@@ -0,0 +1,595 @@
+"""Mostly Right data outputs rendered from portable Jupyter MIME bundles.
+
+The column inspector is stored as a normal ``display_data`` output.  Its structured MIME payload
+is the rich contract used by this renderer; the generated notebook also carries ``text/html`` and
+``text/plain`` fallbacks so ordinary Jupyter viewers still show useful, non-interactive content.
+Nothing in this module reads the candidate or executes notebook code.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from typing import Any
+
+from mostlyright.data_harness.nbrender.parse import Output, RenderContext, esc
+from mostlyright.data_harness.nbrender.tokens import (
+    CHART_BAR_MAX_WIDTH,
+    CHART_BAR_MIN_WIDTH,
+    CHART_BASELINE,
+    CHART_LABEL_BASELINES,
+    CHART_LABEL_CHAR_WIDTH,
+    CHART_LABEL_GAP,
+    CHART_PLOT_HEIGHT,
+    CHART_PLOT_LEFT,
+    CHART_PLOT_RIGHT,
+    CHART_VIEWBOX_HEIGHT,
+    CHART_VIEWBOX_WIDTH,
+)
+
+MIME = "application/vnd.mostlyright.column-profile.v1+json"
+_MAX_CHART_ITEMS = 12
+_MAX_COLUMNS = 256
+_MAX_CUSTOM_MIME_BYTES = 1024 * 1024
+_MAX_TEXT = 4096
+_MAX_COUNT = (1 << 63) - 1
+_SEMANTIC_AUTHORITIES = {"approved_recipe", "declared_local_plan", "none"}
+_SEMANTIC_COMPLETENESS = {"complete", "partial", "absent"}
+_SEMANTIC_RENDER_SCOPES = {"full", "reduced", "minimal"}
+_UNIT_STATES = {"declared", "normalized", "not_applicable", "unknown"}
+_CATEGORY_STATUSES = {"complete", "partial", "not_applicable", "unknown"}
+_EVIDENCE_KINDS = {"source_id"}
+
+
+def _bounded_payload(value: Any) -> bool:
+    """Reject oversized/deep custom data before rendering or allocating a JSON copy."""
+
+    remaining = _MAX_CUSTOM_MIME_BYTES
+    stack: list[tuple[Any, int]] = [(value, 0)]
+    while stack:
+        item, depth = stack.pop()
+        if depth > 16:
+            return False
+        if item is None or isinstance(item, bool):
+            remaining -= 16
+        elif isinstance(item, int):
+            # Custom JSON is browser-facing. Keep integers inside the count domain used by the
+            # renderer and avoid later decimal/float conversions on attacker-sized Python ints.
+            if item.bit_length() > 63:
+                return False
+            remaining -= 16
+        elif isinstance(item, float):
+            if not math.isfinite(item):
+                return False
+            remaining -= 16
+        elif isinstance(item, str):
+            if len(item) > _MAX_CUSTOM_MIME_BYTES:
+                return False
+            remaining -= len(item.encode("utf-8")) + 2
+        elif isinstance(item, Mapping):
+            if len(item) > 1024:
+                return False
+            remaining -= len(item) * 4
+            stack.extend((key, depth + 1) for key in item)
+            stack.extend((nested, depth + 1) for nested in item.values())
+        elif isinstance(item, list):
+            if len(item) > 4096:
+                return False
+            remaining -= len(item) * 2
+            stack.extend((nested, depth + 1) for nested in item)
+        else:
+            return False
+        if remaining < 0:
+            return False
+    return True
+
+
+def is_valid_column_profiles_payload(value: Any) -> bool:
+    """Return whether an untrusted custom MIME value has the bounded v1 render shape."""
+
+    if not isinstance(value, Mapping) or not _bounded_payload(value):
+        return False
+    if value.get("schema_version") != "mostlyright.column-profile.v1":
+        return False
+    semantics = value.get("semantics")
+    if semantics is not None and not _valid_semantics(semantics):
+        return False
+    raw_columns = value.get("columns")
+    if not isinstance(raw_columns, list) or not 1 <= len(raw_columns) <= _MAX_COLUMNS:
+        return False
+    names: set[str] = set()
+    for profile in raw_columns:
+        if not isinstance(profile, Mapping):
+            return False
+        name = profile.get("name")
+        if not isinstance(name, str) or not name or len(name) > _MAX_TEXT or name in names:
+            return False
+        names.add(name)
+        if not all(
+            isinstance(profile.get(field), str) and len(profile[field]) <= _MAX_TEXT
+            for field in ("logical_type", "physical_type", "distinct_status")
+        ):
+            return False
+        if "display_label" in profile and (
+            not isinstance(profile["display_label"], str)
+            or not 1 <= len(profile["display_label"]) <= 120
+        ):
+            return False
+        if "description" in profile and (
+            not isinstance(profile["description"], str)
+            or not 1 <= len(profile["description"]) <= 1000
+        ):
+            return False
+        unit = profile.get("unit")
+        if isinstance(unit, Mapping) and "state" in unit:
+            if unit.get("state") not in _UNIT_STATES:
+                return False
+            if unit.get("normalization") is not None and (
+                not isinstance(unit.get("normalization"), str) or len(unit["normalization"]) > 64
+            ):
+                return False
+            if unit.get("reversible") is not None and type(unit.get("reversible")) is not bool:
+                return False
+        if (
+            "categories_status" in profile
+            and profile["categories_status"] not in _CATEGORY_STATUSES
+        ):
+            return False
+        if "categories" in profile and not _valid_categories(profile["categories"]):
+            return False
+        if "evidence_refs" in profile and not _valid_evidence_refs(profile["evidence_refs"]):
+            return False
+        for field in ("row_count", "missing_count", "distinct_count", "sample_size"):
+            number = profile.get(field)
+            if (
+                isinstance(number, bool)
+                or not isinstance(number, int)
+                or not 0 <= number <= _MAX_COUNT
+            ):
+                return False
+        if profile["sample_size"] > profile["row_count"]:
+            return False
+        if profile["missing_count"] > profile["row_count"]:
+            return False
+        for field in ("missing_rate", "distinct_rate"):
+            rate = profile.get(field)
+            if isinstance(rate, bool) or not isinstance(rate, (int, float)):
+                return False
+            if isinstance(rate, int):
+                if rate not in (0, 1):
+                    return False
+            elif not math.isfinite(rate) or not 0 <= rate <= 1:
+                return False
+        if not all(
+            isinstance(profile.get(field), Mapping)
+            for field in ("unit", "summary", "chart", "source")
+        ):
+            return False
+        operations = profile.get("transformations")
+        if not isinstance(operations, list) or any(
+            not isinstance(operation, str) or len(operation) > _MAX_TEXT for operation in operations
+        ):
+            return False
+        items = profile["chart"].get("items")
+        if not isinstance(items, list) or len(items) > _MAX_CHART_ITEMS:
+            return False
+        for item in items:
+            if (
+                not isinstance(item, Mapping)
+                or not isinstance(item.get("label"), str)
+                or len(item["label"]) > _MAX_TEXT
+            ):
+                return False
+            count = item.get("count")
+            if isinstance(count, bool) or not isinstance(count, (int, float)) or count < 0:
+                return False
+            if isinstance(count, int):
+                if count > _MAX_COUNT:
+                    return False
+            elif not math.isfinite(count):
+                return False
+    return True
+
+
+def _valid_semantics(value: Any) -> bool:
+    if not isinstance(value, Mapping):
+        return False
+    required = {
+        "schema_version",
+        "authority",
+        "completeness",
+        "message",
+        "missing_column_count",
+        "column_count",
+        "render_scope",
+    }
+    allowed = required | {"summary", "grain_statement", "coverage", "limitations"}
+    if not required <= set(value) or not set(value) <= allowed:
+        return False
+    if value.get("schema_version") != "mostlyright.column-semantics.v1":
+        return False
+    if value.get("authority") not in _SEMANTIC_AUTHORITIES:
+        return False
+    if value.get("completeness") not in _SEMANTIC_COMPLETENESS:
+        return False
+    if value.get("render_scope") not in _SEMANTIC_RENDER_SCOPES:
+        return False
+    if not isinstance(value.get("message"), str) or not 1 <= len(value["message"]) <= 512:
+        return False
+    for field in ("missing_column_count", "column_count"):
+        number = value.get(field)
+        if type(number) is not int or not 0 <= number <= _MAX_COLUMNS:
+            return False
+    if value["missing_column_count"] > value["column_count"]:
+        return False
+    for field, maximum in (("summary", 1000), ("grain_statement", 400)):
+        if field in value and (
+            not isinstance(value[field], str) or not 1 <= len(value[field]) <= maximum
+        ):
+            return False
+    coverage = value.get("coverage")
+    if coverage is not None:
+        if not isinstance(coverage, Mapping) or set(coverage) != {
+            "time_range",
+            "population",
+            "completeness_note",
+        }:
+            return False
+        if not isinstance(coverage.get("population"), str) or len(coverage["population"]) > 2000:
+            return False
+        if (
+            not isinstance(coverage.get("completeness_note"), str)
+            or len(coverage["completeness_note"]) > 1000
+        ):
+            return False
+        time_range = coverage.get("time_range")
+        if time_range is not None and (
+            not isinstance(time_range, Mapping)
+            or set(time_range) != {"start_inclusive", "end_exclusive"}
+            or any(
+                not isinstance(time_range.get(field), str) or len(time_range[field]) > 128
+                for field in ("start_inclusive", "end_exclusive")
+            )
+        ):
+            return False
+    limitations = value.get("limitations")
+    if limitations is not None and (
+        not isinstance(limitations, list)
+        or len(limitations) > 32
+        or any(not isinstance(item, str) or not 1 <= len(item) <= 1000 for item in limitations)
+    ):
+        return False
+    return True
+
+
+def _valid_categories(value: Any) -> bool:
+    return value is None or (
+        isinstance(value, list)
+        and 1 <= len(value) <= 64
+        and all(
+            isinstance(item, Mapping)
+            and set(item) == {"code", "label"}
+            and isinstance(item.get("code"), str)
+            and 1 <= len(item["code"]) <= 64
+            and isinstance(item.get("label"), str)
+            and 1 <= len(item["label"]) <= 120
+            for item in value
+        )
+    )
+
+
+def _valid_evidence_refs(value: Any) -> bool:
+    return (
+        isinstance(value, list)
+        and 1 <= len(value) <= 8
+        and all(
+            isinstance(item, Mapping)
+            and set(item) == {"kind", "ref"}
+            and item.get("kind") in _EVIDENCE_KINDS
+            and isinstance(item.get("ref"), str)
+            and 1 <= len(item["ref"]) <= 128
+            for item in value
+        )
+    )
+
+
+def _mapping(value: Any) -> Mapping[str, Any]:
+    return value if isinstance(value, Mapping) else {}
+
+
+def _text(value: Any, fallback: str = "—") -> str:
+    if value is None or value == "":
+        return fallback
+    return str(value)
+
+
+def _percent(value: Any) -> str:
+    try:
+        return f"{float(value) * 100:.1f}%"
+    except (TypeError, ValueError):
+        return "—"
+
+
+def _fact(label: str, value: Any, detail: Any) -> str:
+    return (
+        '<div class="mr-col-fact">'
+        f'<span class="mr-col-label">{esc(label)}</span>'
+        f"<strong>{esc(_text(value))}</strong>"
+        f"<small>{esc(_text(detail))}</small>"
+        "</div>"
+    )
+
+
+def _semantics_banner(payload: Mapping[str, Any]) -> str:
+    semantics = _mapping(payload.get("semantics"))
+    message = _text(
+        semantics.get("message"),
+        "This Build declares no column meanings. Nothing here is inferred from column names.",
+    )
+    details: list[str] = []
+    if semantics.get("summary"):
+        details.append(f"<p><strong>Dataset.</strong> {esc(_text(semantics['summary']))}</p>")
+    if semantics.get("grain_statement"):
+        details.append(
+            f"<p><strong>One row.</strong> {esc(_text(semantics['grain_statement']))}</p>"
+        )
+    coverage = semantics.get("coverage")
+    if isinstance(coverage, Mapping):
+        time_range = coverage.get("time_range")
+        if isinstance(time_range, Mapping):
+            details.append(
+                "<p><strong>Time coverage.</strong> "
+                f"{esc(_text(time_range.get('start_inclusive'), 'Not declared'))} (inclusive) "
+                f"to {esc(_text(time_range.get('end_exclusive'), 'Not declared'))} "
+                "(exclusive).</p>"
+            )
+        else:
+            details.append("<p><strong>Time coverage.</strong> Not declared.</p>")
+        details.append(
+            "<p><strong>Population.</strong> "
+            f"{esc(_text(coverage.get('population'), 'Not declared'))}</p>"
+        )
+        details.append(
+            "<p><strong>Completeness.</strong> "
+            f"{esc(_text(coverage.get('completeness_note'), 'Not declared'))}</p>"
+        )
+    limitations = semantics.get("limitations")
+    if isinstance(limitations, list):
+        if limitations:
+            details.append(
+                "<p><strong>Known limitations.</strong> "
+                + esc("; ".join(str(item) for item in limitations))
+                + "</p>"
+            )
+        else:
+            details.append("<p><strong>Known limitations.</strong> None declared.</p>")
+    scope = semantics.get("render_scope")
+    if scope in {"reduced", "minimal"}:
+        omitted = (
+            "coverage, limitations, column descriptions, categorical definitions, and semantic "
+            "evidence references"
+        )
+        if scope == "minimal":
+            omitted = "dataset summary, row grain, column labels, and " + omitted
+        details.append(
+            "<p><strong>Interactive semantic detail truncated.</strong> "
+            f"This bounded view omits {esc(omitted)}. Open the notebook's Overview, Data "
+            "dictionary, HTML fallback, or plain-text fallback for the bounded semantic text "
+            "and explicit omission counts.</p>"
+        )
+    return (
+        '<aside class="mr-semantics-banner">'
+        f"<strong>{esc(message)}</strong>{''.join(details)}"
+        "</aside>"
+    )
+
+
+def _middle_elide(value: str, maximum: int) -> str:
+    """Fit one axis label while preserving its identifying beginning and end."""
+
+    if len(value) <= maximum:
+        return value
+    visible = max(2, maximum - 1)
+    leading = (visible + 1) // 2
+    trailing = visible - leading
+    return value[:leading] + "…" + value[-trailing:]
+
+
+def _chart(profile: Mapping[str, Any]) -> str:
+    chart = _mapping(profile.get("chart"))
+    raw_items = chart.get("items")
+    items = (
+        [item for item in raw_items if isinstance(item, Mapping)]
+        if isinstance(raw_items, list)
+        else []
+    )
+    items = items[:_MAX_CHART_ITEMS]
+    title = _text(chart.get("title"), "Distribution")
+    note = _text(chart.get("note"), "No chart available")
+    if not items:
+        return (
+            '<section class="mr-col-chart mr-col-chart--empty">'
+            f"<header><strong>{esc(title)}</strong><span>{esc(note)}</span></header>"
+            '<div class="mr-col-empty">This column does not need a distribution chart.</div>'
+            "</section>"
+        )
+
+    counts: list[float] = []
+    count_texts: list[str] = []
+    for item in items:
+        raw_count = item.get("count", 0)
+        if isinstance(raw_count, int) and not isinstance(raw_count, bool):
+            bounded_count = max(0, raw_count)
+            counts.append(float(bounded_count))
+            count_texts.append(str(bounded_count))
+            continue
+        try:
+            count = max(0.0, float(raw_count))
+            counts.append(count)
+            count_texts.append(str(int(count)) if count.is_integer() else f"{count:.2f}")
+        except (OverflowError, TypeError, ValueError):
+            counts.append(0.0)
+            count_texts.append("0")
+    maximum = max(counts, default=1.0) or 1.0
+    slot = (CHART_PLOT_RIGHT - CHART_PLOT_LEFT) / max(1, len(items))
+    bar_width = min(CHART_BAR_MAX_WIDTH, max(CHART_BAR_MIN_WIDTH, slot * 0.62))
+    # Every other label shares a row. Two slots are therefore available between peers, while the
+    # symmetric outer margin bounds the first and last labels. The mono-cell estimate is
+    # deliberately conservative for the complete local system-font stack.
+    label_width = min(
+        (2 * slot) - CHART_LABEL_GAP,
+        (2 * CHART_PLOT_LEFT) + slot - CHART_LABEL_GAP,
+    )
+    label_characters = max(3, math.floor(label_width / CHART_LABEL_CHAR_WIDTH))
+    bars: list[str] = []
+    accessible_rows: list[str] = []
+    for index, (item, count, count_text) in enumerate(zip(items, counts, count_texts, strict=True)):
+        x = CHART_PLOT_LEFT + (slot * index) + ((slot - bar_width) / 2)
+        height = 3.0 if count == 0 else max(5.0, (count / maximum) * CHART_PLOT_HEIGHT)
+        y = CHART_BASELINE - height
+        label = _text(item.get("label"), "")
+        short = _middle_elide(label, label_characters)
+        label_baseline = CHART_LABEL_BASELINES[index % len(CHART_LABEL_BASELINES)]
+        rendered_label_width = min(label_width, max(1, len(short)) * CHART_LABEL_CHAR_WIDTH)
+        accessible_rows.append(
+            f'<tr><th scope="row">{esc(label)}</th><td>{esc(count_text)}</td></tr>'
+        )
+        bars.append(
+            "<g>"
+            f"<title>{esc(label)}: {esc(count_text)}</title>"
+            f'<rect x="{x:.2f}" y="{y:.2f}" width="{bar_width:.2f}" '
+            f'height="{height:.2f}" rx="4"></rect>'
+            f'<text class="mr-col-bar-value" x="{x + bar_width / 2:.2f}" y="{y - 8:.2f}" '
+            f'text-anchor="middle">{esc(count_text)}</text>'
+            f'<text class="mr-col-axis-label" x="{x + bar_width / 2:.2f}" '
+            f'y="{label_baseline:.2f}" text-anchor="middle" '
+            f'textLength="{rendered_label_width:.2f}" lengthAdjust="spacingAndGlyphs">'
+            f"{esc(short)}</text>"
+            "</g>"
+        )
+    name = _text(profile.get("name"), "column")
+    return (
+        '<section class="mr-col-chart">'
+        f"<header><strong>{esc(title)}</strong><span>{esc(note)}</span></header>"
+        f'<svg viewBox="0 0 {CHART_VIEWBOX_WIDTH:g} {CHART_VIEWBOX_HEIGHT:g}" role="img" '
+        f'aria-label="{esc(name)} {esc(title.lower())}">'
+        f'<line x1="{CHART_PLOT_LEFT}" y1="{CHART_BASELINE}" '
+        f'x2="{CHART_PLOT_RIGHT}" y2="{CHART_BASELINE}"></line>'
+        f"{''.join(bars)}</svg>"
+        '<table class="mr-col-chart-data">'
+        f"<caption>Full distribution data for {esc(name)}</caption>"
+        "<thead><tr><th>Label</th><th>Count</th></tr></thead>"
+        f"<tbody>{''.join(accessible_rows)}</tbody></table>"
+        "</section>"
+    )
+
+
+def _profile(profile: Mapping[str, Any], *, active: bool) -> str:
+    name = _text(profile.get("name"), "column")
+    label = _text(profile.get("display_label"), name)
+    description = _text(profile.get("description"), "No description was declared.")
+    logical = _text(profile.get("logical_type"), "unknown")
+    physical = _text(profile.get("physical_type"), "unknown")
+    unit = _mapping(profile.get("unit"))
+    unit_value = _text(unit.get("symbol"), _text(unit.get("label"), "Not declared"))
+    unit_detail = _text(unit.get("label"), "No unit metadata")
+    if unit.get("state") == "normalized":
+        reversible = "reversible" if unit.get("reversible") else "not reversible"
+        unit_detail = f"{_text(unit.get('normalization'), 'unknown normalization')} · {reversible}"
+    missing = int(profile.get("missing_count", 0) or 0)
+    distinct = int(profile.get("distinct_count", 0) or 0)
+    rows = int(profile.get("row_count", 0) or 0)
+    summary = _mapping(profile.get("summary"))
+    source = _mapping(profile.get("source"))
+    operations = profile.get("transformations")
+    operations_text = (
+        " · ".join(str(item) for item in operations) if isinstance(operations, list) else ""
+    )
+    source_id = _text(source.get("source_id"), "Not recorded")
+    source_column = _text(source.get("column"), name)
+    hidden = "" if active else " hidden"
+    active_class = " is-active" if active else ""
+    missing_detail = f"{_percent(profile.get('missing_rate'))} of {rows:,} rows"
+    distinct_detail = f"{_percent(profile.get('distinct_rate'))} unique"
+    distinct_status = _text(profile.get("distinct_status"), "exact")
+    if distinct_status != "exact":
+        distinct_detail = f"{distinct_detail} · {distinct_status}"
+    summary_label = _text(summary.get("label"), "Summary")
+    categories = profile.get("categories")
+    category_lines = ""
+    if isinstance(categories, list) and categories:
+        items = "".join(
+            f"<li><code>{esc(_text(item.get('code')))}</code> — "
+            f"{esc(_text(item.get('label')))}</li>"
+            for item in categories[:64]
+            if isinstance(item, Mapping)
+        )
+        category_lines = (
+            '<section class="mr-col-categories"><strong>Value definitions</strong>'
+            f"<ul>{items}</ul><small>"
+            f"{esc(_text(profile.get('categories_status'), 'unknown'))}</small>"
+            "</section>"
+        )
+    refs = profile.get("evidence_refs")
+    evidence_text = " · ".join(
+        f"{_text(item.get('kind'))}: {_text(item.get('ref'))}"
+        for item in refs or []
+        if isinstance(item, Mapping)
+    )
+    return (
+        f'<article class="mr-column-profile{active_class}" '
+        f'data-column-profile="{esc(name)}"{hidden}>'
+        f'<header class="mr-col-semantic"><strong>{esc(label)}</strong>'
+        f"<code>{esc(name)}</code><p>{esc(description)}</p></header>"
+        '<div class="mr-col-facts">'
+        f"{_fact('Type', logical, physical)}"
+        f"{_fact('Unit', unit_value, unit_detail)}"
+        f"{_fact('Missing', missing, missing_detail)}"
+        f"{_fact('Distinct', distinct, distinct_detail)}"
+        f"{_fact(summary_label, summary.get('value'), summary.get('detail'))}"
+        "</div>"
+        f"{_chart(profile)}"
+        f"{category_lines}"
+        '<footer class="mr-col-lineage">'
+        f"<span>Source <strong>{esc(source_id)}.{esc(source_column)}</strong></span>"
+        f"<span>Transformations <strong>{esc(operations_text or 'None')}</strong></span>"
+        f"<span>Semantic evidence <strong>{esc(evidence_text or 'Not declared')}</strong></span>"
+        "</footer>"
+        "</article>"
+    )
+
+
+def render_column_profiles(out: Output, _ctx: RenderContext) -> str:
+    """Render the all-column inspector from its structured, versioned MIME payload."""
+
+    raw_payload = out.data.get(MIME)
+    if not is_valid_column_profiles_payload(raw_payload):
+        raise ValueError("invalid column-profile MIME payload")
+    payload = _mapping(raw_payload)
+    raw_columns = payload.get("columns")
+    columns = (
+        [column for column in raw_columns if isinstance(column, Mapping)]
+        if isinstance(raw_columns, list)
+        else []
+    )
+    if not columns:
+        return '<div class="mr-columns-empty">No column profiles were recorded.</div>'
+    options = "".join(
+        f'<option value="{esc(_text(column.get("name"), "column"))}">'
+        f"{esc(_text(column.get('display_label'), _text(column.get('name'), 'column')))}</option>"
+        for column in columns
+    )
+    profiles = "".join(_profile(column, active=index == 0) for index, column in enumerate(columns))
+    first_title = _text(columns[0].get("display_label"), _text(columns[0].get("name"), "column"))
+    return (
+        '<section class="mr-columns" data-column-inspector>'
+        f"{_semantics_banner(payload)}"
+        '<header class="mr-columns-head">'
+        "<div><span>Column inspector</span><strong data-column-title>"
+        f"{esc(first_title)}</strong></div>"
+        "<label><span>Inspect column</span>"
+        f'<select data-column-select aria-label="Inspect column">{options}</select></label>'
+        "</header>"
+        f"{profiles}"
+        "</section>"
+    )