mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/notebook.py

@@ -0,0 +1,1710 @@
+"""Deterministic nbformat-4.5 visual sidecar for a sealed candidate.
+
+The generated ``table.ipynb`` is useful without a kernel: it embeds a data preview and an
+all-column analytics inspector as normal ``display_data`` outputs. The inspector's versioned rich
+MIME payload includes logical and physical types, declared units, missingness, cardinality,
+type-appropriate summaries and distributions, lineage, and transformations. Portable
+``text/html`` and ``text/plain`` fallbacks keep the notebook useful in ordinary Jupyter viewers.
+
+Runnable pandas cells remain available for local exploration. Generated code cells carry
+nbclient's ``skip-execution`` tag, preserving the deterministic pre-filled outputs during headless
+execution while remaining manually runnable in an interactive kernel.
+
+The generator is deterministic: every cell ``id`` is the first 12 hex of ``sha256(cell_source)``
+and every value is a native Python scalar, so two renders of the same sealed candidate are
+byte-identical. The provenance block is derived the same way — no clock, no randomness.
+
+It adds NO new dependency: the notebook is nbformat 4.5 JSON built with the stdlib ``json``, and
+the head-N Parquet sample is read through the already-pinned ``pyarrow`` (imported lazily). It never
+imports ``nbformat``/``nbconvert``.
+
+The sidecar is written as a run-dir SIBLING of ``candidate/`` (the run dir stays owner-writable
+``0o700``; only ``candidate/`` is sealed ``0o555``/``0o444``), so it never enters the sealed member
+set, never moves a digest, and is invisible to ``verify_candidate`` (which polices only
+``candidate/``).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import html
+import json
+import math
+import os
+import secrets
+import stat
+from collections import Counter
+from copy import deepcopy
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - secure candidate verification already refuses Windows
+    fcntl = None  # type: ignore[assignment]
+
+from mostlyright.data_harness import pipeline
+from mostlyright.data_harness import recipe as recipe_contracts
+from mostlyright.data_harness.local_contracts import UNIT_STATES
+from mostlyright.data_harness.preparation.contracts import PHYSICAL_TYPES
+from mostlyright.data_harness.recipe import LOGICAL_TYPES, UNITS
+from mostlyright.data_harness.units import is_declarable_unit, legacy_aliases
+
+# The one name this module writes the generated notebook under. It is a name other lanes have
+# to be able to say without writing it themselves -- the hosted handoff reports the path it
+# installed and never opens it -- so it is a constant rather than four literals.
+TABLE_NOTEBOOK_NAME = "table.ipynb"
+# Private compatibility import for the hosted handoff lane added before the V3 vocabulary cutover.
+# It names the same Table sidecar and never crosses a wire or UI boundary.
+DATASET_NOTEBOOK_NAME = TABLE_NOTEBOOK_NAME
+_SAMPLE_ROWS = 10
+_NBFORMAT_MAJOR = 4
+_NBFORMAT_MINOR = 5
+_PROVENANCE_DIGEST_PREFIX = 12
+# ``nbclient``'s default ``skip_cells_with_tag``. Every generated code cell carries it, so the
+# headless execute path skips them and the hand-built display outputs survive untouched.
+_SKIP_EXECUTION_TAG = "skip-execution"
+_COLUMN_PROFILE_MIME = "application/vnd.mostlyright.column-profile.v1+json"
+_PROFILE_SAMPLE_ROWS = 100_000
+_MAX_CATEGORY_BARS = 12
+_FALLBACK_MAX_COLUMNS = 64
+_FALLBACK_MAX_LIMITATIONS = 16
+_FALLBACK_MAX_CATEGORIES = 16
+_FALLBACK_TEXT_LIMIT = 512
+
+_UNIT_DISPLAY: dict[str, tuple[str | None, str]] = {
+    "none": (None, "Not applicable"),
+    "count": ("count", "count"),
+    "ratio": ("ratio", "unitless ratio"),
+    "percent": ("%", "percent"),
+    "celsius": ("°C", "degrees Celsius"),
+    "fahrenheit": ("°F", "degrees Fahrenheit"),
+    "kelvin": ("K", "kelvin"),
+    "meter": ("m", "meters"),
+    "kilometer": ("km", "kilometers"),
+    "mile": ("mi", "miles"),
+    "microgram_per_cubic_meter": ("µg/m³", "micrograms per cubic meter"),
+    "second": ("s", "seconds"),
+    "minute": ("min", "minutes"),
+    "hour": ("h", "hours"),
+    "hectopascal": ("hPa", "hectopascals"),
+}
+# These are the written-out labels for the fifteen tokens the vocabulary began with, not a second
+# vocabulary. Import-time equality holds them honest: an alias added to the pinned table without a
+# label here is an explicit implementation failure rather than a silent gap.
+assert set(_UNIT_DISPLAY) == set(UNITS)
+# The same fifteen labels, reached from the codes those names stand for. ``celsius`` and ``Cel``
+# are one unit, so the page a person reads has to say so: without this, adopting the spelling the
+# unit grammar promotes would silently downgrade "degrees Celsius" to "Cel".
+_CODE_DISPLAY: dict[str, tuple[str | None, str]] = {
+    code: _UNIT_DISPLAY[token] for token, code in legacy_aliases().items()
+}
+_UNIT_STATE_DISPLAY = {
+    "declared": "Declared physical unit",
+    "normalized": "Normalized (dimensionless)",
+    "not_applicable": "Not applicable",
+    "unknown": "Unit not recoverable",
+}
+assert set(_UNIT_STATE_DISPLAY) == set(UNIT_STATES)
+assert set(PHYSICAL_TYPES) == set(LOGICAL_TYPES)
+
+
+# --- nbformat 4.5 cell primitives --------------------------------------------------------------
+
+
+def _cell_id(source: str) -> str:
+    """First 12 hex of ``sha256(source)`` — a content-derived, stable, deterministic cell id."""
+
+    return hashlib.sha256(source.encode("utf-8")).hexdigest()[:12]
+
+
+def _markdown_cell(source: str) -> dict[str, Any]:
+    return {
+        "cell_type": "markdown",
+        "id": _cell_id(source),
+        "metadata": {},
+        "source": source,
+    }
+
+
+def _code_cell(source: str, outputs: tuple[dict[str, Any], ...] = ()) -> dict[str, Any]:
+    """A generated code cell, tagged ``skip-execution`` so a headless execute leaves it alone.
+
+    The tag is cell METADATA, not cell source, so the content-derived ``id`` is unaffected and the
+    render stays byte-deterministic.
+    """
+
+    return {
+        "cell_type": "code",
+        "execution_count": None,
+        "id": _cell_id(source),
+        "metadata": {"tags": [_SKIP_EXECUTION_TAG]},
+        "outputs": list(outputs),
+        "source": source,
+    }
+
+
+def _display_html(markup: str) -> dict[str, Any]:
+    return {"output_type": "display_data", "data": {"text/html": markup}, "metadata": {}}
+
+
+def _fallback_text(value: Any, *, limit: int = _FALLBACK_TEXT_LIMIT) -> tuple[str, bool]:
+    """Return deterministic bounded text and whether an explicit truncation marker was added."""
+
+    rendered = "" if value is None else str(value)
+    if len(rendered) <= limit:
+        return rendered, False
+    marker = " [truncated]"
+    return rendered[: max(0, limit - len(marker))] + marker, True
+
+
+def _bounded_items(value: Any, maximum: int) -> tuple[list[Any], int]:
+    items = value if isinstance(value, list) else []
+    return items[:maximum], max(0, len(items) - maximum)
+
+
+def _fallback_semantic_sections(payload: dict[str, Any]) -> tuple[str, list[str]]:
+    """Render bounded deterministic dataset and column semantics for portable fallbacks."""
+
+    semantics = payload.get("semantics") if isinstance(payload.get("semantics"), dict) else {}
+    message, _ = _fallback_text(
+        semantics.get(
+            "message",
+            "This Build declares no column meanings. Nothing here is inferred from column names.",
+        )
+    )
+    summary, _ = _fallback_text(semantics.get("summary") or "Not declared")
+    grain, _ = _fallback_text(semantics.get("grain_statement") or "Not declared")
+    coverage = semantics.get("coverage") if isinstance(semantics.get("coverage"), dict) else {}
+    time_range = (
+        coverage.get("time_range") if isinstance(coverage.get("time_range"), dict) else None
+    )
+    if time_range is None:
+        time_text = "Not declared"
+    else:
+        start, _ = _fallback_text(time_range.get("start_inclusive") or "Not declared")
+        end, _ = _fallback_text(time_range.get("end_exclusive") or "Not declared")
+        time_text = f"{start} (inclusive) to {end} (exclusive)"
+    population, _ = _fallback_text(coverage.get("population") or "Not declared")
+    completeness, _ = _fallback_text(coverage.get("completeness_note") or "Not declared")
+
+    limitations, omitted_limitations = _bounded_items(
+        semantics.get("limitations"), _FALLBACK_MAX_LIMITATIONS
+    )
+    limitation_texts = [_fallback_text(item)[0] for item in limitations]
+    if not limitation_texts:
+        limitation_texts = ["None declared"]
+
+    html_parts = [
+        '<section class="mostlyright-table-semantics">',
+        "<h3>Table semantics</h3>",
+        f"<p>{html.escape(message)}</p>",
+        f"<p><strong>Summary:</strong> {html.escape(summary)}</p>",
+        f"<p><strong>One row:</strong> {html.escape(grain)}</p>",
+        f"<p><strong>Time coverage:</strong> {html.escape(time_text)}</p>",
+        f"<p><strong>Population:</strong> {html.escape(population)}</p>",
+        f"<p><strong>Completeness:</strong> {html.escape(completeness)}</p>",
+        "<h4>Known limitations</h4><ul>",
+        *(f"<li>{html.escape(item)}</li>" for item in limitation_texts),
+    ]
+    plain = [
+        "Table semantics",
+        message,
+        f"Summary: {summary}",
+        f"One row: {grain}",
+        f"Time coverage: {time_text}",
+        f"Population: {population}",
+        f"Completeness: {completeness}",
+        "Known limitations:",
+        *(f"- {item}" for item in limitation_texts),
+    ]
+    if omitted_limitations:
+        notice = f"[truncated: {omitted_limitations} additional limitations omitted]"
+        html_parts.append(f"<li>{html.escape(notice)}</li>")
+        plain.append(notice)
+    html_parts.extend(["</ul>", "</section>"])
+
+    columns = payload.get("columns") if isinstance(payload.get("columns"), list) else []
+    displayed_columns = columns[:_FALLBACK_MAX_COLUMNS]
+    omitted_columns = max(0, len(columns) - _FALLBACK_MAX_COLUMNS)
+    header = (
+        "<tr><th>column</th><th>label</th><th>meaning</th><th>type</th><th>unit / scaling</th>"
+        "<th>categorical definitions</th><th>evidence</th></tr>"
+    )
+    rows: list[str] = []
+    plain.extend(["", "Column semantics"])
+    for column in displayed_columns:
+        if not isinstance(column, dict):
+            continue
+        name, _ = _fallback_text(column.get("name", ""), limit=120)
+        label, _ = _fallback_text(column.get("display_label") or "Not declared", limit=120)
+        description, _ = _fallback_text(column.get("description") or "Not declared")
+        logical, _ = _fallback_text(column.get("logical_type", "unknown"), limit=64)
+        physical, _ = _fallback_text(column.get("physical_type", "unknown"), limit=64)
+        unit = column.get("unit") if isinstance(column.get("unit"), dict) else {}
+        unit_text, _ = _fallback_text(unit.get("symbol") or unit.get("label") or "not declared")
+        if unit.get("state") == "normalized":
+            normalization, _ = _fallback_text(unit.get("normalization") or "unknown", limit=64)
+            reversible = "reversible" if unit.get("reversible") else "not reversible"
+            unit_text = f"{unit_text}; {normalization}; {reversible}"
+
+        categories, omitted_categories = _bounded_items(
+            column.get("categories"), _FALLBACK_MAX_CATEGORIES
+        )
+        category_parts = []
+        for item in categories:
+            if isinstance(item, dict):
+                code, _ = _fallback_text(item.get("code"), limit=64)
+                category_label, _ = _fallback_text(item.get("label"), limit=120)
+                category_parts.append(f"{code} = {category_label}")
+        category_status = str(column.get("categories_status") or "not declared").replace("_", " ")
+        category_text = (
+            f"{category_status}: {'; '.join(category_parts)}" if category_parts else category_status
+        )
+        if omitted_categories:
+            category_text += f"; [truncated: {omitted_categories} additional definitions omitted]"
+
+        refs, omitted_refs = _bounded_items(column.get("evidence_refs"), 8)
+        evidence_parts = [
+            f"{item.get('kind')}: {item.get('ref')}" for item in refs if isinstance(item, dict)
+        ]
+        evidence_text = "; ".join(evidence_parts) or "No evidence reference declared"
+        if omitted_refs:
+            evidence_text += f"; [truncated: {omitted_refs} additional references omitted]"
+
+        rows.append(
+            "<tr>"
+            f"<td>{html.escape(name)}</td><td>{html.escape(label)}</td>"
+            f"<td>{html.escape(description)}</td>"
+            f"<td>{html.escape(logical)} / {html.escape(physical)}</td>"
+            f"<td>{html.escape(unit_text)}</td>"
+            f"<td>{html.escape(category_text)}</td>"
+            f"<td>{html.escape(evidence_text)}</td></tr>"
+        )
+        plain.extend(
+            [
+                f"- {name} — {label}",
+                f"  Meaning: {description}",
+                f"  Type: {logical} / {physical}",
+                f"  Unit / scaling: {unit_text}",
+                f"  Categorical definitions: {category_text}",
+                f"  Evidence: {evidence_text}",
+            ]
+        )
+    if omitted_columns:
+        notice = f"[truncated: {omitted_columns} additional columns omitted]"
+        rows.append(f'<tr><td colspan="7">{html.escape(notice)}</td></tr>')
+        plain.append(notice)
+    html_parts.extend(
+        [
+            '<section class="mostlyright-column-semantics">',
+            "<h3>Column semantics</h3>",
+            '<table border="1" class="dataframe"><thead>',
+            header,
+            f"</thead><tbody>{''.join(rows)}</tbody></table>",
+            "</section>",
+        ]
+    )
+    return "".join(html_parts), plain
+
+
+def _display_column_profiles(
+    payload: dict[str, Any], *, fallback_payload: dict[str, Any] | None = None
+) -> dict[str, Any]:
+    """Portable custom output with bounded, complete semantic fallbacks."""
+
+    raw_columns = payload.get("columns", [])
+    columns = raw_columns[:_FALLBACK_MAX_COLUMNS] if isinstance(raw_columns, list) else []
+    semantic_html, plain = _fallback_semantic_sections(fallback_payload or payload)
+    header = (
+        "<tr><th>column</th><th>label</th><th>logical</th><th>physical</th><th>unit state</th>"
+        "<th>missing</th><th>distinct</th></tr>"
+    )
+    rows: list[str] = []
+    plain.extend(["", "Column inspector"])
+    for column in columns if isinstance(columns, list) else []:
+        if not isinstance(column, dict):
+            continue
+        unit = column.get("unit") if isinstance(column.get("unit"), dict) else {}
+        unit_text = unit.get("symbol") or unit.get("label") or "not declared"
+        name = str(column.get("name", ""))
+        label = str(column.get("display_label") or "Not declared")
+        logical = str(column.get("logical_type", "unknown"))
+        physical = str(column.get("physical_type", "unknown"))
+        missing = int(column.get("missing_count", 0) or 0)
+        distinct = int(column.get("distinct_count", 0) or 0)
+        rows.append(
+            "<tr>"
+            f"<td>{html.escape(name)}</td><td>{html.escape(label)}</td>"
+            f"<td>{html.escape(logical)}</td>"
+            f"<td>{html.escape(physical)}</td><td>{html.escape(str(unit_text))}</td>"
+            f"<td>{missing}</td><td>{distinct}</td></tr>"
+        )
+        plain.append(
+            f"- {name} ({label}): {logical} / {physical}; unit {unit_text}; "
+            f"{missing} missing; {distinct} distinct"
+        )
+    omitted_inspector_columns = max(
+        0, len(raw_columns) - _FALLBACK_MAX_COLUMNS if isinstance(raw_columns, list) else 0
+    )
+    if omitted_inspector_columns:
+        notice = f"[truncated: {omitted_inspector_columns} additional profiles omitted]"
+        rows.append(f'<tr><td colspan="7">{html.escape(notice)}</td></tr>')
+        plain.append(notice)
+    fallback = (
+        '<div class="mostlyright-column-profile">'
+        f"{semantic_html}"
+        "<p><strong>Column inspector</strong> — open this notebook in Mostly Right for charts "
+        "and interactive column selection.</p>"
+        '<table border="1" class="dataframe"><thead>'
+        f"{header}</thead><tbody>{''.join(rows)}</tbody></table></div>"
+    )
+    return {
+        "output_type": "display_data",
+        "data": {
+            _COLUMN_PROFILE_MIME: payload,
+            "text/html": fallback,
+            "text/plain": "\n".join(plain),
+        },
+        "metadata": {},
+    }
+
+
+# --- hand-built, kernel-free display outputs ---------------------------------------------------
+
+
+def _cell_text(value: Any) -> str:
+    return "" if value is None else str(value)
+
+
+def _html_table(rows: list[dict[str, Any]], columns: list[str]) -> dict[str, Any]:
+    """A hand-built HTML ``<table>`` of the head-N sample rows (every cell HTML-escaped)."""
+
+    # Match pandas' structural index signal: a blank leading header and a row-header ``th``. The
+    # renderer can then distinguish the index from real data columns without guessing from values.
+    header = "<th></th>" + "".join(f"<th>{html.escape(str(column))}</th>" for column in columns)
+    body_rows: list[str] = []
+    for index, row in enumerate(rows):
+        cells = "".join(
+            f"<td>{html.escape(_cell_text(row.get(column)))}</td>" for column in columns
+        )
+        body_rows.append(f"<tr><th>{index}</th>{cells}</tr>")
+    if not body_rows:
+        body_rows.append(
+            f'<tr><th></th><td colspan="{max(1, len(columns))}"><em>no rows</em></td></tr>'
+        )
+    markup = (
+        '<table border="1" class="dataframe">'
+        f"<thead><tr>{header}</tr></thead>"
+        f"<tbody>{''.join(body_rows)}</tbody>"
+        "</table>"
+    )
+    return _display_html(markup)
+
+
+# --- markdown evidence sections ----------------------------------------------------------------
+
+
+def _esc(value: Any) -> str:
+    """HTML-escape an evidence-derived value before interpolating it into a markdown cell.
+
+    These cells are rendered to HTML by the viewer's ``HTMLExporter``, which passes raw HTML in
+    markdown through live — so every evidence-derived value (question, profiled types, lineage ops,
+    quality rows, join facts, source origin/rights) is escaped here, exactly as the sample-row table
+    and SVG labels already escape. Escaping is safe in every markdown context (code span, table
+    cell, or bold flow): an escaped ``&lt;`` never re-opens as a tag even if the value breaks a
+    backtick.
+    """
+
+    return html.escape("" if value is None else str(value))
+
+
+def _header_markdown(question: str, row_count: int, columns: int) -> str:
+    title = question.strip().rstrip("?") or "Research dataset"
+    return f"# {_esc(title)}\n\n{row_count:,} rows · {columns} columns."
+
+
+def _overview_markdown(semantics: dict[str, Any]) -> str:
+    lines = [
+        "## Dataset contract",
+        "",
+        _esc(semantics.get("message", "Semantic status unavailable.")),
+    ]
+    summary = semantics.get("summary")
+    grain = semantics.get("grain_statement")
+    coverage = semantics.get("coverage")
+    if isinstance(summary, str):
+        lines.extend(["", "### Definition", "", _esc(summary)])
+    else:
+        lines.extend(["", "### Definition", "", "No dataset-level summary was declared."])
+    if isinstance(grain, str):
+        lines.append(f"**Row grain:** {_esc(grain)}")
+    else:
+        lines.append("**Row grain:** Not declared.")
+    lines.extend(["", "### Coverage", ""])
+    if isinstance(coverage, dict):
+        time_range = coverage.get("time_range")
+        if isinstance(time_range, dict):
+            lines.append(
+                "**Time:** "
+                f"{_esc(time_range.get('start_inclusive'))} (inclusive) to "
+                f"{_esc(time_range.get('end_exclusive'))} (exclusive)."
+            )
+        else:
+            lines.append("**Time:** Not declared.")
+        lines.append(f"**Population:** {_esc(coverage.get('population'))}")
+        lines.append(f"**Completeness:** {_esc(coverage.get('completeness_note'))}")
+    limitations = semantics.get("limitations")
+    lines.extend(["", "### Known limitations", ""])
+    if isinstance(limitations, list) and limitations:
+        lines.extend(f"- {_esc(item)}" for item in limitations)
+    elif isinstance(limitations, list):
+        lines.append("No limitations were declared.")
+    else:
+        lines.append("No limitations statement was declared.")
+    return "\n".join(lines)
+
+
+def _dictionary_markdown(
+    column_names: list[str],
+    semantics_by_name: dict[str, dict[str, Any]],
+    logical_by_name: dict[str, str],
+) -> str:
+    headings = ("Column", "Label", "Meaning", "Type", "Unit / scaling", "Values", "Evidence")
+    rows: list[str] = []
+
+    def table_value(value: Any) -> str:
+        return (
+            " ".join(("" if value is None else str(value)).split())
+            .replace("\\", r"\\")
+            .replace("|", r"\|")
+        )
+
+    for name in column_names:
+        record = semantics_by_name.get(name)
+        unit = _unit_metadata(record)
+        unit_text = unit["label"]
+        if unit.get("state") == "normalized":
+            reversible = "reversible" if unit.get("reversible") else "not reversible"
+            unit_text = f"{unit_text}: {unit.get('normalization')} ({reversible})"
+        categories = record.get("categories") if isinstance(record, dict) else None
+        category_status = (
+            record.get("categories_status", "unknown") if isinstance(record, dict) else "unknown"
+        )
+        category_text = str(category_status).replace("_", " ")
+        if isinstance(categories, list) and categories:
+            definitions = "; ".join(
+                f"{item.get('code')} = {item.get('label')}"
+                for item in categories
+                if isinstance(item, dict)
+            )
+            category_text = f"{category_text}: {definitions}"
+        refs = record.get("evidence_refs") if isinstance(record, dict) else None
+        evidence = "; ".join(
+            f"{item.get('kind')}: {item.get('ref')}"
+            for item in refs or []
+            if isinstance(item, dict)
+        )
+        values = (
+            name,
+            record.get("display_label", "Not declared")
+            if isinstance(record, dict)
+            else "Not declared",
+            record.get("description", "Not declared")
+            if isinstance(record, dict)
+            else "Not declared",
+            logical_by_name.get(name, "unknown"),
+            unit_text,
+            category_text,
+            evidence or "No evidence reference declared",
+        )
+        rows.append("| " + " | ".join(table_value(value) for value in values) + " |")
+    header = "| " + " | ".join(headings) + " |"
+    rule = "| " + " | ".join("---" for _heading in headings) + " |"
+    return "\n".join(("### Data dictionary", "", header, rule, *rows))
+
+
+def _quality_markdown(quality: dict[str, Any]) -> str:
+    checks = quality.get("checks", []) if isinstance(quality, dict) else []
+    passed = sum(bool(check.get("passed")) for check in checks if isinstance(check, dict))
+    failed = [check for check in checks if isinstance(check, dict) and not check.get("passed")]
+    lines = [
+        "## Validation",
+        "",
+        "### Build checks",
+        "",
+        f"{passed} of {len(checks)} checks passed.",
+    ]
+    for check in failed:
+        lines.append(f"- {_esc(check.get('check_id', 'Check'))}: needs attention")
+    return "\n".join(lines)
+
+
+def _join_markdown(join: dict[str, Any]) -> str:
+    keys = ", ".join(f"`{_esc(key)}`" for key in join.get("keys", [])) or "—"
+    lines = [
+        "### Join",
+        "",
+        f"- **cardinality:** {_esc(join.get('cardinality', '—'))}",
+        f"- **keys:** {keys}",
+        f"- **left:** `{_esc(join.get('left_source', '—'))}` "
+        f"({_esc(join.get('left_rows', '—'))} rows)",
+        f"- **right:** `{_esc(join.get('right_source', '—'))}` "
+        f"({_esc(join.get('right_rows', '—'))} rows)",
+        f"- **output rows:** {_esc(join.get('output_rows', '—'))}",
+        f"- **unmatched left rows:** {_esc(join.get('unmatched_left_rows', '—'))}",
+    ]
+    return "\n".join(lines)
+
+
+def _operations_markdown(evidence: dict[str, Any]) -> str:
+    nodes = evidence.get("nodes", []) if isinstance(evidence, dict) else []
+    lines = ["## How this dataset was built", "", "### Deterministic operations", ""]
+    for node in nodes if isinstance(nodes, list) else []:
+        if not isinstance(node, dict):
+            continue
+        inputs = ", ".join(f"`{_esc(item)}`" for item in node.get("inputs", [])) or "source"
+        lines.append(
+            f"- **{_esc(node.get('node_id', 'node'))}** — "
+            f"`{_esc(node.get('operation', 'operation'))}` from {inputs}; "
+            f"{_esc(node.get('input_rows', '—'))} → {_esc(node.get('output_rows', '—'))} rows"
+        )
+        parameters = node.get("parameters")
+        if isinstance(parameters, dict):
+            rendered = json.dumps(
+                parameters,
+                ensure_ascii=False,
+                sort_keys=True,
+                separators=(",", ":"),
+            )
+            markdown_safe = rendered.replace("`", "\\u0060")
+            lines.append(f"  - parameters: `{markdown_safe}`")
+    inspectable = evidence.get("schema_version") == pipeline.GRAPH_OPERATION_EVIDENCE_VERSION
+    lines.extend(
+        [
+            "",
+            (
+                "Exact parameters, ordering, lineage, engine identity, and logical digests are "
+                "sealed in `evidence/operations.json`."
+                if inspectable
+                else "Parameter digests, ordering, lineage, engine identity, and logical digests "
+                "are sealed in `evidence/operations.json`."
+            ),
+        ]
+    )
+    return "\n".join(lines)
+
+
+def _sources_markdown(sources: tuple[dict[str, Any], ...]) -> str:
+    lines = ["## Provenance", "", "### Sources", ""]
+    for source in sources:
+        source = source if isinstance(source, dict) else {}
+        rights = source.get("rights", {})
+        status = rights.get("status", "unspecified") if isinstance(rights, dict) else "unspecified"
+        origin = source.get("origin", "")
+        lines.append(
+            f"- **{_esc(source.get('source_id', 'source'))}** — {_esc(origin) or 'local source'} "
+            f"({_esc(status).replace('_', ' ')})"
+        )
+    return "\n".join(lines)
+
+
+def _provenance_metadata(inspection: Any) -> dict[str, str]:
+    """Build ``metadata.mostlyright.provenance`` from the rendered evidence members.
+
+    Four string fields in renderer order — ``source``, ``join``, ``revision``, ``cache`` — each
+    derived from the already-read build reports (``evidence/sources.json``, ``evidence/join.json``,
+    the verified manifest identity). No clock and no randomness: two renders of the same candidate
+    produce the same block, exactly like the cells. Values are raw text, not HTML-escaped — a
+    renderer escapes metadata at interpolation, so escaping here would double-encode.
+    """
+
+    source_ids = [
+        str(source.get("source_id", ""))
+        for source in inspection.sources
+        if isinstance(source, dict) and source.get("source_id")
+    ]
+    join = inspection.join if isinstance(inspection.join, dict) else {}
+    if join.get("schema_version") in {
+        pipeline.GRAPH_OPERATION_EVIDENCE_V1,
+        pipeline.GRAPH_OPERATION_EVIDENCE_VERSION,
+    }:
+        nodes = join.get("nodes", [])
+        join_text = f"{len(nodes) if isinstance(nodes, list) else 0} graph operations"
+    else:
+        keys = [str(key) for key in (join.get("keys") or [])]
+        cardinality = str(join.get("cardinality") or "")
+        if cardinality and keys:
+            join_text = f"{cardinality} on {', '.join(keys)}"
+        else:
+            join_text = cardinality or "none"
+    snapshots = len(source_ids)
+    noun = "snapshot" if snapshots == 1 else "snapshots"
+    digest = inspection.candidate_digest[:_PROVENANCE_DIGEST_PREFIX]
+    return {
+        "source": ", ".join(source_ids) or "none",
+        "join": join_text,
+        # Digest first: a renderer dims the parenthetical tail, and the identity is the digest —
+        # the manifest version is the format constant.
+        "revision": f"{digest} ({inspection.manifest_version})",
+        "cache": f"{snapshots} raw {noun} in raw/",
+    }
+
+
+# --- all-column analytics ---------------------------------------------------------------------
+
+
+def _number(value: int | float | Decimal) -> str:
+    """Format a number without first narrowing an integer through binary64."""
+
+    if isinstance(value, int):
+        return f"{value:,}"
+    if isinstance(value, Decimal):
+        if not value.is_finite():
+            return "Not finite"
+        # Keep exact fixed-point rendering across the complete int64 range, while avoiding a
+        # hundreds-of-characters label if a binary64-derived Decimal reaches an extreme exponent.
+        if value and (value.adjusted() >= 20 or value.adjusted() <= -5):
+            return _trim_scientific(format(value, ".4g"))
+        if value == value.to_integral_value():
+            return f"{int(value):,}"
+        return format(value, ",f")
+    if not math.isfinite(value):
+        return "Not finite"
+    # ``float.is_integer`` is also true for 1e308. Expanding that value through ``int`` creates a
+    # 309-character chart label, so reserve exact-looking grouped integers for ordinary magnitudes.
+    if value and (abs(value) >= 1e20 or abs(value) < 1e-4):
+        return _trim_scientific(f"{value:.4g}")
+    if math.isfinite(value) and value.is_integer():
+        return f"{int(value):,}"
+    return f"{value:,.4g}"
+
+
+def _trim_scientific(value: str) -> str:
+    """Normalize a compact general-format number without changing its significant digits."""
+
+    rendered = value.lower()
+    if "e" not in rendered:
+        return rendered
+    mantissa, exponent = rendered.split("e", 1)
+    mantissa = mantissa.rstrip("0").rstrip(".")
+    return f"{mantissa}e{exponent}"
+
+
+def _unit_metadata(semantics: dict[str, Any] | None) -> dict[str, Any]:
+    if semantics is None:
+        return {
+            "value": None,
+            "symbol": None,
+            "label": "Not declared",
+            "status": "unknown",
+            "state": "unknown",
+            "normalization": None,
+            "reversible": None,
+        }
+    raw = str(semantics.get("unit", "none"))
+    semantic_type = str(semantics.get("semantic_type", ""))
+    state = semantics.get("unit_state")
+    if state not in UNIT_STATES:
+        state = (
+            "declared"
+            if raw != "none"
+            else ("unknown" if semantic_type == "measure" else "not_applicable")
+        )
+    if state == "normalized":
+        return {
+            "value": raw,
+            "symbol": None,
+            "label": _UNIT_STATE_DISPLAY[state],
+            "status": state,
+            "state": state,
+            "normalization": semantics.get("normalization"),
+            "reversible": semantics.get("reversible"),
+        }
+    if state == "not_applicable":
+        return {
+            "value": raw,
+            "symbol": None,
+            "label": _UNIT_STATE_DISPLAY[state],
+            "status": "not_applicable",
+            "state": state,
+            "normalization": None,
+            "reversible": None,
+        }
+    if state == "unknown":
+        return {
+            "value": raw,
+            "symbol": None,
+            "label": _UNIT_STATE_DISPLAY[state],
+            "status": "unknown",
+            "state": state,
+            "normalization": None,
+            "reversible": None,
+        }
+    # A declared unit that is neither one of the written-out names nor a code the grammar resolves
+    # cannot have got past Recipe validation. Keep this branch fail-visible for a caller building a
+    # fixture without going through that validation.
+    if raw not in _UNIT_DISPLAY and raw not in _CODE_DISPLAY and not is_declarable_unit(raw):
+        return {
+            "value": raw,
+            "symbol": None,
+            "label": "Unknown unit",
+            "status": "unknown",
+            "state": "unknown",
+            "normalization": None,
+            "reversible": None,
+        }
+    symbol, label = _UNIT_DISPLAY.get(raw) or _CODE_DISPLAY.get(raw) or (raw, raw)
+    return {
+        "value": raw,
+        "symbol": symbol,
+        "label": label,
+        "status": "declared",
+        "state": "declared",
+        "normalization": None,
+        "reversible": None,
+    }
+
+
+def _value_text(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, float):
+        if math.isnan(value):
+            return "NaN"
+        # ``repr`` is the shortest round-trippable spelling. Category labels therefore cannot
+        # merge near-equal binary64 values merely because the chart uses compact typography.
+        if value == 0.0:
+            return "0.0"
+        return repr(value)
+    return str(value)
+
+
+def _analysis_note(analysis_rows: int, total_rows: int) -> str:
+    if analysis_rows >= total_rows:
+        return f"{analysis_rows:,} observations · all rows"
+    return f"{analysis_rows:,} observations · first {analysis_rows:,} of {total_rows:,} rows"
+
+
+def _histogram(
+    values: list[int | float],
+    *,
+    unit_symbol: str | None,
+    analysis_rows: int,
+    total_rows: int,
+) -> dict[str, Any]:
+    if not values:
+        return {"title": "Distribution", "note": "No non-missing values", "items": []}
+    minimum, maximum = min(values), max(values)
+    suffix = f" {unit_symbol}" if unit_symbol else ""
+    if minimum == maximum:
+        return {
+            "title": "Distribution",
+            "note": _analysis_note(analysis_rows, total_rows),
+            "items": [{"label": f"{_number(minimum)}{suffix}", "count": len(values)}],
+        }
+    bins = min(8, max(4, round(math.sqrt(len(values)))))
+    counts = [0] * bins
+    items = []
+    if all(isinstance(value, int) for value in values):
+        # Integer arithmetic throughout: neither +/-2**53 nor int64 extrema are rounded through a
+        # float. Inclusive integer buckets are gap-free and cover both extrema exactly.
+        integer_minimum = int(minimum)
+        integer_maximum = int(maximum)
+        span = integer_maximum - integer_minimum + 1
+        bins = min(bins, span)
+        counts = [0] * bins
+        for value in values:
+            index = min(bins - 1, ((int(value) - integer_minimum) * bins) // span)
+            counts[index] += 1
+        for index, count in enumerate(counts):
+            low = integer_minimum + ((span * index + bins - 1) // bins)
+            high = integer_minimum + ((span * (index + 1) + bins - 1) // bins) - 1
+            items.append({"label": f"{_number(low)} to {_number(high)}{suffix}", "count": count})
+    else:
+        float_minimum = float(minimum)
+        float_maximum = float(maximum)
+        crosses_zero = float_minimum < 0.0 < float_maximum
+        if crosses_zero:
+            # Scaling first keeps the normalized span in [1, 2]. Directly subtracting valid
+            # binary64 extrema (for example -1e308 and 1e308) would overflow to infinity.
+            scale = max(-float_minimum, float_maximum)
+            scaled_minimum = float_minimum / scale
+            scaled_maximum = float_maximum / scale
+            scaled_span = scaled_maximum - scaled_minimum
+
+            def position(value: int | float) -> float:
+                return ((float(value) / scale) - scaled_minimum) / scaled_span
+
+            def boundary(fraction: float) -> float:
+                # A convex combination never exceeds either finite endpoint. Unlike
+                # ``minimum + fraction * (maximum - minimum)``, it needs no overflowing span.
+                return float_minimum * (1.0 - fraction) + float_maximum * fraction
+
+        else:
+            span = float_maximum - float_minimum
+
+            def position(value: int | float) -> float:
+                return (float(value) - float_minimum) / span
+
+            def boundary(fraction: float) -> float:
+                return float_minimum + span * fraction
+
+        for value in values:
+            index = min(bins - 1, max(0, int(position(value) * bins)))
+            counts[index] += 1
+        for index, count in enumerate(counts):
+            low = boundary(index / bins)
+            high = boundary((index + 1) / bins)
+            items.append({"label": f"{_number(low)} to {_number(high)}{suffix}", "count": count})
+    return {
+        "title": "Distribution",
+        "note": _analysis_note(analysis_rows, total_rows),
+        "items": items,
+    }
+
+
+def _is_missing(value: Any) -> bool:
+    """Notebook policy: null and every NaN payload are missing; signed zero is observed."""
+
+    return value is None or (isinstance(value, float) and math.isnan(value))
+
+
+def _distinct_key(value: Any) -> tuple[str, Any]:
+    """Typed exact identity for cardinality/counting, separate from display formatting.
+
+    All NaNs are excluded by ``_is_missing``. IEEE signed zeros intentionally form one value,
+    matching Arrow/Python equality semantics for an analytical distinct count.
+    """
+
+    if _is_missing(value):  # pragma: no cover - guarded by callers, retained as an invariant
+        raise ValueError("missing values do not have a distinct key")
+    if isinstance(value, bool):
+        return ("boolean", value)
+    if isinstance(value, int):
+        return ("int64", value)
+    if isinstance(value, float):
+        return ("float64", 0.0 if value == 0.0 else value)
+    return (type(value).__qualname__, value)
+
+
+def _value_counts(
+    values: list[Any],
+    *,
+    title: str = "Value counts",
+    analysis_rows: int,
+    total_rows: int,
+) -> dict[str, Any]:
+    keyed = Counter(_distinct_key(value) for value in values if not _is_missing(value))
+    counts = [(_value_text(key[1]), count) for key, count in keyed.items()]
+    ordered = sorted(counts, key=lambda item: (-item[1], item[0]))
+    visible = ordered[:_MAX_CATEGORY_BARS]
+    hidden = max(0, len(ordered) - len(visible))
+    note = _analysis_note(analysis_rows, total_rows)
+    if hidden:
+        note += f" · top {_MAX_CATEGORY_BARS} of {len(ordered):,} values"
+    return {
+        "title": title,
+        "note": note,
+        "items": [{"label": label, "count": count} for label, count in visible],
+    }
+
+
+def _column_summary(
+    values: list[Any],
+    logical: str,
+    unit: dict[str, Any],
+    *,
+    analysis_rows: int,
+    total_rows: int,
+) -> tuple[dict[str, str], dict[str, Any]]:
+    non_null = [value for value in values if not _is_missing(value)]
+    if not non_null:
+        return (
+            {"label": "Observed", "value": "0", "detail": "No non-missing values"},
+            {"title": "Distribution", "note": "No non-missing values", "items": []},
+        )
+    symbol = unit.get("symbol") if isinstance(unit.get("symbol"), str) else None
+    suffix = f" {symbol}" if symbol else ""
+    if logical in {"int64", "float64"}:
+        numeric = [
+            value
+            for value in non_null
+            if isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and (isinstance(value, int) or math.isfinite(value))
+        ]
+        if numeric:
+            if logical == "int64" and all(isinstance(value, int) for value in numeric):
+                ordered = sorted(numeric)
+                middle = len(ordered) // 2
+                if len(ordered) % 2:
+                    median: int | float | Decimal = ordered[middle]
+                else:
+                    total = ordered[middle - 1] + ordered[middle]
+                    median = total // 2 if total % 2 == 0 else Decimal(total) / Decimal(2)
+            else:
+                ordered_floats = sorted(float(value) for value in numeric)
+                middle = len(ordered_floats) // 2
+                if len(ordered_floats) % 2:
+                    median = ordered_floats[middle]
+                else:
+                    low = ordered_floats[middle - 1]
+                    high = ordered_floats[middle]
+                    if low < 0.0 < high:
+                        # Opposite signs make the sum safe even when the direct span overflows.
+                        median = (low + high) / 2.0
+                    else:
+                        # Same-sign subtraction is finite; this also preserves repeated extrema.
+                        median = low + (high - low) / 2.0
+            summary = {
+                "label": "Median",
+                "value": f"{_number(median)}{suffix}",
+                "detail": f"{_number(min(numeric))} → {_number(max(numeric))}{suffix}",
+            }
+            return summary, _histogram(
+                numeric,
+                unit_symbol=symbol,
+                analysis_rows=analysis_rows,
+                total_rows=total_rows,
+            )
+    if logical in {"date", "timestamp_utc"}:
+        rendered = sorted(_value_text(value) for value in non_null)
+        summary = {
+            "label": "Range",
+            "value": rendered[0],
+            "detail": f"through {rendered[-1]}",
+        }
+        return summary, _value_counts(
+            non_null,
+            title="Observations over time",
+            analysis_rows=analysis_rows,
+            total_rows=total_rows,
+        )
+    keyed_counts = Counter(_distinct_key(value) for value in non_null)
+    mode_key, count = sorted(
+        keyed_counts.items(), key=lambda item: (-item[1], _value_text(item[0][1]))
+    )[0]
+    mode = _value_text(mode_key[1])
+    summary = {
+        "label": "Most common",
+        "value": mode,
+        "detail": f"{count:,} of {len(non_null):,} observed",
+    }
+    title = "Boolean counts" if logical == "boolean" else "Value counts"
+    return summary, _value_counts(
+        non_null,
+        title=title,
+        analysis_rows=analysis_rows,
+        total_rows=total_rows,
+    )
+
+
+def _semantics_context(
+    candidate_dir: Path,
+    members: set[str],
+    column_names: list[str],
+    *,
+    member_bytes: dict[str, bytes] | None = None,
+    verified_recipe: dict[str, Any] | None = None,
+) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
+    """Derive semantic authority only from verified bindings or sealed local declarations."""
+
+    document: dict[str, Any] | None = None
+    records: list[Any] = []
+    if "recipe.json" in members:
+        authority = "approved_recipe" if isinstance(verified_recipe, dict) else "none"
+        raw_document = verified_recipe.get("table_semantics") if verified_recipe else None
+        if isinstance(raw_document, dict):
+            document = raw_document
+            records = raw_document.get("columns", [])
+        elif verified_recipe and isinstance(verified_recipe.get("output_semantics"), list):
+            records = verified_recipe["output_semantics"]
+    elif "semantics.json" in members:
+        authority = "declared_local_plan"
+        raw_document = _read_member_json(
+            candidate_dir,
+            "semantics.json",
+            members,
+            member_bytes=member_bytes,
+        )
+        if isinstance(raw_document, dict):
+            document = raw_document
+            records = raw_document.get("columns", [])
+    else:
+        authority = "none"
+
+    by_name = {
+        str(item.get("name")): item
+        for item in records
+        if isinstance(item, dict) and isinstance(item.get("name"), str)
+    }
+    missing = sum(
+        1
+        for name in column_names
+        if not isinstance(by_name.get(name, {}).get("display_label"), str)
+        or not by_name[name].get("display_label")
+        or not isinstance(by_name[name].get("description"), str)
+        or not by_name[name].get("description")
+        or _unit_metadata(by_name[name]).get("state") == "unknown"
+    )
+    complete = document is not None and missing == 0
+    completeness = "absent" if authority == "none" else "complete" if complete else "partial"
+    total = len(column_names)
+    if completeness == "absent":
+        message = (
+            "This Build declares no column meanings. Nothing here is inferred from column names."
+        )
+    elif authority == "approved_recipe" and complete:
+        message = "Column meanings are declared in an approved Recipe and sealed with this Build."
+    elif authority == "approved_recipe":
+        message = (
+            "Some column meanings are declared in the approved Recipe. "
+            f"{missing} of {total} columns have no declared meaning."
+        )
+    elif complete:
+        message = (
+            "Column meanings were declared with this local plan and sealed with this Build. "
+            "They are not human-approved."
+        )
+    else:
+        message = (
+            "Some column meanings were declared with this local plan. "
+            f"{missing} of {total} columns have no declared meaning. They are not human-approved."
+        )
+    context = {
+        "schema_version": "mostlyright.column-semantics.v1",
+        "authority": authority,
+        "completeness": completeness,
+        "message": message,
+        "missing_column_count": missing if authority != "none" else total,
+        "column_count": total,
+        "render_scope": "full",
+    }
+    if document is not None:
+        context.update(
+            {
+                "summary": document.get("summary"),
+                "grain_statement": document.get("grain_statement"),
+                "coverage": document.get("coverage"),
+                "limitations": document.get("limitations"),
+            }
+        )
+    return context, by_name
+
+
+def _fit_column_profile_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    """Reduce optional semantic detail deterministically until the portable MIME is bounded."""
+
+    from mostlyright.data_harness.nbrender.outputs_data import is_valid_column_profiles_payload
+
+    if is_valid_column_profiles_payload(payload):
+        return payload
+    reduced = deepcopy(payload)
+    semantics = reduced.get("semantics")
+    if isinstance(semantics, dict):
+        semantics["render_scope"] = "reduced"
+        for key in ("coverage", "limitations"):
+            semantics.pop(key, None)
+    for column in reduced.get("columns", []):
+        if isinstance(column, dict):
+            for key in ("description", "categories", "evidence_refs"):
+                column.pop(key, None)
+    if is_valid_column_profiles_payload(reduced):
+        return reduced
+    minimal = deepcopy(reduced)
+    semantics = minimal.get("semantics")
+    if isinstance(semantics, dict):
+        semantics["render_scope"] = "minimal"
+        for key in ("summary", "grain_statement"):
+            semantics.pop(key, None)
+    for column in minimal.get("columns", []):
+        if isinstance(column, dict):
+            for key in ("display_label", "categories_status"):
+                column.pop(key, None)
+    if is_valid_column_profiles_payload(minimal):
+        return minimal
+    legacy = deepcopy(minimal)
+    for column in legacy.get("columns", []):
+        if isinstance(column, dict):
+            column["unit"] = {
+                key: column.get("unit", {}).get(key)
+                for key in ("value", "symbol", "label", "status")
+            }
+    if is_valid_column_profiles_payload(legacy):
+        return legacy
+
+    # A verified string column may legitimately contain a very large value. Summary and chart
+    # labels derived from ten such columns can exceed the portable 1 MiB rich-MIME ceiling even
+    # after semantic detail is reduced. Keep the analytical shape and semantic authority, but
+    # deterministically bound presentation strings before considering refusal. Complete values
+    # never belonged in this MIME contract; the portable fallbacks describe the dataset rather
+    # than repeating profiled cell contents.
+    bounded_rich = deepcopy(legacy)
+    for column in bounded_rich.get("columns", []):
+        if not isinstance(column, dict):
+            continue
+        for field in ("summary", "source"):
+            value = column.get(field)
+            if isinstance(value, dict):
+                for key, item in tuple(value.items()):
+                    if isinstance(item, str):
+                        value[key] = _bounded_rich_label(item)
+        chart = column.get("chart")
+        if isinstance(chart, dict):
+            for key in ("title", "note"):
+                if isinstance(chart.get(key), str):
+                    chart[key] = _bounded_rich_label(chart[key])
+            items = chart.get("items")
+            if isinstance(items, list):
+                for item in items:
+                    if isinstance(item, dict) and isinstance(item.get("label"), str):
+                        item["label"] = _bounded_rich_label(item["label"])
+        unit = column.get("unit")
+        if isinstance(unit, dict):
+            for key, item in tuple(unit.items()):
+                if isinstance(item, str):
+                    unit[key] = _bounded_rich_label(item)
+        operations = column.get("transformations")
+        if isinstance(operations, list):
+            column["transformations"] = [
+                _bounded_rich_label(item) if isinstance(item, str) else item for item in operations
+            ]
+    if not is_valid_column_profiles_payload(bounded_rich):
+        raise ValueError("generated column-profile MIME exceeds its bounded contract")
+    return bounded_rich
+
+
+def _bounded_rich_label(value: str, *, limit: int = 128) -> str:
+    """Bound one derived rich-MIME label with an explicit deterministic marker."""
+
+    if len(value) <= limit:
+        return value
+    marker = " [truncated]"
+    return value[: limit - len(marker)] + marker
+
+
+def _column_profiles(
+    candidate_dir: Path,
+    table: Any,
+    inspection: Any,
+    profile: dict[str, Any],
+    members: set[str],
+    *,
+    member_bytes: dict[str, bytes] | None = None,
+    verified_recipe: dict[str, Any] | None = None,
+    fit_payload: bool = True,
+) -> dict[str, Any]:
+    """Compute bounded visual analytics over the verified Parquet and bind declared units."""
+
+    semantics_context, semantics_by_name = _semantics_context(
+        candidate_dir,
+        members,
+        list(table.column_names),
+        member_bytes=member_bytes,
+        verified_recipe=verified_recipe,
+    )
+    logical_by_name = {
+        str(item.get("name")): str(item.get("logical_type", "unknown"))
+        for item in profile.get("logical_schema", [])
+        if isinstance(item, dict)
+    }
+    type_by_name = (
+        {str(name): str(value) for name, value in profile.get("types", {}).items()}
+        if isinstance(profile.get("types"), dict)
+        else {}
+    )
+    nulls_by_name = (
+        profile.get("null_counts", {}) if isinstance(profile.get("null_counts"), dict) else {}
+    )
+    rows = int(profile.get("row_count", table.num_rows) or 0)
+    columns: list[dict[str, Any]] = []
+    for name in table.column_names:
+        semantics = semantics_by_name.get(name)
+        logical = str(
+            logical_by_name.get(name)
+            or (semantics or {}).get("logical_type")
+            or type_by_name.get(name)
+            or "unknown"
+        )
+        physical = PHYSICAL_TYPES.get(logical, str(table.schema.field(name).type))
+        unit = _unit_metadata(semantics)
+        array = table[name]
+        sample_size = min(table.num_rows, _PROFILE_SAMPLE_ROWS)
+        sample = array.slice(0, sample_size).to_pylist()
+        null_count = int(nulls_by_name.get(name, array.null_count) or 0)
+        nan_count = 0
+        if logical == "float64":
+            import pyarrow.compute as pc
+
+            # Arrow's sealed profile counts nulls. The notebook's visual missingness policy also
+            # treats every NaN payload as missing, so calculate that second exact count over the
+            # same verified Arrow snapshot and state both parts in the payload.
+            nan_count = int(pc.sum(pc.is_nan(array)).as_py() or 0)
+        missing = null_count + nan_count
+        non_missing_sample = [value for value in sample if not _is_missing(value)]
+        distinct_sample = len({_distinct_key(value) for value in non_missing_sample})
+        # Exact for small/medium columns; high-cardinality columns are clearly labelled sampled.
+        if table.num_rows <= _PROFILE_SAMPLE_ROWS:
+            distinct = distinct_sample
+            distinct_status = "exact"
+        else:
+            distinct = distinct_sample
+            distinct_status = f"sampled from first {sample_size:,} rows"
+        summary, chart = _column_summary(
+            sample,
+            logical,
+            unit,
+            analysis_rows=sample_size,
+            total_rows=table.num_rows,
+        )
+        detail = inspection.lineage.get(name, {}) if isinstance(inspection.lineage, dict) else {}
+        detail = detail if isinstance(detail, dict) else {}
+        column = {
+            "name": name,
+            "logical_type": logical,
+            "physical_type": physical,
+            "semantic_type": (semantics or {}).get("semantic_type"),
+            "unit": unit,
+            "row_count": rows,
+            "null_count": null_count,
+            "nan_count": nan_count,
+            "missing_count": missing,
+            "missing_rate": (missing / rows) if rows else 0.0,
+            "missing_policy": "null_or_nan",
+            "distinct_count": distinct,
+            "distinct_rate": (
+                distinct / max(1, rows if distinct_status == "exact" else sample_size)
+            ),
+            "distinct_status": distinct_status,
+            "distinct_policy": "typed_exact; null_and_nan_excluded; signed_zero_collapsed",
+            "sample_size": sample_size,
+            "summary": summary,
+            "chart": chart,
+            "source": {
+                "source_id": detail.get("source_id"),
+                "column": detail.get("source_column", name),
+            },
+            "transformations": list(detail.get("operations", []))
+            if isinstance(detail.get("operations"), list)
+            else [],
+        }
+        if semantics is not None:
+            column.update(
+                {
+                    "display_label": semantics.get("display_label"),
+                    "description": semantics.get("description"),
+                    "categories": semantics.get("categories"),
+                    "categories_status": semantics.get("categories_status"),
+                    "evidence_refs": semantics.get("evidence_refs"),
+                }
+            )
+        columns.append(column)
+    payload = {
+        "schema_version": "mostlyright.column-profile.v1",
+        "table_digest": inspection.table_sha256,
+        "row_count": rows,
+        "column_count": len(columns),
+        "analysis_scope": "all rows"
+        if table.num_rows <= _PROFILE_SAMPLE_ROWS
+        else f"first {_PROFILE_SAMPLE_ROWS:,} rows for charts",
+        "columns": columns,
+        "semantics": semantics_context,
+    }
+    return _fit_column_profile_payload(payload) if fit_payload else payload
+
+
+# --- reading the sealed candidate --------------------------------------------------------------
+
+
+def _read_member_json(
+    candidate_dir: Path,
+    relative: str,
+    members: set[str],
+    *,
+    member_bytes: dict[str, bytes] | None = None,
+) -> Any | None:
+    """Parse ``relative`` from ``candidate_dir`` only when it is a recorded, present member."""
+
+    if relative not in members:
+        return None
+    try:
+        if member_bytes is not None:
+            raw = member_bytes.get(relative)
+            return json.loads(raw.decode("utf-8")) if raw is not None else None
+        path = candidate_dir / relative
+        if not path.exists():
+            return None
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def _read_dataset(
+    candidate_dir: Path,
+    limit: int,
+    *,
+    member_bytes: dict[str, bytes] | None = None,
+) -> tuple[Any, list[dict[str, Any]], list[str]]:
+    """Read the verified Parquet once, returning the table and a native head sample."""
+
+    import pyarrow.parquet as pq  # lazy — the already-pinned base dependency
+
+    if member_bytes is not None:
+        import pyarrow as pa
+
+        table = pq.read_table(pa.BufferReader(member_bytes["data/table.parquet"]))
+    else:
+        table = pq.read_table(candidate_dir / "data" / "table.parquet")
+    sample = table.slice(0, limit).to_pylist()
+    return table, sample, list(table.column_names)
+
+
+# --- notebook assembly -------------------------------------------------------------------------
+
+
+def build_notebook(candidate_dir: Path) -> dict[str, Any]:
+    """Render a sealed candidate from one retained replay-verified snapshot."""
+
+    candidate_dir = Path(candidate_dir)
+    with pipeline.open_verified_snapshot(candidate_dir.parent) as verified_snapshot:
+        return _build_notebook_from_snapshot(candidate_dir, verified_snapshot)
+
+
+def _build_notebook_from_snapshot(
+    candidate_dir: Path,
+    verified_snapshot: pipeline.VerifiedSnapshot,
+) -> dict[str, Any]:
+    """Derive notebook bytes without reopening any candidate member pathname."""
+
+    member_bytes = dict(verified_snapshot.members)
+    inspection = verified_snapshot.inspection()
+    members = set(inspection.member_paths)
+    profile = (
+        _read_member_json(
+            candidate_dir,
+            "evidence/profile.json",
+            members,
+            member_bytes=member_bytes,
+        )
+        or {}
+    )
+    table = verified_snapshot.parquet_file().read()
+    sample_rows = table.slice(0, _SAMPLE_ROWS).to_pylist()
+    sample_columns = list(table.column_names)
+    verified_recipe: dict[str, Any] | None = None
+    if "recipe.json" in members:
+        try:
+            bindings = recipe_contracts.verify_recipe_authority_snapshot(verified_snapshot)
+        except recipe_contracts.RecipeError:
+            # Generic Build verification proves bytes, not human approval. A malformed or
+            # unbound Recipe candidate remains inspectable, but it must not gain approved copy.
+            verified_recipe = None
+        else:
+            verified_recipe = bindings["recipe"].to_dict()
+
+    full_column_profiles = _column_profiles(
+        candidate_dir,
+        table,
+        inspection,
+        profile,
+        members,
+        member_bytes=member_bytes,
+        verified_recipe=verified_recipe,
+        fit_payload=False,
+    )
+    column_profiles = _fit_column_profile_payload(full_column_profiles)
+    semantics_context, semantics_by_name = _semantics_context(
+        candidate_dir,
+        members,
+        sample_columns,
+        member_bytes=member_bytes,
+        verified_recipe=verified_recipe,
+    )
+    logical_by_name = {
+        str(item.get("name")): str(item.get("logical_type", "unknown"))
+        for item in profile.get("logical_schema", [])
+        if isinstance(item, dict)
+    }
+
+    cells: list[dict[str, Any]] = [
+        _markdown_cell(
+            _header_markdown(
+                inspection.question,
+                inspection.row_count,
+                len(inspection.columns),
+            )
+        ),
+        _markdown_cell(_overview_markdown(semantics_context)),
+        _markdown_cell(
+            "## Data\n\n### Preview\n\nThe first rows are embedded so the result is readable "
+            "without starting Python. Run the cell to load the complete Parquet file."
+        ),
+        _code_cell(
+            "import pandas as pd\n"
+            # The sidecar is a run-dir SIBLING of candidate/, so every relative path a runnable
+            # cell uses must go through candidate/ to resolve from the notebook's own directory.
+            "df = pd.read_parquet('candidate/data/table.parquet')\n"
+            f"df.head({_SAMPLE_ROWS})  # first rows",
+            (_html_table(sample_rows, sample_columns),),
+        ),
+        _markdown_cell(_dictionary_markdown(sample_columns, semantics_by_name, logical_by_name)),
+        _markdown_cell(
+            "### Column profiles\n\nSelect a column to inspect its type, unit or scaling state, "
+            "missingness, "
+            "cardinality, distribution, source, and transformations. Charts are computed from "
+            "the verified Parquet; semantic claims come only from sealed Build members."
+        ),
+        _code_cell(
+            "# visual profile for every column\ndf.describe(include='all')",
+            (
+                _display_column_profiles(
+                    column_profiles,
+                    fallback_payload=full_column_profiles,
+                ),
+            ),
+        ),
+        _markdown_cell(_quality_markdown(inspection.quality)),
+    ]
+
+    if isinstance(inspection.join, dict) and inspection.join:
+        if inspection.join.get("schema_version") in {
+            pipeline.GRAPH_OPERATION_EVIDENCE_V1,
+            pipeline.GRAPH_OPERATION_EVIDENCE_VERSION,
+        }:
+            cells.append(_markdown_cell(_operations_markdown(inspection.join)))
+        else:
+            cells.append(_markdown_cell(_join_markdown(inspection.join)))
+
+    cells.append(_markdown_cell(_sources_markdown(inspection.sources)))
+
+    cells.extend(
+        [
+            _markdown_cell(
+                "## Analysis\n\nThe following cells are optional local queries. They can be "
+                "edited without changing the packaged Parquet data."
+            ),
+            _code_cell("df.describe(include='all')"),
+            _code_cell("df.dtypes"),
+            _code_cell("df[df.columns[0]].value_counts().head(15)"),
+        ]
+    )
+
+    notebook = {
+        "cells": cells,
+        "metadata": {
+            "mostlyright": {
+                "provenance": _provenance_metadata(inspection),
+                "column_profile_mime": _COLUMN_PROFILE_MIME,
+            }
+        },
+        "nbformat": _NBFORMAT_MAJOR,
+        "nbformat_minor": _NBFORMAT_MINOR,
+    }
+    verified_snapshot.validate()
+    return notebook
+
+
+def render_table_notebook(
+    run_dir: Path,
+    *,
+    replace_existing: bool = True,
+    _run_fd: int | None = None,
+    _ancestor_validator: Any | None = None,
+) -> Path:
+    """Build the notebook for the candidate under ``run_dir`` and write ``table.ipynb`` beside it.
+
+    Resolves the candidate root via ``_candidate_run_target`` (which follows the workspace
+    ``result/`` indirection) and writes ``table.ipynb`` (0o644) as a SIBLING of ``candidate/`` —
+    never inside it. The sidecar is atomically replaced through the verified Build directory's
+    retained descriptor, without following an existing output link or reopening its path.
+
+    ``replace_existing`` separates the command a person ran from the viewer's unattended recovery.
+    ``mr-data notebook`` was asked for a fresh notebook and replaces whatever is there. The watcher
+    may only fill an absence, and it checks that absence long before this write: pass ``False`` and
+    a sidecar that appeared in between raises :class:`FileExistsError` instead of being replaced.
+    """
+
+    # Imported inside the function to break an import cycle: ``cli`` imports this module at module
+    # level for the notebook sidecar, so this module cannot import ``cli`` at module level.
+    from mostlyright.data_harness.cli import _candidate_run_target
+
+    run_dir = Path(run_dir).absolute()
+    target = run_dir if _run_fd is not None else _candidate_run_target(run_dir)
+    handle = None if _run_fd is not None else pipeline._open_candidate_run_handle(target)
+    active_run_fd = _run_fd if _run_fd is not None else handle.run_fd
+    sidecar_lock_fd = -1
+    sidecar_lock_identity: tuple[int, int, int] | None = None
+
+    def validate() -> None:
+        if handle is not None:
+            pipeline._validate_candidate_run_handle(handle)
+        if sidecar_lock_fd >= 0 and sidecar_lock_identity is not None:
+            _validate_table_sidecar_lock(
+                active_run_fd,
+                sidecar_lock_fd,
+                sidecar_lock_identity,
+            )
+        if _ancestor_validator is not None:
+            _ancestor_validator()
+
+    try:
+        if handle is not None:
+            sidecar_lock_fd, sidecar_lock_identity = _acquire_table_sidecar_lock(active_run_fd)
+        with pipeline.open_verified_snapshot(
+            target,
+            _run_fd=active_run_fd,
+            _ancestor_validator=validate,
+        ) as verified_snapshot:
+            notebook = _build_notebook_from_snapshot(target / "candidate", verified_snapshot)
+        validate()
+        raw = (json.dumps(notebook, indent=1, ensure_ascii=False) + "\n").encode("utf-8")
+        _atomic_write_table_notebook(active_run_fd, raw, replace_existing=replace_existing)
+        validate()
+    finally:
+        try:
+            if sidecar_lock_fd >= 0:
+                if fcntl is not None:
+                    fcntl.flock(sidecar_lock_fd, fcntl.LOCK_UN)
+                os.close(sidecar_lock_fd)
+        finally:
+            if handle is not None:
+                handle.close()
+    return target / TABLE_NOTEBOOK_NAME
+
+
+def _acquire_table_sidecar_lock(run_fd: int) -> tuple[int, tuple[int, int, int]]:
+    """Exclusively serialize every sidecar writer on the Build's structural activity file."""
+
+    if fcntl is None:
+        raise pipeline.BuildError(
+            "CANDIDATE_PLATFORM_UNSUPPORTED",
+            "secure table notebook locking is unavailable on this platform",
+        )
+    lock_fd = -1
+    try:
+        named = os.stat(
+            pipeline.RUN_BUILD_ACTIVITY_LOCK,
+            dir_fd=run_fd,
+            follow_symlinks=False,
+        )
+        if not stat.S_ISREG(named.st_mode) or named.st_nlink != 1:
+            raise pipeline.BuildError(
+                "CANDIDATE_MEMBER_INVALID",
+                "run build activity lock is not a stable regular file",
+            )
+        lock_fd = os.open(
+            pipeline.RUN_BUILD_ACTIVITY_LOCK,
+            os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0),
+            dir_fd=run_fd,
+        )
+        opened = os.fstat(lock_fd)
+        identity = pipeline._entry_identity(opened)
+        if (
+            not stat.S_ISREG(opened.st_mode)
+            or opened.st_nlink != 1
+            or identity != pipeline._entry_identity(named)
+        ):
+            raise pipeline.BuildError(
+                "CANDIDATE_MEMBER_INVALID",
+                "run build activity lock changed identity",
+            )
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+        _validate_table_sidecar_lock(run_fd, lock_fd, identity)
+        return lock_fd, identity
+    except BaseException:
+        if lock_fd >= 0:
+            os.close(lock_fd)
+        raise
+
+
+def _validate_table_sidecar_lock(
+    run_fd: int,
+    lock_fd: int,
+    identity: tuple[int, int, int],
+) -> None:
+    """Require the held activity lock to remain the exact named run child."""
+
+    opened = os.fstat(lock_fd)
+    named = os.stat(
+        pipeline.RUN_BUILD_ACTIVITY_LOCK,
+        dir_fd=run_fd,
+        follow_symlinks=False,
+    )
+    if (
+        not stat.S_ISREG(opened.st_mode)
+        or opened.st_nlink != 1
+        or not stat.S_ISREG(named.st_mode)
+        or named.st_nlink != 1
+        or pipeline._entry_identity(opened) != identity
+        or pipeline._entry_identity(named) != identity
+    ):
+        raise pipeline.BuildError(
+            "CANDIDATE_MEMBER_INVALID",
+            "run build activity lock changed identity",
+        )
+
+
+def _atomic_write_table_notebook(run_fd: int, raw: bytes, *, replace_existing: bool = True) -> None:
+    """Install the unsealed sidecar inside one descriptor-pinned Build directory.
+
+    Linking a complete temporary file is an atomic no-replace installation, so a caller that may
+    only fill an absence never overwrites a sidecar that arrived while it was rendering.
+    """
+
+    temporary = ""
+    descriptor = -1
+    try:
+        for _attempt in range(100):
+            temporary = f".{TABLE_NOTEBOOK_NAME}.{secrets.token_hex(12)}"
+            try:
+                descriptor = os.open(
+                    temporary,
+                    os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC | os.O_NOFOLLOW,
+                    0o600,
+                    dir_fd=run_fd,
+                )
+                break
+            except FileExistsError:
+                continue
+        else:
+            raise FileExistsError("could not allocate a table notebook temporary file")
+
+        offset = 0
+        while offset < len(raw):
+            written = os.write(descriptor, raw[offset:])
+            if written < 1:
+                raise OSError("table notebook write did not make progress")
+            offset += written
+        os.fsync(descriptor)
+        os.fchmod(descriptor, 0o644)
+        os.fsync(descriptor)
+        os.close(descriptor)
+        descriptor = -1
+        if replace_existing:
+            os.replace(
+                temporary,
+                TABLE_NOTEBOOK_NAME,
+                src_dir_fd=run_fd,
+                dst_dir_fd=run_fd,
+            )
+        else:
+            os.link(
+                temporary,
+                TABLE_NOTEBOOK_NAME,
+                src_dir_fd=run_fd,
+                dst_dir_fd=run_fd,
+                follow_symlinks=False,
+            )
+            os.unlink(temporary, dir_fd=run_fd)
+        temporary = ""
+        os.fsync(run_fd)
+    finally:
+        if descriptor >= 0:
+            os.close(descriptor)
+        if temporary:
+            try:
+                os.unlink(temporary, dir_fd=run_fd)
+            except FileNotFoundError:
+                pass