mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/nbrender/outputs_source.py

@@ -0,0 +1,260 @@
+"""Native passive component for deterministic source-observation evidence."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from mostlyright.data_harness.nbrender.parse import Output, RenderContext, esc
+from mostlyright.data_harness.visual_run.observations import (
+    safe_column_for_display,
+    safe_for_display,
+)
+
+MIME = "application/vnd.mostlyright.source-observation+json"
+FIELDS = frozenset(
+    {
+        "schema_version",
+        "source_id",
+        "content_sha256",
+        "bytes",
+        "format",
+        "rights",
+        "classification",
+        "row_count",
+        "typed_schema",
+        "missingness",
+        "sample",
+        "warnings",
+        "fitness",
+    }
+)
+FORMAT_FIELDS = frozenset({"name", "media_type", "encoding"})
+RIGHTS_FIELDS = frozenset({"status", "evidence", "permissions"})
+SCHEMA_FIELDS = frozenset({"name", "logical_type", "nullable"})
+WARNING_FIELDS = frozenset({"code", "message"})
+FITNESS_FIELDS = frozenset({"parseable", "has_columns", "has_rows", "sample_values_permitted"})
+CLASSIFICATIONS = frozenset({"public", "internal", "confidential", "restricted"})
+RIGHTS_STATUSES = frozenset(
+    {
+        "public_domain",
+        "permissive_license",
+        "project_owned",
+        "authorized_internal",
+        "user_authorized",
+    }
+)
+PERMISSIONS = frozenset({"local_use", "redistribute", "host"})
+LOGICAL_TYPES = frozenset(
+    {
+        "null",
+        "boolean",
+        "int64",
+        "float64",
+        "decimal",
+        "date",
+        "timestamp",
+        "timestamp_utc",
+        "string",
+        "mixed",
+    }
+)
+_IDENTIFIER = re.compile(r"\A[A-Za-z0-9](?:[A-Za-z0-9._:-]{0,127})\Z")
+_WARNING_CODE = re.compile(r"\A[A-Z][A-Z0-9_]{0,63}\Z")
+_DIGEST = re.compile(r"\A[0-9a-f]{64}\Z")
+MAX_COLUMNS = 512
+MAX_SAMPLE_ROWS = 5
+MAX_WARNINGS = 128
+MAX_TEXT = 16_384
+
+
+def _text(value: Any, *, maximum: int = MAX_TEXT) -> bool:
+    return isinstance(value, str) and bool(value) and len(value) <= maximum
+
+
+def _nonnegative_integer(value: Any, *, maximum: int = 2**63 - 1) -> bool:
+    return type(value) is int and 0 <= value <= maximum
+
+
+def valid_payload(value: Any) -> bool:
+    """Validate the exact closed payload emitted by ``_visual_source_summary``."""
+
+    if not isinstance(value, dict) or set(value) != FIELDS:
+        return False
+    if (
+        value.get("schema_version") != "mostlyright.source-observation.v1"
+        or not isinstance(value.get("source_id"), str)
+        or _IDENTIFIER.fullmatch(value["source_id"]) is None
+        or not isinstance(value.get("content_sha256"), str)
+        or _DIGEST.fullmatch(value["content_sha256"]) is None
+        or not _nonnegative_integer(value.get("bytes"))
+        or not _nonnegative_integer(value.get("row_count"))
+        or value.get("classification") not in CLASSIFICATIONS
+    ):
+        return False
+
+    format_value = value.get("format")
+    if (
+        not isinstance(format_value, dict)
+        or set(format_value) != FORMAT_FIELDS
+        or not _text(format_value.get("name"), maximum=64)
+        or format_value.get("media_type") != "text/csv"
+        or format_value.get("encoding") != "utf-8"
+    ):
+        return False
+
+    rights = value.get("rights")
+    if not isinstance(rights, dict) or set(rights) != RIGHTS_FIELDS:
+        return False
+    permissions = rights.get("permissions")
+    if (
+        rights.get("status") not in RIGHTS_STATUSES
+        or not _text(rights.get("evidence"))
+        or not safe_for_display(rights["evidence"])
+        or not isinstance(permissions, list)
+        or not permissions
+        or len(permissions) != len(set(permissions))
+        or any(permission not in PERMISSIONS for permission in permissions)
+    ):
+        return False
+
+    typed_schema = value.get("typed_schema")
+    if (
+        not isinstance(typed_schema, list)
+        or not 1 <= len(typed_schema) <= MAX_COLUMNS
+        or any(not isinstance(item, dict) or set(item) != SCHEMA_FIELDS for item in typed_schema)
+    ):
+        return False
+    columns = [item["name"] for item in typed_schema]
+    if (
+        len(columns) != len(set(columns))
+        or any(not safe_column_for_display(column) for column in columns)
+        or any(item["logical_type"] not in LOGICAL_TYPES for item in typed_schema)
+        or any(type(item["nullable"]) is not bool for item in typed_schema)
+    ):
+        return False
+
+    missingness = value.get("missingness")
+    if (
+        not isinstance(missingness, dict)
+        or set(missingness) != set(columns)
+        or any(
+            not _nonnegative_integer(count, maximum=value["row_count"])
+            for count in missingness.values()
+        )
+        or any(item["nullable"] != (missingness[item["name"]] > 0) for item in typed_schema)
+    ):
+        return False
+
+    fitness = value.get("fitness")
+    if (
+        not isinstance(fitness, dict)
+        or set(fitness) != FITNESS_FIELDS
+        or any(type(fitness[field]) is not bool for field in FITNESS_FIELDS)
+        or fitness["parseable"] is not True
+        or fitness["has_columns"] != bool(columns)
+        or fitness["has_rows"] != (value["row_count"] > 0)
+    ):
+        return False
+    public_values = (
+        rights["status"] in {"public_domain", "permissive_license"}
+        and value["classification"] == "public"
+    )
+    if fitness["sample_values_permitted"] != public_values:
+        return False
+
+    sample = value.get("sample")
+    if not isinstance(sample, list) or len(sample) > MAX_SAMPLE_ROWS:
+        return False
+    if (not public_values and sample) or any(
+        not isinstance(row, dict)
+        or list(row) != columns
+        or any(
+            type(cell) not in {bool, int, str, type(None)}
+            or (isinstance(cell, str) and (len(cell) > 512 or not safe_for_display(cell)))
+            for cell in row.values()
+        )
+        for row in sample
+    ):
+        return False
+
+    warnings = value.get("warnings")
+    return bool(
+        isinstance(warnings, list)
+        and len(warnings) <= MAX_WARNINGS
+        and all(
+            isinstance(item, dict)
+            and set(item) == WARNING_FIELDS
+            and isinstance(item.get("code"), str)
+            and _WARNING_CODE.fullmatch(item["code"]) is not None
+            and _text(item.get("message"))
+            and safe_for_display(item["message"])
+            for item in warnings
+        )
+    )
+
+
+def _sample_table(columns: list[str], rows: list[dict[str, Any]]) -> str:
+    if not rows:
+        return '<p class="nb-stage-policy">Source values withheld by display policy.</p>'
+    head = "".join(f'<th scope="col">{esc(column)}</th>' for column in columns)
+    body = "".join(
+        "<tr>" + "".join(f"<td>{esc(row[column])}</td>" for column in columns) + "</tr>"
+        for row in rows
+    )
+    return (
+        '<div class="nb-stage-table" tabindex="0" aria-label="Bounded source sample"><table>'
+        f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>"
+    )
+
+
+def render_source_observation(out: Output, _ctx: RenderContext) -> str:
+    value = out.data.get(MIME)
+    if not valid_payload(value):
+        raise ValueError("invalid source-observation MIME payload")
+    assert isinstance(value, dict)
+    schema = "".join(
+        "<li><code>"
+        + esc(item["name"])
+        + "</code><span>"
+        + esc(item["logical_type"])
+        + (" · nullable" if item["nullable"] else "")
+        + f" · {esc(value['missingness'][item['name']])} missing</span></li>"
+        for item in value["typed_schema"]
+    )
+    warnings = (
+        "".join(
+            f'<li class="is-warning"><strong>{esc(item["code"])}</strong>'
+            f"<span>{esc(item['message'])}</span></li>"
+            for item in value["warnings"]
+        )
+        or '<li class="is-info"><strong>source</strong><span>No source warnings</span></li>'
+    )
+    fitness = value["fitness"]
+    return (
+        '<section class="nb-source-observation" aria-label="Source evidence">'
+        "<header><span>Source evidence</span>"
+        f"<strong>{esc(value['source_id'])}</strong>"
+        f"<small>{esc(value['classification'])}</small></header>"
+        "<dl><div><dt>Rows</dt><dd>"
+        f"{esc(value['row_count'])}</dd></div><div><dt>Bytes</dt>"
+        f"<dd>{esc(value['bytes'])}</dd></div><div><dt>Format</dt>"
+        f"<dd>{esc(value['format']['name'])}</dd></div></dl>"
+        '<div class="nb-stage-evidence">'
+        "<section><h4>Rights & classification</h4>"
+        f"<p><strong>{esc(value['rights']['status'])}</strong> · "
+        f"{esc(', '.join(value['rights']['permissions']))}</p>"
+        f"<p>{esc(value['rights']['evidence'])}</p></section>"
+        "<section><h4>Schema & missingness</h4>"
+        f'<ul class="nb-stage-schema">{schema}</ul></section>'
+        "<section><h4>Fitness</h4>"
+        f"<p>parseable: {esc(fitness['parseable'])} · columns: {esc(fitness['has_columns'])} · "
+        f"rows: {esc(fitness['has_rows'])}</p></section>"
+        "<section><h4>Warnings</h4>"
+        f'<ul class="nb-stage-findings">{warnings}</ul></section></div>'
+        + _sample_table([item["name"] for item in value["typed_schema"]], value["sample"])
+        + f"<footer><code>{esc(value['content_sha256'])}</code></footer></section>"
+    )
+
+
+__all__ = ["MIME", "render_source_observation", "valid_payload"]

mostlyright/data_harness/nbrender/outputs_stage.py

@@ -0,0 +1,176 @@
+"""Passive native component for bounded stage-observation MIME data."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from mostlyright.data_harness.nbrender.parse import Output, RenderContext, esc
+from mostlyright.data_harness.visual_run.observations import validate_stage_observation
+
+MIME = "application/vnd.mostlyright.stage-observation.v1+json"
+MIME_CURRENT = "application/vnd.mostlyright.stage-observation+json"
+MIMES = (MIME_CURRENT, MIME)
+_OPERATION_COMPONENTS = {
+    "union": ("union", "Union compatibility"),
+    "join": ("join", "Join contract"),
+    "aggregate": ("aggregate", "Aggregation"),
+    "resample": ("resample", "Temporal resampling"),
+    "normalize": ("normalization", "Normalization"),
+    "cast": ("normalization", "Type normalization"),
+    "rename": ("normalization", "Schema normalization"),
+}
+
+
+def valid_payload(value: Any) -> bool:
+    try:
+        validate_stage_observation(value)
+    except (TypeError, ValueError):
+        return False
+    return True
+
+
+def _table(columns: list[str], rows: list[dict[str, Any]], *, label: str) -> str:
+    if not rows:
+        return '<p class="nb-stage-policy">Values withheld by display policy.</p>'
+    head = "".join(f'<th scope="col">{esc(column)}</th>' for column in columns)
+    body = "".join(
+        "<tr>"
+        + "".join(
+            f"<td>{esc(json.dumps(row.get(column), ensure_ascii=False, sort_keys=True))}</td>"
+            for column in columns
+        )
+        + "</tr>"
+        for row in rows
+    )
+    return (
+        f'<div class="nb-stage-table" tabindex="0" aria-label="{esc(label)}"><table>'
+        f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>"
+    )
+
+
+def _facts(value: dict[str, Any]) -> str:
+    facts = (
+        ("Input rows", value["input_rows"]),
+        ("Output rows", value["output_rows"]),
+        ("Rejected", value["rejected_rows"]),
+        ("Duplicates", value["duplicate_rows"]),
+    )
+    return (
+        "<dl>"
+        + "".join(
+            f"<div><dt>{esc(label)}</dt><dd>{esc(number)}</dd></div>" for label, number in facts
+        )
+        + "</dl>"
+    )
+
+
+def _v2_evidence(value: dict[str, Any]) -> str:
+    if value.get("schema_version") != "stage-observation.v2":
+        return ""
+    audit = value["audit"]
+    schemas = "".join(
+        "<li><strong data-semantic-label>"
+        f"{esc(column['name'])}</strong><span>{esc(column['logical_type'])}"
+        f"{' · nullable' if column['nullable'] else ''}</span></li>"
+        for column in value["output_schema"]
+    )
+    profiles = _table(
+        ["column", "rows_examined", "null_count", "distinct_count"],
+        value["profiles"],
+        label="Column profiles",
+    )
+    lineage = "".join(
+        "<li><strong>"
+        + esc(item["column"])
+        + "</strong><span>"
+        + esc(
+            ", ".join(f"{source['coordinate']}.{source['column']}" for source in item["sources"])
+            or "derived"
+        )
+        + "</span><small>"
+        + esc(" → ".join(item["operations"]) or "identity")
+        + "</small></li>"
+        for item in value["lineage"]
+    )
+    findings = (
+        "".join(
+            f'<li class="is-{esc(item["severity"])}"><strong>{esc(item["code"])}</strong>'
+            f"<span>{esc(item['message'])}</span>"
+            f"<small>{esc(item['column'] or 'dataset')}</small></li>"
+            for item in value["findings"]
+        )
+        or '<li class="is-info"><strong>checks</strong><span>No quality findings</span></li>'
+    )
+    sources = ", ".join(value["source_coordinates"]) or "derived inputs"
+    operation_component = ""
+    component = _OPERATION_COMPONENTS.get(value["operation"])
+    if component is not None:
+        kind, label = component
+        parameters = json.dumps(
+            value["operation_parameters"], ensure_ascii=False, sort_keys=True, separators=(",", ":")
+        )
+        operation_component = (
+            f'<section class="nb-stage-operation nb-stage-operation--{kind}">'
+            f"<h4>{esc(label)}</h4><p>{esc(len(value['inputs']))} input coordinate(s) → "
+            f"{esc(value['output_rows'])} rows</p><code>{esc(parameters)}</code></section>"
+        )
+    representative_component = ""
+    representative = (
+        value["coverage"].get("representative_window") if value["display"] == "full" else None
+    )
+    if representative is not None:
+        input_rows = [row for item in representative["inputs"] for row in item["rows"]]
+        input_columns = list(input_rows[0]) if input_rows else []
+        representative_component = (
+            '<details class="nb-stage-window"><summary>Representative sealed window</summary>'
+            f"<p>{esc(representative['start'])} → {esc(representative['end'])}</p>"
+            + _table(input_columns, input_rows, label="Representative input rows")
+            + _table(
+                list(representative["output"]),
+                [representative["output"]] if representative["output"] else [],
+                label="Representative resample output",
+            )
+            + "</details>"
+        )
+    return (
+        '<div class="nb-stage-evidence">'
+        + operation_component
+        + representative_component
+        + '<section><h4>Schema & semantic labels</h4><ul class="nb-stage-schema">'
+        + schemas
+        + "</ul></section>"
+        "<section><h4>Profiles</h4>" + profiles + "</section>"
+        '<section><h4>Quality examples</h4><ul class="nb-stage-findings">'
+        + findings
+        + "</ul></section>"
+        '<section><h4>Column lineage</h4><ul class="nb-stage-lineage">'
+        + lineage
+        + "</ul></section>"
+        '<section class="nb-stage-audit"><h4>Sealed evidence</h4>'
+        f"<p><strong>{esc(audit['engine'])}</strong> · {esc(audit['order_contract'])}</p>"
+        f"<p>{esc(sources)}</p><code>{esc(value['output_digest'])}</code></section></div>"
+    )
+
+
+def render_stage_observation(
+    out: Output, _ctx: RenderContext, *, mimetype: str | None = None
+) -> str:
+    selected = mimetype or next((name for name in MIMES if name in out.data), MIME)
+    value = out.data.get(selected)
+    if not valid_payload(value):
+        raise ValueError("invalid stage-observation MIME payload")
+    assert isinstance(value, dict)
+    truncation = " · bounded sample" if value["truncated"] else ""
+    return (
+        '<section class="nb-stage-observation" aria-label="Data operation evidence">'
+        "<header><span>Data operation</span>"
+        f"<strong>{esc(value['operation'])}</strong><small>{esc(value['node_id'])}</small></header>"
+        + _facts(value)
+        + _v2_evidence(value)
+        + _table(value["columns"], value["sample"], label="Bounded stage sample")
+        + f"<footer>{esc(value['display'])}{esc(truncation)}</footer></section>"
+    )
+
+
+__all__ = ["MIME", "MIMES", "MIME_CURRENT", "render_stage_observation", "valid_payload"]