dbdiff 0.7.1__py3-none-any.whl

dbdiff/report.py

@@ -0,0 +1,165 @@
+import pandas as pd
+from jinja2 import Environment, PackageLoader
+
+MAX_EXCEL_SHEET_NAME_LEN = 31
+JINJA_ENV = Environment(loader=PackageLoader("dbdiff", "templates"))
+
+
+def get_max_diferences(column_info: pd.DataFrame) -> int:
+    if len(list(column_info.values())) > 0:
+        return list(column_info.values())[0]["count"]
+    else:
+        return 0
+
+
+def reformat_missing_join_info(d: dict, x_table: str, y_table: str) -> dict:
+    return {x_table: d["x"], y_table: d["y"]}
+
+
+def reformat_hierarchical_join_info(d: dict, x_table: str, y_table: str) -> dict:
+    return {col: reformat_missing_join_info(val, x_table, y_table) for col, val in d.items()}
+
+
+def html_report(
+    x_schema: str,
+    y_schema: str,
+    x_table: str,
+    y_table: str,
+    join_cols: list,
+    diff_summary: dict,
+    total_row_count: int,
+    column_info: dict,
+    column_match_info: pd.DataFrame,
+    missing_join_info: dict,
+    hierarchical_join_info: dict,
+    dedup_info: dict,
+) -> str:
+
+    def comma(value, format="{0:,d}"):
+        return format.format(value)
+
+    def code(value, codeclass="plaintext"):
+        return f'<code class="{codeclass}">{value}</code>'
+
+    def dfhtml(df, classes=["table", "table-bordered", "table-striped", "table-hover", "table-sm"]):
+        return df.to_html(index=False, classes=classes)
+
+    JINJA_ENV.filters["comma"] = comma
+    JINJA_ENV.filters["code"] = code
+    JINJA_ENV.filters["dfhtml"] = dfhtml
+
+    t = JINJA_ENV.get_template("html/report.html")
+
+    max_differences = get_max_diferences(column_info)
+    missing_join_info = reformat_missing_join_info(missing_join_info, x_table, y_table)
+    hierarchical_join_info = reformat_hierarchical_join_info(
+        hierarchical_join_info, x_table, y_table
+    )
+
+    return t.render(
+        {
+            "x_schema": x_schema,
+            "y_schema": y_schema,
+            "x_table": x_table,
+            "y_table": y_table,
+            "join_cols": join_cols,
+            "diff_summary": diff_summary,
+            "total_row_count": total_row_count,
+            "column_info": column_info,
+            "max_differences": max_differences,
+            "missing_join_info": missing_join_info,
+            "hierarchical_join_info": hierarchical_join_info,
+            "dedup_info": dedup_info,
+            # can't do these filters in Jinja
+            # could write a filter function that takes a list of
+            # positive and a list of negative filter columns
+            # but this is good enough for the one case:
+            "compared_column_count": (
+                (~column_match_info.exclude)
+                & column_match_info.comparable
+                & (~column_match_info.x_dtype.isnull())
+                & (~column_match_info.y_dtype.isnull())
+                & (~column_match_info.index.isin(join_cols))
+            ).sum(),
+            "column_match_info": column_match_info,
+        }
+    )
+
+
+def excel_report(
+    x_schema: str,
+    y_schema: str,
+    x_table: str,
+    y_table: str,
+    join_cols: list,
+    diff_summary: dict,
+    total_row_count: int,
+    column_info: dict,
+    column_match_info: pd.DataFrame,
+    missing_join_info: dict,
+    hierarchical_join_info: dict,
+    dedup_info: dict,
+) -> list:
+    """
+    Return a list with [(sheet_name: str, df: pd.DataFrame) ... ]
+    """
+    all_sheets = []
+    summary_sheet_data = [
+        {
+            "Summary": 'Diff report between tables {x_table} (herein, "x") and {y_table} (herein, "y").'.format(
+                x_table=x_table, y_table=y_table
+            )
+        }
+    ]
+    summary_sheet_data.append({"Summary": "----"})
+    summary_sheet_data.append(
+        {
+            "Summary": "There are {x_missing_count} rows in {y_table} that are not in {x_table}.".format(
+                x_table=x_table, y_table=y_table, x_missing_count=missing_join_info["x"]["count"]
+            )
+        }
+    )
+    summary_sheet_data.append(
+        {
+            "Summary": "There are {y_missing_count} rows in {x_table} that are not in {y_table}.".format(
+                x_table=x_table, y_table=y_table, y_missing_count=missing_join_info["y"]["count"]
+            )
+        }
+    )
+    summary_sheet_data.append(
+        {
+            "Summary": "There are {diff_row_count} rows matched between tables that don't line up exactly.".format(
+                diff_row_count=diff_summary["count"]
+            )
+        }
+    )
+    summary_sheet_data.append(
+        {
+            "Summary": "There are {column_info} columns that have differences.".format(
+                column_info=len(column_info)
+            )
+        }
+    )
+
+    max_differences = get_max_diferences(column_info)
+
+    summary_sheet_data.append(
+        {
+            "Summary": "The maximum number of differences on any individual column is {max_differences}.".format(
+                max_differences=max_differences
+            )
+        }
+    )
+    all_sheets.append(("Summary", pd.DataFrame(summary_sheet_data)))
+
+    if missing_join_info["x"]["count"] > 0:
+        # all_sheets.append(('Missing rows in {x_table}'.format(x_table=x_table), x_missing_ids))
+        all_sheets.append(("Missing in x", missing_join_info["x"]["sample"]))
+    if missing_join_info["y"]["count"] > 0:
+        # all_sheets.append(('Missing rows in {y_table}'.format(y_table=y_table), y_missing_ids))
+        all_sheets.append(("Missing in y", missing_join_info["y"]["sample"]))
+    if diff_summary["count"] > 0:
+        all_sheets.append(("Mismatched rows", diff_summary["sample"]))
+    for column, info in column_info.items():
+        all_sheets.append((column[:MAX_EXCEL_SHEET_NAME_LEN], info["df_raw"]))
+    return all_sheets

dbdiff/templates/all_keys_count.sql

@@ -0,0 +1,6 @@
+     SELECT COUNT(*)
+       FROM {{ x_schema }}.{{ x_table }} x
+ FULL OUTER JOIN {{ y_schema }}.{{ y_table }} y
+           ON {% for col in join_cols %}x.{{ col }} <=> y.{{ col }}{% if not loop.last %} AND {% endif %}
+           {% endfor -%}
+           WHERE {% if x %}y{% else %}x{% endif %}.{{ join_cols[0] }} IS NULL

dbdiff/templates/all_keys_sample.sql

@@ -0,0 +1,10 @@
+SELECT {% for col in join_cols -%}
+                   {% if x %}x{% else %}y{% endif %}.{{ col }} AS {{ col }}{% if not loop.last %},
+                   {% endif %}{% endfor %}
+              FROM {{ x_schema }}.{{ x_table }} x
+   FULL OUTER JOIN {{ y_schema }}.{{ y_table }} y
+                   ON {% for col in join_cols %}x.{{ col }} <=> y.{{ col }}{% if not loop.last %} AND {% endif %}
+                   {% endfor -%}
+                   WHERE {% if x %}y{% else %}x{% endif %}.{{ join_cols[0] }} IS NULL
+                   ORDER BY {% for col in join_cols %}{% if x %}x{% else %}y{% endif %}.{{ col }}{% if not loop.last %}, {% endif %}{% endfor %}
+                   LIMIT {{ max_rows_column }}

dbdiff/templates/create_dedup.sql

@@ -0,0 +1,11 @@
+    SELECT x.*
+{% if not use_temp_table %}INTO {{ schema_name }}.{{ table_name_dedup }}{% endif %}
+      FROM {{ schema_name }}.{{ table_name }} x
+INNER JOIN (
+    SELECT {{ group_cols }},
+           COUNT(*) AS dup_count
+      FROM {{ schema_name }}.{{ table_name }}
+  GROUP BY {{ group_cols }}
+           ) y
+           ON {{ join_cols }}
+     WHERE y.dup_count = 1

dbdiff/templates/create_dup.sql

@@ -0,0 +1,11 @@
+    SELECT x.*, y.dup_count
+{% if not use_temp_table %}INTO {{ schema_name }}.{{ table_name_dup }}{% endif %}
+      FROM {{ schema_name }}.{{ table_name }} x
+INNER JOIN (
+    SELECT {{ group_cols }},
+           COUNT(*) AS dup_count
+      FROM {{ schema_name }}.{{ table_name }}
+  GROUP BY {{ group_cols }}
+           ) y
+           ON {{ join_cols }}
+     WHERE y.dup_count > 1

dbdiff/templates/create_joined_table.sql

@@ -0,0 +1,13 @@
+CREATE TABLE {{ joined_schema }}.{{ joined_table }} (
+    {% for i, row in compare_cols.iterrows() %}
+    {% if row.name in join_cols %}
+    {{ row.name }} {{ row.x_dtype }}
+    {% else %}
+    x_{{ row.name }} {{ row.x_dtype }},
+    y_{{ row.name }} {{ row.x_dtype }}
+    {% endif %}
+    {% if not loop.last %},{% endif %}
+    {% endfor %}
+)
+ORDER BY {{ join_cols|join(", ") }}
+;

dbdiff/templates/create_joined_table_from_selectinto.sql

@@ -0,0 +1,14 @@
+     SELECT {% for i, row in compare_cols.iterrows() -%}
+            {% if row.name in join_cols -%}
+            COALESCE(x.{{ row.name }}, y.{{ row.name }}) AS {{ row.name -}}
+            {% else -%}
+            x.{{ row.name }}::{{ row.x_dtype }} AS x_{{ row.name }},
+            y.{{ row.name }}::{{ row.x_dtype }} AS y_{{ row.name -}}
+            {%- endif -%}
+            {%- if not loop.last %},{% endif -%}
+            {%- endfor %}
+       INTO {{ joined_schema }}.{{ joined_table }}
+       FROM {{ x_schema }}.{{ x_table }} AS x
+ INNER JOIN {{ y_schema }}.{{ y_table }} AS y
+            ON {% for col in join_cols %}x.{{ col }} <=> y.{{ col }}{% if not loop.last %} AND {% endif %}
+            {% endfor %}

dbdiff/templates/create_temp_table.sql

@@ -0,0 +1 @@
+CREATE LOCAL TEMP TABLE {{ table_name }} ON COMMIT PRESERVE ROWS AS ({{ query }})

dbdiff/templates/diff_column.sql

@@ -0,0 +1,28 @@
+-- could pull them out of the original table:
+-- SELECT x_{{ column }},
+--   y_{{ column }},
+--   COUNT(*) AS ct
+-- FROM (
+-- SELECT x_{{ column }}, y_{{ column }},
+--   x_{{ column }} <=> y_{{ column }} AS {{ column }}_eq
+-- FROM {{ joined_schema }}.{{ joined_table }}
+--   ) AS t1
+-- WHERE t1.{{ column }}_eq IS FALSE
+-- GROUP BY x_{{ column }}, y_{{ column }}
+-- ORDER BY ct DESC
+-- but instead use the results we already have
+-- some critique of the following: the group by in the subquery isn't
+-- necessary based on the design (the insert into this diff table)
+SELECT joined.x_{{ column }},
+   joined.y_{{ column }},
+   COUNT(*) AS ct
+FROM {{ joined_schema }}.{{ joined_table }} joined
+INNER JOIN (
+SELECT {{ group_cols }}
+FROM {{ diff_schema }}.{{ diff_table }}
+WHERE column_name = '{{ column }}'
+GROUP BY {{ group_cols }}
+   ) diff
+   ON {{ join_cols }}
+GROUP BY joined.x_{{ column }}, joined.y_{{ column }}
+ORDER BY ct DESC

dbdiff/templates/diff_column_hier.sql

@@ -0,0 +1,5 @@
+        SELECT {{ join_cols }},
+               {{ column }}
+          FROM {{ schema }}.{{ table }}
+         WHERE {{ first_join_col }} IN (SELECT {{ first_join_col }} FROM {{ diff_schema }}.{{ diff_table }} WHERE column_name = '{{ column }}' GROUP BY {{ first_join_col }} ORDER BY {{ first_join_col }} {% if limit %}LIMIT {{ limit }}{% endif %})
+      ORDER BY {{ join_cols }}

dbdiff/templates/diff_column_numeric_diffs_binned.sql

@@ -0,0 +1,15 @@
+SELECT MIN(diff) AS min_diff,
+       MAX(diff) AS max_diff,
+       SUM(ct) AS ct
+  FROM (
+      SELECT NTILE({{ tiles }}) OVER (ORDER BY ABS(raw.x_{{ column }}-raw.y_{{ column }})) AS n_tile,
+            ABS(raw.x_{{ column }}-raw.y_{{ column }}) AS diff,
+            raw.ct
+            FROM (
+                {% include "diff_column.sql" %}
+            ) raw
+            WHERE raw.x_{{ column }} IS NOT NULL
+                  AND raw.y_{{ column }} IS NOT NULL
+        ) tiled
+GROUP BY n_tile
+ORDER BY MIN(diff) ASC, MAX(diff) ASC

dbdiff/templates/diff_column_numeric_diffs_sorted.sql

@@ -0,0 +1,6 @@
+        SELECT {{ join_cols|join(", ")  }},
+               x_{{ column }},
+               y_{{ column }},
+               ABS(x_{{ column }} - y_{{ column }}) AS abs_diff
+          FROM ({% include "diff_column_raw.sql" %}) q_raw
+      ORDER BY ABS(x_{{ column }} - y_{{ column }}) DESC

dbdiff/templates/diff_column_raw.sql

@@ -0,0 +1,11 @@
+    SELECT {% for col in join_cols %}joined.{{ col }},
+           {% endfor %}joined.x_{{ column }},
+           joined.y_{{ column }}
+      FROM {{ joined_schema }}.{{ joined_table }} joined
+INNER JOIN (
+    SELECT {{ join_cols|join(", ") }}
+      FROM {{ diff_schema }}.{{ diff_table }}
+     WHERE column_name = '{{ column }}'
+  GROUP BY {{ join_cols|join(", ") }}
+           ) diff
+           ON {{ join_cols_join }}

dbdiff/templates/diff_column_summary.sql

@@ -0,0 +1,5 @@
+  SELECT column_name,
+         COUNT(*)
+    FROM {{ schema_name }}.{{ table_name }}
+GROUP BY 1
+ORDER BY 2 DESC;

dbdiff/templates/diff_rows_sample.sql

@@ -0,0 +1,8 @@
+    SELECT x.*
+      FROM {{ schema_name }}.{{ joined_table }} x
+INNER JOIN (
+    SELECT {{ group_cols }}
+      FROM {{ schema_name }}.{{ diff_table }}
+  GROUP BY {{ group_cols }}
+           ) joined
+        ON {{ join_cols }}

dbdiff/templates/first_key_base.sql

@@ -0,0 +1,18 @@
+{% block select %}
+{% endblock %}
+{% block core %}
+           FROM (
+         SELECT {{ join_col }} FROM {{ x_schema }}.{{ x_table }} GROUP BY 1
+                ) x
+FULL OUTER JOIN (
+         SELECT {{ join_col }} FROM {{ y_schema }}.{{ y_table }} GROUP BY 1
+                ) y
+                ON x.{{ join_col }} = y.{{ join_col }}
+{% endblock %}
+{% block where %}
+                WHERE {% if x %}y{% else %}x{% endif %}.{{ join_col }} IS NULL
+{% endblock %}
+{% block orderby %}
+{% endblock %}
+{% block limit %}
+{% endblock %}

dbdiff/templates/first_key_count.sql

@@ -0,0 +1,4 @@
+{% extends "first_key_base.sql" %}
+{% block select %}
+         SELECT COUNT(*)
+{% endblock %}

dbdiff/templates/first_key_sample.sql

@@ -0,0 +1,10 @@
+{% extends "first_key_base.sql" %}
+{% block select %}
+         SELECT {% if x %}x{% else %}y{% endif %}.{{ join_col }} AS {{ join_col }}
+{% endblock %}
+{% block orderby %}
+                ORDER BY {% if x %}x{% else %}y{% endif %}.{{ join_col }}
+{% endblock %}
+{% block limit %}
+                LIMIT {{ max_rows_column }}
+{% endblock %}

dbdiff/templates/html/base.html

@@ -0,0 +1,34 @@
+<!doctype html>
+<html lang="en">
+  <head>
+{% block header %}
+{% block headermeta %}
+    <!-- Required meta tags -->
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+{% endblock %}
+{% block headercss %}
+    <!-- Bootstrap CSS -->
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
+{% endblock %}
+{% block headerextracss %}
+{% endblock %}
+{% block headertitle %}
+    <title>Base HTML templae</title>
+{% endblock %}
+{% endblock %}
+  </head>
+  <body>
+{% block body %}
+{% endblock %}
+{% block bodyjs %}
+    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
+{% endblock %}
+    <!-- Optional JavaScript -->
+{% block bodyextrajs %}
+{% endblock %}
+  </body>
+</html>

dbdiff/templates/html/report.html

@@ -0,0 +1,241 @@
+{% extends "html/base.html" %}
+
+{% block headerextracss %}
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/styles/atom-one-light.min.css" integrity="sha256-VCcaD9+X/d4QGYRX7l5aMJ8BWgwfA8d3S7i/HC9rvvw=" crossorigin="anonymous" />
+{% endblock %}
+{% block headertitle %}
+<title>Diff Report</title>
+{% endblock %}
+
+{% block body %}
+<div class="container">
+    <div class="jumbotron" style="margin-top: 30px;">
+        <h1 class="display-4">
+            Diff report between tables <ul>
+                <li>{{ x_table|code }} and</li>
+                <li>{{ y_table|code }}.</li>
+            </ul>
+        </h1>
+        <h2>
+            Joined using columns: {% for col in join_cols %}{{ col|code }}{% if not loop.last %}, {% endif %}{% endfor %}.
+        </h2>
+        <hr class="my-4">
+        <h3>Summary:</h3>
+        <ul>
+            {% for side, info in dedup_info.items() %}
+            {% if info.count > 0 %}
+            <li>There are {{ info.count|comma|code }} rows in {{ side|code }} that were not uniquely identified by the join keys, they will be ignored for cell-by-cell differences, but considered for heirarchical join key analysis.</li>
+            {% endif %}
+            {% endfor %}
+            {% for side, info in missing_join_info.items() %}
+            <li>There are {{ info.count|comma|code }} rows in {{ side|code }} that are not in other table.</li>
+            {% endfor %}
+            {% if diff_summary.count %}
+            <li>There are {{ diff_summary.count|comma|code }} / {{ total_row_count|comma|code }} rows matched between tables that don't line up exactly, and {{ diff_summary.total_count|comma|code }} total cell differences.</li>
+            {% else %}
+            <li>There are {{ total_row_count|comma|code }} rows matched between tables, and {{ diff_summary.total_count|comma|code }} total cell differences.</li>
+            <li>There are {{ diff_summary.total_count|comma|code }} total cell differences.</li>
+            {% endif %}
+            {% if compared_column_count == 1 %}
+            <li>There is {{ 1|comma|code }} / {{ compared_column_count|comma|code }} columns that has a difference.</li>
+            {% else %}
+            <li>There are {{ column_info|length|comma|code }} / {{ compared_column_count|comma|code }} columns that have differences.</li>
+            {% endif %}
+            {% if (column_match_info.uncomparable).sum() > 0 %}
+            <li>There {% if (column_match_info.uncomparable).sum() > 1 %}are{% else %}is{% endif %} {{ (column_match_info.uncomparable).sum()|comma|code }} column{% if (column_match_info.uncomparable).sum() > 1 %}s{% endif %} that matched on name
+                but were not compared based dtype matching:
+                <ul>
+                    {% for i, row in column_match_info.loc[column_match_info.uncomparable].iterrows() %}
+                    <li>Column {{ row.name|code }} has type {{ row.x_dtype|code }} in x and type {{ row.y_dtype|code }} in y.</li>
+                    {% endfor %}
+                </ul>
+            </li>
+            {% endif %}
+            {% if column_match_info.x_dtype.isnull().sum() > 0 %}
+            <li>There {% if column_match_info.x_dtype.isnull().sum() > 1 %}are{% else %}is{% endif %} {{ column_match_info.x_dtype.isnull().sum()|comma|code }} column(s) missing from x:
+                <ul>
+                    {% for i, row in column_match_info.loc[column_match_info.x_dtype.isnull()].iterrows() %}
+                    <li>Column {{ row.name|code }}.</li>
+                    {% endfor %}
+                </ul>
+            </li>
+            {% endif %}
+            {% if column_match_info.y_dtype.isnull().sum() > 0 %}
+            <li>There {% if column_match_info.y_dtype.isnull().sum() > 1 %}are{% else %}is{% endif %} {{ column_match_info.y_dtype.isnull().sum()|comma|code }} column(s) missing from y.
+                <ul>
+                    {% for i, row in column_match_info.loc[column_match_info.y_dtype.isnull()].iterrows() %}
+                    <li>Column {{ row.name|code }}.</li>
+                    {% endfor %}
+                </ul>
+            </li>
+            {% endif %}
+            <li>The maximum number of differences on any individual column is {{ max_differences|comma|code }}.</li>
+            {% if column_match_info.exclude.sum() > 0 %}
+            <li>For reference, this report excluded the following {{ column_match_info.exclude.sum()|comma|code }} column(s):
+                <ul>
+                    {% for i, row in column_match_info.loc[column_match_info.exclude].iterrows() %}
+                    <li>Column {{ row.name|code }}.</li>
+                    {% endfor %}
+                </ul>
+            </li>
+            {% endif %}
+        </ul>
+    </div>
+
+    <div class="row">
+        <div class="col">
+            <h2>
+                (1) Missing rows, (2) mismatched rows, and (3) differences by each column.
+                Click to expand.
+            </h2>
+            <p>Columns are ordered by the number of differences on each column.
+                For the mismatched rows, columns with prefix "x" are from <code class="plaintext">{{ x_schema }}.{{ x_table }}</code> and columns with prefix "y" are from <code class="plaintext">{{ x_schema }}.{{ y_table }}</code>.
+                The same columns labels are in the "grouped differences" output,
+                in which we take all pairs of cells that are different for the given column,
+                and count up the number of such pairs.
+                The code to pull the difference data is printed below it.</p>
+        </div>
+    </div>
+
+    <div class="row" style="margin-bottom: 30px;">
+        <div class="col">
+            <div class="accordion" id="accordionExample">
+                {% for side, info in missing_join_info.items() %}
+                {% if info.count > 0 %}
+                <div class="card">
+                    <div class="card-header" id="headingmissing{{ side }}">
+                        <h2 class="mb-0">
+                            <button class="btn btn-link" type="button" data-toggle="collapse" data-target="#collapsemissing{{ side }}" aria-expanded="true" aria-controls="collapsemissing{{ side }}">
+                                Rows in {{ side|code }} that aren't in the other table.
+                            </button>
+                        </h2>
+                    </div>
+                    <div id="collapsemissing{{ side }}" class="collapse show" aria-labelledby="headingmissing{{ side }}" data-parent="#accordionExample">
+                        <div class="card-body overflow-auto">
+                            {{ info.sample|dfhtml|safe }}
+                            <h3>Query:</h3>
+                            <pre>{{ info.query|code("pgsql") }}</pre>
+                        </div>
+                    </div>
+                </div>
+                {% endif %}
+                {% endfor %}
+                {% for col, m in hierarchical_join_info.items() %}
+                {% for side, info in m.items() %}
+                {% if info.count > 0 %}
+                <div class="card">
+                    <div class="card-header" id="headingx{{ col }}{{ info.count }}">
+                        <h2 class="mb-0">
+                            <button class="btn btn-link" type="button" data-toggle="collapse" data-target="#collapsex{{ col }}{{ side }}{{ info.count }}" aria-expanded="true" aria-controls="collapsex{{ col }}{{ col }}{{ info.count }}">
+                                Rows in table {{ side|code }} that aren't in the other table for column {{ col|code }} (sample out of {{ info.count|comma }})
+                            </button>
+                        </h2>
+                    </div>
+                    <div id="collapsex{{ col }}{{ side }}{{ info.count }}" class="collapse show" aria-labelledby="headingx{{ col }}{{ side }}{{ info.count }}" data-parent="#accordionExample">
+                        <div class="card-body overflow-auto">
+                            {{ info.sample|dfhtml|safe }}
+                            {% if "sample_grouped" in info %}
+                            <h4>Grouped:</h4>
+                            {{ info.sample_grouped|dfhtml|safe }}
+                            {% endif %}
+                            <h3>Queries:</h3>
+                            <pre>{{ info.query|code("pgsql") }}</pre>
+                            {% if "sample_grouped" in info %}
+                            <pre>{{ info.query_grouped|code("pgsql") }}</pre>
+                            {% endif %}
+                        </div>
+                    </div>
+                </div>
+                {% endif %}
+                {% endfor %}
+                {% endfor %}
+                {% if diff_summary.count %}
+                <div class="card">
+                    <div class="card-header" id="headingThree">
+                        <h2 class="mb-0">
+                            <button class="btn btn-link" type="button" data-toggle="collapse" data-target="#collapseThree" aria-expanded="true" aria-controls="collapseThree">
+                                Sample mismatched rows
+                            </button>
+                        </h2>
+                    </div>
+                    <div id="collapseThree" class="collapse show" aria-labelledby="headingThree" data-parent="#accordionExample">
+                        <div class="card-body overflow-auto">
+                            {{ diff_summary.sample|dfhtml|safe }}
+                        </div>
+                    </div>
+                </div>
+                {% endif %}
+                {% for column, info in column_info.items() %}
+                <div class="card">
+                    <div class="card-header" id="heading{{ loop.index }}">
+                        <h2 class="mb-0">
+                            <button class="btn btn-link collapsed" type="button" data-toggle="collapse" data-target="#collapse{{ loop.index }}" aria-expanded="false" aria-controls="collapse{{ loop.index }}">
+                                Column {{ column|code("highlighter-rogue") }} has {{ info.count|comma }} differences.
+                            </button>
+                        </h2>
+                    </div>
+                    <div id="collapse{{ loop.index }}" class="collapse" aria-labelledby="heading{{ loop.index }}" data-parent="#accordionExample">
+                        <div class="card-body overflow-auto">
+                            <h3>Grouped differences, 100 most common:</h3>
+                            <p>
+                                {{ info.df|dfhtml|safe }}
+                            </p>
+                            <h3>Raw differences samples:</h3>
+                            <p>
+                                {{ info.df_raw|dfhtml|safe }}
+                            </p>
+                            {% if info.q_h_x %}
+                            <h3>Raw differences by first join column sample for x:</h3>
+                            <p>
+                                {{ info.df_h_x|dfhtml|safe }}
+                            </p>
+                            <h3>Raw differences by first join column sample for y:</h3>
+                            <p>
+                                {{ info.df_h_y|dfhtml|safe }}
+                            </p>
+                            {% endif %}
+                            {% if info.q_n %}
+                            <h3>Binned differences between values:</h3>
+                            <p>
+                                {{ info.df_n|dfhtml|safe }}
+                            </p>
+                            <h3>Biggest differences between values:</h3>
+                            <p>
+                                {{ info.df_n_sample|dfhtml|safe }}
+                            </p>
+                            {% endif %}
+                            <hr class="my-4">
+                            <h3>Query for grouped differences:</h3>
+                            <pre>{{ info.q|code("pgsql") }}</pre>
+                            <h3>Query for raw differences:</h3>
+                            <pre>{{ info.q_raw|code("pgsql") }}</pre>
+                            {% if info.q_h_x %}
+                            <h3>Query for raw differences by first join column in x:</h3>
+                            <pre>{{ info.q_h_x|code("pgsql") }}</pre>
+                            <h3>Query for raw differences by first join column in y:</h3>
+                            <pre>{{ info.q_h_y|code("pgsql") }}</pre>
+                            {% endif %}
+                            {% if info.q_n %}
+                            <h3>Query for grouped numeric differences:</h3>
+                            <pre>{{ info.q_n|code("pgsql") }}</pre>
+                            <h3>Query for sample biggest numeric differences:</h3>
+                            <pre>{{ info.q_n_sample|code("pgsql") }}</pre>
+                            {% endif %}
+                        </div>
+                    </div>
+                </div>
+                {% endfor %}
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}
+
+{% block bodyextrajs %}
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/highlight.min.js" integrity="sha256-1zu+3BnLYV9LdiY85uXMzii3bdrkelyp37e0ZyTAQh0=" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.10/languages/pgsql.min.js" integrity="sha256-kS8L87/ytXlzSR5Zcd2qCLLg8j0uVuGcirrE9+Oi7Ns=" crossorigin="anonymous"></script>
+<script>
+    hljs.initHighlightingOnLoad();
+</script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script>
+{% endblock %}

dbdiff/templates/insert_diff.sql

@@ -0,0 +1,9 @@
+   INSERT INTO {{ diff_schema }}.{{ diff_table }} ( {{ join_cols|join(", ") }}, column_name )
+        SELECT {{ join_cols|join(", ") }},
+               '{{ column }}' as column_name
+          FROM (
+        SELECT {{ join_cols|join(", ") }},
+               x_{{ column }} <=> y_{{ column }} AS {{ column }}_eq
+          FROM {{ joined_schema }}.{{ joined_table }}
+               ) AS t1
+         WHERE t1.{{ column }}_eq IS FALSE;