pipeverify 0.1.0__tar.gz → 0.2.0__tar.gz

{pipeverify-0.1.0 → pipeverify-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipeverify
-Version: 0.1.0
+Version: 0.2.0
 Summary: Check that a pipeline change left the data right, not just the job green.
 Project-URL: Homepage, https://github.com/akshaysharma1088/pipeverify
 Project-URL: Issues, https://github.com/akshaysharma1088/pipeverify/issues
@@ -37,6 +37,8 @@ Description-Content-Type: text/markdown
 [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
 ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
 ![Status](https://img.shields.io/badge/status-alpha-orange)
+[![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542278.svg)](https://doi.org/10.5281/zenodo.23542278)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -161,6 +163,44 @@ Exit code is `1` when the verdict reaches `--fail-on` (default `block`), otherwi
 Columns listed in `expect_change` are reported but never raise the verdict. All thresholds can be
 changed globally or per table; see [`pipeverify/checks.py`](pipeverify/checks.py) for every option.
 
+## Source checks
+
+A before/after diff only sees damage to rows that already existed. Errors in **new** data need
+evidence the pipeline does not control: the raw source. Add a `source` block to a table:
+
+```yaml
+tables:
+  - name: silver.transactions
+    after: snapshots/pr/silver_transactions.parquet
+    before: snapshots/main/silver_transactions.parquet   # optional
+    key: [txn_id]
+    source:
+      data: landing/raw_transactions.parquet
+      key: [txn_id]
+      columns: {amount: amount}      # output column: source column
+      checks: [conflict]             # default; also mismatch, missing, orphan
+      where: "status = 'POSTED'"     # source rows that should reach the output
+```
+
+| Check | Catches | Default |
+|---|---|---|
+| `conflict` | The source sent several values for one key and the output silently kept one | on, REVIEW |
+| `mismatch` | An output value the source never sent (use only for pass-through columns) | opt-in |
+| `missing` | Source keys that never reached the output (set `where` for intended filters) | opt-in |
+| `orphan` | Output keys that are not in the source | opt-in |
+
+From the command line:
+
+```bash
+pipeverify compare - silver.parquet -k txn_id --source raw.parquet --source-column amount
+```
+
+In a pilot on the 209 executed fixes in the PipeMend benchmark (25 harmful), production-style
+monitors caught 16 harmful fixes, and the source conflict check caught the 9 that the monitors
+approved. Together they caught all 25, at the cost of a REVIEW on 21 of 184 correct fixes,
+all on incidents where the source data really was ambiguous. This is a small pilot on synthetic
+data; a larger evaluation is in progress.
+
 ## GitHub Action
 
 ```yaml

{pipeverify-0.1.0 → pipeverify-0.2.0}/README.md

@@ -6,6 +6,8 @@
 [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
 ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
 ![Status](https://img.shields.io/badge/status-alpha-orange)
+[![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542278.svg)](https://doi.org/10.5281/zenodo.23542278)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -130,6 +132,44 @@ Exit code is `1` when the verdict reaches `--fail-on` (default `block`), otherwi
 Columns listed in `expect_change` are reported but never raise the verdict. All thresholds can be
 changed globally or per table; see [`pipeverify/checks.py`](pipeverify/checks.py) for every option.
 
+## Source checks
+
+A before/after diff only sees damage to rows that already existed. Errors in **new** data need
+evidence the pipeline does not control: the raw source. Add a `source` block to a table:
+
+```yaml
+tables:
+  - name: silver.transactions
+    after: snapshots/pr/silver_transactions.parquet
+    before: snapshots/main/silver_transactions.parquet   # optional
+    key: [txn_id]
+    source:
+      data: landing/raw_transactions.parquet
+      key: [txn_id]
+      columns: {amount: amount}      # output column: source column
+      checks: [conflict]             # default; also mismatch, missing, orphan
+      where: "status = 'POSTED'"     # source rows that should reach the output
+```
+
+| Check | Catches | Default |
+|---|---|---|
+| `conflict` | The source sent several values for one key and the output silently kept one | on, REVIEW |
+| `mismatch` | An output value the source never sent (use only for pass-through columns) | opt-in |
+| `missing` | Source keys that never reached the output (set `where` for intended filters) | opt-in |
+| `orphan` | Output keys that are not in the source | opt-in |
+
+From the command line:
+
+```bash
+pipeverify compare - silver.parquet -k txn_id --source raw.parquet --source-column amount
+```
+
+In a pilot on the 209 executed fixes in the PipeMend benchmark (25 harmful), production-style
+monitors caught 16 harmful fixes, and the source conflict check caught the 9 that the monitors
+approved. Together they caught all 25, at the cost of a REVIEW on 21 of 184 correct fixes,
+all on incidents where the source data really was ambiguous. This is a small pilot on synthetic
+data; a larger evaluation is in progress.
+
 ## GitHub Action
 
 ```yaml

{pipeverify-0.1.0 → pipeverify-0.2.0}/pipeverify/__init__.py

@@ -1,6 +1,6 @@
 """pipeverify: check that a pipeline change left the data right, not just the job green."""
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 from .checks import Finding, Thresholds  # noqa: E402
 from .core import RunResult, TableResult, run_config, verify  # noqa: E402

{pipeverify-0.1.0 → pipeverify-0.2.0}/pipeverify/checks.py

@@ -38,6 +38,10 @@ class Thresholds:
     sum_shift_block_pct: float = 5.0
     category_max_distinct: int = 50          # treat text columns with few values as categories
     float_tolerance: float = 1e-9
+    source_mismatch_review_pct: float = 0.0
+    source_mismatch_block_pct: float = 5.0
+    source_missing_review_pct: float = 0.0
+    source_missing_block_pct: float = 5.0
 
     @classmethod
     def from_dict(cls, data: dict[str, Any] | None) -> "Thresholds":

{pipeverify-0.1.0 → pipeverify-0.2.0}/pipeverify/cli.py

@@ -97,27 +97,40 @@ def main() -> None:
 @main.command()
 @click.argument("before")
 @click.argument("after")
+@click.option("--source", "source_data", help="Raw source table the output was built from (enables source checks).")
+@click.option("--source-key", help="Key column(s) in the source, if named differently from --key.")
+@click.option("--source-column", "source_columns", multiple=True,
+              help="Column to check against the source, as out=src or just name. Repeatable.")
+@click.option("--source-where", help="SQL filter for source rows that should reach the output.")
 @click.option("--key", "-k", help="Comma-separated key column(s) that identify a row.")
 @click.option("--expect-change", "-e", multiple=True, help="Column this change is meant to modify. Repeatable.")
 @click.option("--ignore", "-i", multiple=True, help="Column to skip, such as a load timestamp. Repeatable.")
 @click.option("--expect-row-change", is_flag=True, help="Rows are expected to be added or removed.")
 @click.option("--name", default=None, help="Name to show for the table.")
 @with_output_options
-def compare(before, after, key, expect_change, ignore, expect_row_change, name,
+def compare(before, after, source_data, source_key, source_columns, source_where,
+            key, expect_change, ignore, expect_row_change, name,
             fmt, output, summary, fail_on, explain, show_info, json_out, markdown_out):
     """Compare one table BEFORE and AFTER a change.
 
+    Use - as BEFORE to run only the source checks.
+
     \b
     Examples:
       pipeverify compare before.parquet after.parquet --key order_id
+      pipeverify compare - silver.parquet -k txn_id --source raw.parquet --source-column amount
       pipeverify compare duckdb:prod.db:gold.sales duckdb:dev.db:gold.sales -k id
     """
     import time
     start = time.perf_counter()
+    source = None
+    if source_data:
+        cols = dict((c.split("=", 1) if "=" in c else (c, c)) for c in source_columns)
+        source = {"data": source_data, "key": source_key, "columns": cols, "where": source_where}
     try:
-        res = verify(before, after, name=name or os.path.basename(str(after)), key=key,
+        res = verify(None if before == "-" else before, after, name=name or os.path.basename(str(after)), key=key,
                      expect_change=list(expect_change), ignore=list(ignore),
-                     expect_row_change=expect_row_change)
+                     expect_row_change=expect_row_change, source=source)
     except Exception as exc:
         raise click.ClickException(str(exc))
     result = RunResult(tables=[res], seconds=time.perf_counter() - start)

{pipeverify-0.1.0 → pipeverify-0.2.0}/pipeverify/core.py

@@ -10,6 +10,7 @@ import duckdb
 import yaml
 
 from .checks import BLOCK, PASS, RANK, REVIEW, Finding, Thresholds, run_checks, verdict
+from .source_checks import SourceSpec, run_source_checks
 from .sources import load, parse_source
 
 
@@ -50,33 +51,47 @@ class RunResult:
 def verify(before: Any, after: Any, *, name: str = "table", key: list[str] | str | None = None,
            expect_change: list[str] | None = None, ignore: list[str] | None = None,
            expect_row_change: bool = False, thresholds: Thresholds | dict | None = None,
-           base_dir: str = ".") -> TableResult:
-    """Compare two snapshots of one table.
+           source: SourceSpec | dict | None = None, base_dir: str = ".") -> TableResult:
+    """Compare two snapshots of one table, and optionally the output with its raw source.
 
-    `before` and `after` can be file paths, duckdb:<file>:<table> strings,
-    config mappings, or pandas DataFrames.
+    `before`, `after` and `source["data"]` can be file paths, duckdb:<file>:<table>
+    strings, config mappings, or pandas DataFrames. `before` may be None when only
+    source checks are wanted (for example when no earlier snapshot exists).
     """
     if isinstance(key, str):
         key = [k.strip() for k in key.split(",") if k.strip()]
     if not isinstance(thresholds, Thresholds):
         thresholds = Thresholds.from_dict(thresholds)
-    src_b, src_a = parse_source(before, base_dir), parse_source(after, base_dir)
+    if before is None and source is None:
+        raise ValueError("Give a before snapshot, a source, or both")
+    src_b = parse_source(before, base_dir) if before is not None else None
+    src_a = parse_source(after, base_dir)
     con = duckdb.connect()
     try:
-        load(con, src_b, "before")
         load(con, src_a, "after")
-        findings, stats = run_checks(con, "before", "after", key=key, expect_change=expect_change,
-                                     ignore=ignore, expect_row_change=expect_row_change,
-                                     thresholds=thresholds)
-        if not key:
-            findings.append(Finding("keys", "info",
-                                    "No key given, so row-level changes were not compared. Add a key for stronger checks."))
+        if src_b is not None:
+            load(con, src_b, "before")
+            findings, stats = run_checks(con, "before", "after", key=key, expect_change=expect_change,
+                                         ignore=ignore, expect_row_change=expect_row_change,
+                                         thresholds=thresholds)
+            if not key:
+                findings.append(Finding("keys", "info",
+                                        "No key given, so row-level changes were not compared. Add a key for stronger checks."))
+        else:
+            findings = []
+            stats = {"rows_after": con.execute('SELECT count(*) FROM "after"').fetchone()[0]}
+        if source is not None:
+            spec = SourceSpec.from_dict(source)
+            load(con, parse_source(spec.data, base_dir), "source")
+            sf, sstats = run_source_checks(con, "after", "source", spec, key or [], thresholds)
+            findings += sf
+            stats.update(sstats)
     finally:
         con.close()
     order = {BLOCK: 0, REVIEW: 1, "info": 2, PASS: 3}
     findings.sort(key=lambda f: order[f.severity])
     return TableResult(name=name, verdict=verdict(findings), findings=findings, stats=stats,
-                       before=src_b.describe(), after=src_a.describe())
+                       before=src_b.describe() if src_b else "(none)", after=src_a.describe())
 
 
 def load_config(path: str) -> dict[str, Any]:
@@ -99,10 +114,10 @@ def run_config(path: str, only: list[str] | None = None) -> RunResult:
             continue
         th = {**defaults, **(spec.get("thresholds") or {})}
         try:
-            res = verify(spec["before"], spec["after"], name=str(name), key=spec.get("key"),
+            res = verify(spec.get("before"), spec["after"], name=str(name), key=spec.get("key"),
                          expect_change=spec.get("expect_change"), ignore=spec.get("ignore"),
                          expect_row_change=bool(spec.get("expect_row_change", False)),
-                         thresholds=th, base_dir=base_dir)
+                         thresholds=th, source=spec.get("source"), base_dir=base_dir)
         except Exception as exc:  # report the table as blocked, keep checking the rest
             res = TableResult(name=str(name), verdict=BLOCK, stats={},
                               findings=[Finding("error", BLOCK, f"Could not check this table: {exc}")],

{pipeverify-0.1.0 → pipeverify-0.2.0}/pipeverify/report.py

@@ -36,7 +36,10 @@ def to_text(result: RunResult, show_info: bool = False) -> str:
         s = t.stats
         if s:
             extra = f", matched {s['matched_rows']:,}" if "matched_rows" in s else ""
-            lines.append(f"        rows {s.get('rows_before', 0):,} -> {s.get('rows_after', 0):,}{extra}")
+            if "rows_before" in s:
+                lines.append(f"        rows {s['rows_before']:,} -> {s.get('rows_after', 0):,}{extra}")
+            else:
+                lines.append(f"        rows {s.get('rows_after', 0):,} (checked against source)")
         for f in t.findings:
             if f.severity == "info" and not show_info:
                 continue
@@ -57,7 +60,12 @@ def to_markdown(result: RunResult) -> str:
     for t in result.tables:
         issues = sum(1 for f in t.findings if f.severity in ("review", "block"))
         s = t.stats or {}
-        rows = f"{s.get('rows_before', '?'):,} → {s.get('rows_after', '?'):,}" if s else "n/a"
+        if "rows_before" in s:
+            rows = f"{s['rows_before']:,} → {s.get('rows_after', 0):,}"
+        elif "rows_after" in s:
+            rows = f"{s['rows_after']:,} (source check)"
+        else:
+            rows = "n/a"
         out.append(f"| `{t.name}` | {ICON[t.verdict]} {LABEL[t.verdict]} | {rows} | {issues} |")
     for t in result.tables:
         notable = [f for f in t.findings if f.severity in ("review", "block")]

pipeverify-0.2.0/pipeverify/source_checks.py

@@ -0,0 +1,125 @@
+"""Checks against the source: compare an output table with the raw data it was built from.
+
+A before/after diff can only see damage to rows that existed before. Errors in new data
+need evidence the pipeline does not control. These checks use the raw source table:
+
+  source_conflict   the source sent several different values for one key and the output
+                    kept one of them, so a choice was made that a person should confirm
+  source_mismatch   an output value matches none of the values the source sent for that key
+  source_missing    keys in the source (after an optional filter) that never reached the output
+  source_orphan     keys in the output that do not exist in the source
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import duckdb
+
+from .checks import BLOCK, INFO, PASS, REVIEW, Finding, Thresholds, q
+
+
+@dataclass
+class SourceSpec:
+    data: Any
+    key: list[str] | None = None            # key column(s) in the source, default: the table key
+    columns: dict[str, str] = field(default_factory=dict)  # output column -> source column
+    where: str | None = None                # SQL filter for source rows that should reach the output
+    checks: list[str] | None = None         # default ["conflict"]; also mismatch, missing, orphan
+
+    @classmethod
+    def from_dict(cls, d: Any) -> "SourceSpec":
+        if isinstance(d, SourceSpec):
+            return d
+        if not isinstance(d, dict) or "data" not in d:
+            raise ValueError("source needs at least 'data' (where the raw table is)")
+        cols = d.get("columns") or {}
+        if isinstance(cols, list):
+            cols = {c: c for c in cols}
+        key = d.get("key")
+        if isinstance(key, str):
+            key = [k.strip() for k in key.split(",") if k.strip()]
+        return cls(data=d["data"], key=key, columns=dict(cols), where=d.get("where"),
+                   checks=d.get("checks"))
+
+
+def _grade(pct: float, review: float, block: float) -> str:
+    if pct >= block and pct > 0:
+        return BLOCK
+    if pct > review or (review == 0 and pct > 0):
+        return REVIEW
+    return PASS
+
+
+def run_source_checks(con: duckdb.DuckDBPyConnection, after: str, source: str, spec: SourceSpec,
+                      key: list[str], t: Thresholds) -> tuple[list[Finding], dict[str, Any]]:
+    if not key:
+        raise ValueError("Source checks need a key on the table")
+    skey = spec.key or key
+    if len(skey) != len(key):
+        raise ValueError("The source key must have as many columns as the table key")
+    wanted = set(spec.checks or ["conflict"])  # others are opt-in: they need a filter or pass-through columns
+    where = f"WHERE {spec.where}" if spec.where else ""
+    # normalized source view: key columns renamed to the output's key names, compared as text
+    sel = [f"CAST({q(s)} AS VARCHAR) AS {q('_k' + str(i))}" for i, s in enumerate(skey)]
+    sel += [f"{q(sc)} AS {q('_v_' + oc)}" for oc, sc in spec.columns.items()]
+    con.execute(f"CREATE OR REPLACE TEMP VIEW _pv_src AS SELECT {', '.join(sel)} FROM {q(source)} {where}")
+    akeys = ", ".join(f"CAST({q(k)} AS VARCHAR) AS {q('_k' + str(i))}" for i, k in enumerate(key))
+    con.execute(f"CREATE OR REPLACE TEMP VIEW _pv_out AS SELECT {akeys}, * FROM {q(after)}")
+    on = " AND ".join(f"o.{q('_k' + str(i))} IS NOT DISTINCT FROM s.{q('_k' + str(i))}" for i in range(len(key)))
+    kcols = ", ".join(q("_k" + str(i)) for i in range(len(key)))
+
+    findings: list[Finding] = []
+    n_out = con.execute("SELECT count(*) FROM _pv_out").fetchone()[0]
+    n_src = con.execute(f"SELECT count(DISTINCT ({kcols})) FROM _pv_src").fetchone()[0]
+    stats = {"source_keys": n_src, "output_rows": n_out}
+
+    for oc in spec.columns:
+        v = q("_v_" + oc)
+        if "conflict" in wanted:
+            n = con.execute(
+                f"SELECT count(*) FROM (SELECT {kcols} FROM _pv_src GROUP BY ALL "
+                f"HAVING count(DISTINCT CAST({v} AS VARCHAR)) > 1) s "
+                f"WHERE EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchone()[0]
+            if n:
+                ex = con.execute(
+                    f"SELECT {kcols}, list(DISTINCT CAST({v} AS VARCHAR) ORDER BY CAST({v} AS VARCHAR)) FROM _pv_src "
+                    f"GROUP BY ALL HAVING count(DISTINCT CAST({v} AS VARCHAR)) > 1 LIMIT 3").fetchall()
+                pct = n / max(n_out, 1) * 100
+                findings.append(Finding(
+                    "source_conflict", REVIEW,
+                    f"{n:,} key(s) arrived from the source with conflicting '{oc}' values and the output kept one "
+                    f"({pct:.2f}% of rows). Confirm which value is right.",
+                    oc, {"count": n, "pct": round(pct, 4), "examples": [[str(x) for x in r] for r in ex]}))
+        if "mismatch" in wanted:
+            num = (f"TRY_CAST(o.{q(oc)} AS DOUBLE) IS NOT NULL AND TRY_CAST(s.{v} AS DOUBLE) IS NOT NULL AND "
+                   f"abs(TRY_CAST(o.{q(oc)} AS DOUBLE) - TRY_CAST(s.{v} AS DOUBLE)) <= "
+                   f"{t.float_tolerance} * greatest(1.0, abs(TRY_CAST(s.{v} AS DOUBLE)))")
+            txt = f"trim(CAST(o.{q(oc)} AS VARCHAR)) = trim(CAST(s.{v} AS VARCHAR))"
+            n = con.execute(
+                f"SELECT count(*) FROM _pv_out o WHERE EXISTS (SELECT 1 FROM _pv_src s WHERE {on}) "
+                f"AND NOT EXISTS (SELECT 1 FROM _pv_src s WHERE {on} AND (({num}) OR {txt} "
+                f"OR (o.{q(oc)} IS NULL AND s.{v} IS NULL)))").fetchone()[0]
+            if n:
+                pct = n / max(n_out, 1) * 100
+                sev = _grade(pct, t.source_mismatch_review_pct, t.source_mismatch_block_pct)
+                findings.append(Finding("source_mismatch", sev,
+                                        f"{n:,} row(s) have a '{oc}' value the source never sent ({pct:.2f}%)",
+                                        oc, {"count": n, "pct": round(pct, 4)}))
+    if "missing" in wanted:
+        n = con.execute(f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM _pv_src) s "
+                        f"WHERE NOT EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchone()[0]
+        if n:
+            pct = n / max(n_src, 1) * 100
+            findings.append(Finding("source_missing", _grade(pct, t.source_missing_review_pct, t.source_missing_block_pct),
+                                    f"{n:,} source key(s) never reached the output ({pct:.2f}%)", None,
+                                    {"count": n, "pct": round(pct, 4)}))
+    if "orphan" in wanted:
+        n = con.execute(f"SELECT count(*) FROM _pv_out o WHERE NOT EXISTS (SELECT 1 FROM _pv_src s WHERE {on})").fetchone()[0]
+        if n:
+            pct = n / max(n_out, 1) * 100
+            findings.append(Finding("source_orphan", REVIEW if pct > 0 else INFO,
+                                    f"{n:,} output row(s) have keys that are not in the source ({pct:.2f}%)", None,
+                                    {"count": n, "pct": round(pct, 4)}))
+    stats["source_checks"] = sorted(wanted)
+    return findings, stats

{pipeverify-0.1.0 → pipeverify-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pipeverify"
-version = "0.1.0"
+version = "0.2.0"
 description = "Check that a pipeline change left the data right, not just the job green."
 readme = "README.md"
 license = "Apache-2.0"