pipeverify 0.2.0__tar.gz → 0.2.1__tar.gz

{pipeverify-0.2.0 → pipeverify-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipeverify
-Version: 0.2.0
+Version: 0.2.1
 Summary: Check that a pipeline change left the data right, not just the job green.
 Project-URL: Homepage, https://github.com/akshaysharma1088/pipeverify
 Project-URL: Issues, https://github.com/akshaysharma1088/pipeverify/issues
@@ -38,7 +38,7 @@ Description-Content-Type: text/markdown
 ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
 ![Status](https://img.shields.io/badge/status-alpha-orange)
 [![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542278.svg)](https://doi.org/10.5281/zenodo.23542278)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -215,7 +215,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # ... build the "after" snapshots for this PR here ...
-      - uses: akshaysharma1088/pipeverify@v0.1.0
+      - uses: akshaysharma1088/pipeverify@v0.2.0
         with:
           config: pipeverify.yml
           fail-on: block      # or review, or never

{pipeverify-0.2.0 → pipeverify-0.2.1}/README.md

@@ -7,7 +7,7 @@
 ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
 ![Status](https://img.shields.io/badge/status-alpha-orange)
 [![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542278.svg)](https://doi.org/10.5281/zenodo.23542278)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -184,7 +184,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # ... build the "after" snapshots for this PR here ...
-      - uses: akshaysharma1088/pipeverify@v0.1.0
+      - uses: akshaysharma1088/pipeverify@v0.2.0
         with:
           config: pipeverify.yml
           fail-on: block      # or review, or never

{pipeverify-0.2.0 → pipeverify-0.2.1}/pipeverify/__init__.py

@@ -1,6 +1,6 @@
 """pipeverify: check that a pipeline change left the data right, not just the job green."""
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from .checks import Finding, Thresholds  # noqa: E402
 from .core import RunResult, TableResult, run_config, verify  # noqa: E402

{pipeverify-0.2.0 → pipeverify-0.2.1}/pipeverify/checks.py

@@ -113,8 +113,8 @@ def run_checks(
         if k not in sb or k not in sa:
             raise ValueError(f"Key column '{k}' is missing from the before or after table")
 
-    n_before = con.execute(f"SELECT count(*) FROM {q(before)}").fetchone()[0]
-    n_after = con.execute(f"SELECT count(*) FROM {q(after)}").fetchone()[0]
+    n_before = con.execute(f"SELECT count(*) FROM {q(before)}").fetchall()[0][0]
+    n_after = con.execute(f"SELECT count(*) FROM {q(after)}").fetchall()[0][0]
     stats: dict[str, Any] = {"rows_before": n_before, "rows_after": n_after,
                              "columns_before": len(sb), "columns_after": len(sa)}
 
@@ -152,9 +152,9 @@ def run_checks(
     if key:
         kcols = ", ".join(q(k) for k in key)
         dup_before = con.execute(
-            f"SELECT count(*) FROM (SELECT {kcols} FROM {q(before)} GROUP BY ALL HAVING count(*) > 1)").fetchone()[0]
+            f"SELECT count(*) FROM (SELECT {kcols} FROM {q(before)} GROUP BY ALL HAVING count(*) > 1)").fetchall()[0][0]
         dup_after = con.execute(
-            f"SELECT count(*) FROM (SELECT {kcols} FROM {q(after)} GROUP BY ALL HAVING count(*) > 1)").fetchone()[0]
+            f"SELECT count(*) FROM (SELECT {kcols} FROM {q(after)} GROUP BY ALL HAVING count(*) > 1)").fetchall()[0][0]
         stats.update(duplicate_keys_before=dup_before, duplicate_keys_after=dup_after)
         if dup_after > dup_before:
             findings.append(Finding("duplicate_keys", BLOCK,
@@ -167,11 +167,11 @@ def run_checks(
         on = " AND ".join(f"b.{q(k)} IS NOT DISTINCT FROM a.{q(k)}" for k in key)
         missing = con.execute(
             f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM {q(before)}) b "
-            f"WHERE NOT EXISTS (SELECT 1 FROM {q(after)} a WHERE {on})").fetchone()[0]
+            f"WHERE NOT EXISTS (SELECT 1 FROM {q(after)} a WHERE {on})").fetchall()[0][0]
         new = con.execute(
             f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM {q(after)}) a "
-            f"WHERE NOT EXISTS (SELECT 1 FROM {q(before)} b WHERE {on})").fetchone()[0]
-        distinct_before = con.execute(f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM {q(before)})").fetchone()[0]
+            f"WHERE NOT EXISTS (SELECT 1 FROM {q(before)} b WHERE {on})").fetchall()[0][0]
+        distinct_before = con.execute(f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM {q(before)})").fetchall()[0][0]
         stats.update(missing_keys=missing, new_keys=new)
         if missing:
             pct = missing / max(distinct_before, 1) * 100
@@ -193,7 +193,7 @@ def run_checks(
     for c in common:
         nb, na = con.execute(
             f"SELECT (SELECT avg(CASE WHEN {q(c)} IS NULL THEN 1.0 ELSE 0 END) FROM {q(before)}), "
-            f"(SELECT avg(CASE WHEN {q(c)} IS NULL THEN 1.0 ELSE 0 END) FROM {q(after)})").fetchone()
+            f"(SELECT avg(CASE WHEN {q(c)} IS NULL THEN 1.0 ELSE 0 END) FROM {q(after)})").fetchall()[0]
         nb, na = float(nb or 0) * 100, float(na or 0) * 100
         rise = na - nb
         if rise > 0:
@@ -207,7 +207,7 @@ def run_checks(
 
     # 5. changed values on matched keys
     if unique_keys_join is not None:
-        matched = con.execute(f"SELECT count(*) FROM _pv_ub b JOIN _pv_ua a ON {unique_keys_join}").fetchone()[0]
+        matched = con.execute(f"SELECT count(*) FROM _pv_ub b JOIN _pv_ua a ON {unique_keys_join}").fetchall()[0][0]
         stats["matched_rows"] = matched
         for c in common:
             if c in key:
@@ -219,7 +219,7 @@ def run_checks(
             else:
                 diff = f"CAST(b.{q(c)} AS VARCHAR) IS DISTINCT FROM CAST(a.{q(c)} AS VARCHAR)"
             changed = con.execute(
-                f"SELECT count(*) FROM _pv_ub b JOIN _pv_ua a ON {unique_keys_join} WHERE {diff}").fetchone()[0]
+                f"SELECT count(*) FROM _pv_ub b JOIN _pv_ua a ON {unique_keys_join} WHERE {diff}").fetchall()[0][0]
             if not changed:
                 continue
             pct = changed / max(matched, 1) * 100
@@ -243,7 +243,7 @@ def run_checks(
             continue
         tb, ta = con.execute(
             f"SELECT (SELECT sum(CAST({q(c)} AS DOUBLE)) FROM {q(before)}), "
-            f"(SELECT sum(CAST({q(c)} AS DOUBLE)) FROM {q(after)})").fetchone()
+            f"(SELECT sum(CAST({q(c)} AS DOUBLE)) FROM {q(after)})").fetchall()[0]
         if tb is None and ta is None:
             continue
         tb, ta = float(tb or 0), float(ta or 0)
@@ -263,7 +263,7 @@ def run_checks(
     for c in common:
         if c in key or not (_is_text(sb[c]) and _is_text(sa[c])):
             continue
-        d = con.execute(f"SELECT count(DISTINCT {q(c)}) FROM {q(before)}").fetchone()[0]
+        d = con.execute(f"SELECT count(DISTINCT {q(c)}) FROM {q(before)}").fetchall()[0][0]
         if d == 0 or d > t.category_max_distinct:
             continue
         new_vals = [r[0] for r in con.execute(

{pipeverify-0.2.0 → pipeverify-0.2.1}/pipeverify/core.py

@@ -79,7 +79,7 @@ def verify(before: Any, after: Any, *, name: str = "table", key: list[str] | str
                                         "No key given, so row-level changes were not compared. Add a key for stronger checks."))
         else:
             findings = []
-            stats = {"rows_after": con.execute('SELECT count(*) FROM "after"').fetchone()[0]}
+            stats = {"rows_after": con.execute('SELECT count(*) FROM "after"').fetchall()[0][0]}
         if source is not None:
             spec = SourceSpec.from_dict(source)
             load(con, parse_source(spec.data, base_dir), "source")

{pipeverify-0.2.0 → pipeverify-0.2.1}/pipeverify/source_checks.py

@@ -70,8 +70,8 @@ def run_source_checks(con: duckdb.DuckDBPyConnection, after: str, source: str, s
     kcols = ", ".join(q("_k" + str(i)) for i in range(len(key)))
 
     findings: list[Finding] = []
-    n_out = con.execute("SELECT count(*) FROM _pv_out").fetchone()[0]
-    n_src = con.execute(f"SELECT count(DISTINCT ({kcols})) FROM _pv_src").fetchone()[0]
+    n_out = con.execute("SELECT count(*) FROM _pv_out").fetchall()[0][0]
+    n_src = con.execute(f"SELECT count(DISTINCT ({kcols})) FROM _pv_src").fetchall()[0][0]
     stats = {"source_keys": n_src, "output_rows": n_out}
 
     for oc in spec.columns:
@@ -80,7 +80,7 @@ def run_source_checks(con: duckdb.DuckDBPyConnection, after: str, source: str, s
             n = con.execute(
                 f"SELECT count(*) FROM (SELECT {kcols} FROM _pv_src GROUP BY ALL "
                 f"HAVING count(DISTINCT CAST({v} AS VARCHAR)) > 1) s "
-                f"WHERE EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchone()[0]
+                f"WHERE EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchall()[0][0]
             if n:
                 ex = con.execute(
                     f"SELECT {kcols}, list(DISTINCT CAST({v} AS VARCHAR) ORDER BY CAST({v} AS VARCHAR)) FROM _pv_src "
@@ -99,7 +99,7 @@ def run_source_checks(con: duckdb.DuckDBPyConnection, after: str, source: str, s
             n = con.execute(
                 f"SELECT count(*) FROM _pv_out o WHERE EXISTS (SELECT 1 FROM _pv_src s WHERE {on}) "
                 f"AND NOT EXISTS (SELECT 1 FROM _pv_src s WHERE {on} AND (({num}) OR {txt} "
-                f"OR (o.{q(oc)} IS NULL AND s.{v} IS NULL)))").fetchone()[0]
+                f"OR (o.{q(oc)} IS NULL AND s.{v} IS NULL)))").fetchall()[0][0]
             if n:
                 pct = n / max(n_out, 1) * 100
                 sev = _grade(pct, t.source_mismatch_review_pct, t.source_mismatch_block_pct)
@@ -108,14 +108,14 @@ def run_source_checks(con: duckdb.DuckDBPyConnection, after: str, source: str, s
                                         oc, {"count": n, "pct": round(pct, 4)}))
     if "missing" in wanted:
         n = con.execute(f"SELECT count(*) FROM (SELECT DISTINCT {kcols} FROM _pv_src) s "
-                        f"WHERE NOT EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchone()[0]
+                        f"WHERE NOT EXISTS (SELECT 1 FROM _pv_out o WHERE {on})").fetchall()[0][0]
         if n:
             pct = n / max(n_src, 1) * 100
             findings.append(Finding("source_missing", _grade(pct, t.source_missing_review_pct, t.source_missing_block_pct),
                                     f"{n:,} source key(s) never reached the output ({pct:.2f}%)", None,
                                     {"count": n, "pct": round(pct, 4)}))
     if "orphan" in wanted:
-        n = con.execute(f"SELECT count(*) FROM _pv_out o WHERE NOT EXISTS (SELECT 1 FROM _pv_src s WHERE {on})").fetchone()[0]
+        n = con.execute(f"SELECT count(*) FROM _pv_out o WHERE NOT EXISTS (SELECT 1 FROM _pv_src s WHERE {on})").fetchall()[0][0]
         if n:
             pct = n / max(n_out, 1) * 100
             findings.append(Finding("source_orphan", REVIEW if pct > 0 else INFO,

{pipeverify-0.2.0 → pipeverify-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pipeverify"
-version = "0.2.0"
+version = "0.2.1"
 description = "Check that a pipeline change left the data right, not just the job green."
 readme = "README.md"
 license = "Apache-2.0"