pipeverify 0.2.1__tar.gz → 0.2.3__tar.gz

{pipeverify-0.2.1 → pipeverify-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipeverify
-Version: 0.2.1
+Version: 0.2.3
 Summary: Check that a pipeline change left the data right, not just the job green.
 Project-URL: Homepage, https://github.com/akshaysharma1088/pipeverify
 Project-URL: Issues, https://github.com/akshaysharma1088/pipeverify/issues
@@ -22,6 +22,7 @@ Provides-Extra: ai
 Requires-Dist: anthropic>=0.40; extra == 'ai'
 Requires-Dist: openai>=1.40; extra == 'ai'
 Provides-Extra: dev
+Requires-Dist: openpyxl; extra == 'dev'
 Requires-Dist: pandas>=2.0; extra == 'dev'
 Requires-Dist: pytest>=8; extra == 'dev'
 Provides-Extra: sql
@@ -29,16 +30,29 @@ Requires-Dist: pandas>=2.0; extra == 'sql'
 Requires-Dist: sqlalchemy>=2.0; extra == 'sql'
 Description-Content-Type: text/markdown
 
-# pipeverify
+<p align="center">
+  <img src="docs/assets/banner.svg" alt="pipeverify" width="720">
+</p>
 
-**Check that a pipeline change left the data right, not just the job green.**
+<h3 align="center">The open-source verification step for data pipeline changes.</h3>
+
+<p align="center">
+Check that a change left the data right, not just the job green. pipeverify compares tables
+before and after a change, and against the raw source, and returns <b>PASS</b>, <b>REVIEW</b> or <b>BLOCK</b>.
+Works with SQL, Spark, dbt, Airflow, or a fix written by an AI agent.
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/pipeverify/"><img src="https://img.shields.io/pypi/v/pipeverify?label=pypi&color=38bdf8" alt="PyPI"></a>
+  <a href="https://pepy.tech/projects/pipeverify"><img src="https://static.pepy.tech/personalized-badge/pipeverify?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-4ade80" alt="License"></a>
+  <a href="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml"><img src="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://doi.org/10.5281/zenodo.23542277"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg" alt="DOI"></a>
+  <a href="https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open in Colab"></a>
+</p>
+
+---
 
-[![ci](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg)](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml)
-[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
-![Python](https://img.shields.io/badge/python-3.9%2B-blue)
-![Status](https://img.shields.io/badge/status-alpha-orange)
-[![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -65,6 +79,8 @@ so any team can run it on every change.
 
 ## Quick start
 
+No install needed to try it: [open the 5-minute notebook in Colab](https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb).
+
 ```bash
 pip install pipeverify
 
@@ -107,7 +123,7 @@ Every one of these runs successfully and keeps the same schema.
 
 | Source | CLI form | Config form |
 |---|---|---|
-| CSV, Parquet, JSON file | `data/orders.parquet` | `path: data/orders.parquet` |
+| CSV, Parquet, JSON, Excel (.xlsx) file (CSV and JSON can be `.gz` or `.zst` compressed) | `data/orders.parquet` | `path: data/orders.parquet` |
 | DuckDB table | `duckdb:warehouse.db:gold.orders` | `{duckdb: warehouse.db, table: gold.orders}` |
 | DuckDB query | `"duckdb:warehouse.db:SELECT ..."` | `{duckdb: warehouse.db, query: "SELECT ..."}` |
 | Any SQLAlchemy database (Postgres, Snowflake, BigQuery, ...) | | `{url: "postgresql://${PGUSER}@host/db", table: gold.orders}` |
@@ -116,6 +132,8 @@ Every one of these runs successfully and keeps the same schema.
 SQL databases need `pip install "pipeverify[sql]"` plus the driver for your database. Environment
 variables in URLs are expanded, so credentials stay out of the config file.
 
+See [SQL source examples](docs/sources.md) for Postgres, Snowflake and BigQuery.
+
 ## Config
 
 Check many tables at once with `pipeverify.yml` (`pipeverify init` writes a starter file):
@@ -215,7 +233,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # ... build the "after" snapshots for this PR here ...
-      - uses: akshaysharma1088/pipeverify@v0.2.0
+      - uses: akshaysharma1088/pipeverify@v0.2.2
         with:
           config: pipeverify.yml
           fail-on: block      # or review, or never
@@ -250,6 +268,12 @@ This adds a short plain-English explanation of what probably happened and what t
 It is off by default. Only the findings are sent (check names, counts, percentages, column names);
 example values from your data are removed first. The verdict never depends on the LLM.
 
+## Who uses pipeverify
+
+See [ADOPTERS.md](ADOPTERS.md). Using it at work or in a project?
+[Tell us how](https://github.com/akshaysharma1088/pipeverify/issues/new?template=share_usage.yml), even
+in one line. It helps other teams decide whether to try it and shapes what gets built next.
+
 ## Roadmap
 
 - Native adapters for dbt (compare a PR's models against production), Airflow and Spark
@@ -257,7 +281,9 @@ example values from your data are removed first. The verdict never depends on th
 - Rules learned from history: what "normal" change looks like for each table
 - Agent integration: a guardrail that coding and data agents call before applying a change
 
-Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
+Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md) and the
+[good first issues](https://github.com/akshaysharma1088/pipeverify/labels/good%20first%20issue).
+Questions go in [Discussions](https://github.com/akshaysharma1088/pipeverify/discussions).
 
 ## Status
 

{pipeverify-0.2.1 → pipeverify-0.2.3}/README.md

@@ -1,13 +1,26 @@
-# pipeverify
+<p align="center">
+  <img src="docs/assets/banner.svg" alt="pipeverify" width="720">
+</p>
 
-**Check that a pipeline change left the data right, not just the job green.**
+<h3 align="center">The open-source verification step for data pipeline changes.</h3>
+
+<p align="center">
+Check that a change left the data right, not just the job green. pipeverify compares tables
+before and after a change, and against the raw source, and returns <b>PASS</b>, <b>REVIEW</b> or <b>BLOCK</b>.
+Works with SQL, Spark, dbt, Airflow, or a fix written by an AI agent.
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/pipeverify/"><img src="https://img.shields.io/pypi/v/pipeverify?label=pypi&color=38bdf8" alt="PyPI"></a>
+  <a href="https://pepy.tech/projects/pipeverify"><img src="https://static.pepy.tech/personalized-badge/pipeverify?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads" alt="PyPI Downloads"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-4ade80" alt="License"></a>
+  <a href="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml"><img src="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://doi.org/10.5281/zenodo.23542277"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg" alt="DOI"></a>
+  <a href="https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open in Colab"></a>
+</p>
+
+---
 
-[![ci](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg)](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml)
-[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
-![Python](https://img.shields.io/badge/python-3.9%2B-blue)
-![Status](https://img.shields.io/badge/status-alpha-orange)
-[![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
 
 A pipeline change can pass every test, finish on time and keep the schema intact while quietly
 dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -34,6 +47,8 @@ so any team can run it on every change.
 
 ## Quick start
 
+No install needed to try it: [open the 5-minute notebook in Colab](https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb).
+
 ```bash
 pip install pipeverify
 
@@ -76,7 +91,7 @@ Every one of these runs successfully and keeps the same schema.
 
 | Source | CLI form | Config form |
 |---|---|---|
-| CSV, Parquet, JSON file | `data/orders.parquet` | `path: data/orders.parquet` |
+| CSV, Parquet, JSON, Excel (.xlsx) file (CSV and JSON can be `.gz` or `.zst` compressed) | `data/orders.parquet` | `path: data/orders.parquet` |
 | DuckDB table | `duckdb:warehouse.db:gold.orders` | `{duckdb: warehouse.db, table: gold.orders}` |
 | DuckDB query | `"duckdb:warehouse.db:SELECT ..."` | `{duckdb: warehouse.db, query: "SELECT ..."}` |
 | Any SQLAlchemy database (Postgres, Snowflake, BigQuery, ...) | | `{url: "postgresql://${PGUSER}@host/db", table: gold.orders}` |
@@ -85,6 +100,8 @@ Every one of these runs successfully and keeps the same schema.
 SQL databases need `pip install "pipeverify[sql]"` plus the driver for your database. Environment
 variables in URLs are expanded, so credentials stay out of the config file.
 
+See [SQL source examples](docs/sources.md) for Postgres, Snowflake and BigQuery.
+
 ## Config
 
 Check many tables at once with `pipeverify.yml` (`pipeverify init` writes a starter file):
@@ -184,7 +201,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # ... build the "after" snapshots for this PR here ...
-      - uses: akshaysharma1088/pipeverify@v0.2.0
+      - uses: akshaysharma1088/pipeverify@v0.2.2
         with:
           config: pipeverify.yml
           fail-on: block      # or review, or never
@@ -219,6 +236,12 @@ This adds a short plain-English explanation of what probably happened and what t
 It is off by default. Only the findings are sent (check names, counts, percentages, column names);
 example values from your data are removed first. The verdict never depends on the LLM.
 
+## Who uses pipeverify
+
+See [ADOPTERS.md](ADOPTERS.md). Using it at work or in a project?
+[Tell us how](https://github.com/akshaysharma1088/pipeverify/issues/new?template=share_usage.yml), even
+in one line. It helps other teams decide whether to try it and shapes what gets built next.
+
 ## Roadmap
 
 - Native adapters for dbt (compare a PR's models against production), Airflow and Spark
@@ -226,7 +249,9 @@ example values from your data are removed first. The verdict never depends on th
 - Rules learned from history: what "normal" change looks like for each table
 - Agent integration: a guardrail that coding and data agents call before applying a change
 
-Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
+Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md) and the
+[good first issues](https://github.com/akshaysharma1088/pipeverify/labels/good%20first%20issue).
+Questions go in [Discussions](https://github.com/akshaysharma1088/pipeverify/discussions).
 
 ## Status
 

pipeverify-0.2.3/metrics/README.md

@@ -0,0 +1,14 @@
+# Usage metrics
+
+Updated 2026-11-09 by `scripts/collect_metrics.py`. Raw history is in the CSV files in this folder.
+
+| Metric | Value |
+|---|---|
+| GitHub stars | 1 |
+| Forks | 4 |
+| Watchers | 0 |
+| Repositories that depend on it | n/a |
+| Repository views since 2026-10-08 | 128 |
+| Repository clones since 2026-10-08 | 496 |
+| PyPI downloads since 2026-10-21 (without mirrors) | 450 |
+| PyPI downloads, last 30 days | 0 |

{pipeverify-0.2.1 → pipeverify-0.2.3}/pipeverify/__init__.py

@@ -1,6 +1,6 @@
 """pipeverify: check that a pipeline change left the data right, not just the job green."""
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"
 
 from .checks import Finding, Thresholds  # noqa: E402
 from .core import RunResult, TableResult, run_config, verify  # noqa: E402

{pipeverify-0.2.1 → pipeverify-0.2.3}/pipeverify/cli.py

@@ -28,19 +28,19 @@ tables:
 
 
 def _emit(result: RunResult, fmt: str, output: str | None, summary: str | None, show_info: bool,
-          json_out: str | None = None, markdown_out: str | None = None) -> None:
+          json_out: str | None = None, markdown_out: str | None = None, max_examples: int = 1) -> None:
     for path, render in ((json_out, to_json), (markdown_out, to_markdown)):
         if path:
             with open(path, "w", encoding="utf-8") as fh:
                 body = render(result)
                 fh.write(body if body.endswith("\n") else body + "\n")
-    text = {"text": lambda: to_text(result, show_info), "markdown": lambda: to_markdown(result),
+    text = {"text": lambda: to_text(result, show_info, max_examples), "markdown": lambda: to_markdown(result),
             "json": lambda: to_json(result)}[fmt]()
     if output:
         with open(output, "w", encoding="utf-8") as fh:
             fh.write(text if text.endswith("\n") else text + "\n")
         if fmt != "text":
-            click.echo(to_text(result, show_info))
+            click.echo(to_text(result, show_info, max_examples))
     else:
         click.echo(text)
     if summary:
@@ -79,6 +79,8 @@ output_options = [
     click.option("--show-info", is_flag=True, help="Also print informational findings."),
     click.option("--json-out", type=click.Path(dir_okay=False), help="Also write a JSON report here."),
     click.option("--markdown-out", type=click.Path(dir_okay=False), help="Also write a Markdown report here."),
+    click.option("--max-examples", type=int, default=1, show_default=True,
+                 help="Example rows to show per finding."),
 ]
 
 
@@ -110,7 +112,7 @@ def main() -> None:
 @with_output_options
 def compare(before, after, source_data, source_key, source_columns, source_where,
             key, expect_change, ignore, expect_row_change, name,
-            fmt, output, summary, fail_on, explain, show_info, json_out, markdown_out):
+            fmt, output, summary, fail_on, explain, show_info, json_out, markdown_out, max_examples):
     """Compare one table BEFORE and AFTER a change.
 
     Use - as BEFORE to run only the source checks.
@@ -135,7 +137,7 @@ def compare(before, after, source_data, source_key, source_columns, source_where
         raise click.ClickException(str(exc))
     result = RunResult(tables=[res], seconds=time.perf_counter() - start)
     _maybe_explain(result, explain)
-    _emit(result, fmt, output, summary, show_info, json_out, markdown_out)
+    _emit(result, fmt, output, summary, show_info, json_out, markdown_out, max_examples)
     _exit(result, fail_on)
 
 
@@ -144,7 +146,7 @@ def compare(before, after, source_data, source_key, source_columns, source_where
               type=click.Path(exists=True, dir_okay=False), help="Config file listing the tables.")
 @click.option("--table", "-t", multiple=True, help="Only check these table names. Repeatable.")
 @with_output_options
-def run(config, table, fmt, output, summary, fail_on, explain, show_info, json_out, markdown_out):
+def run(config, table, fmt, output, summary, fail_on, explain, show_info, json_out, markdown_out, max_examples):
     """Check every table listed in a config file."""
     try:
         result = run_config(config, only=list(table) or None)
@@ -153,7 +155,7 @@ def run(config, table, fmt, output, summary, fail_on, explain, show_info, json_o
     if not result.tables:
         raise click.ClickException("No tables matched")
     _maybe_explain(result, explain)
-    _emit(result, fmt, output, summary, show_info, json_out, markdown_out)
+    _emit(result, fmt, output, summary, show_info, json_out, markdown_out, max_examples)
     _exit(result, fail_on)
 
 

{pipeverify-0.2.1 → pipeverify-0.2.3}/pipeverify/report.py

@@ -29,7 +29,7 @@ def _color(text: str, sev: str) -> str:
     return f"\033[{code}m{text}\033[0m"
 
 
-def to_text(result: RunResult, show_info: bool = False) -> str:
+def to_text(result: RunResult, show_info: bool = False, max_examples: int = 1) -> str:
     lines = []
     for t in result.tables:
         lines.append(_color(f"{LABEL[t.verdict]:<7}", t.verdict) + f" {t.name}")
@@ -46,7 +46,8 @@ def to_text(result: RunResult, show_info: bool = False) -> str:
             lines.append("        " + _color(f"{LABEL[f.severity]:<6}", f.severity) + f" {f.message}")
             ex = f.metrics.get("examples") if f.metrics else None
             if ex:
-                lines.append(f"               e.g. {ex[0]}")
+                for row in ex[:max(0, max_examples)]:
+                    lines.append(f"               e.g. {row}")
     lines.append("")
     lines.append(_color(f"Verdict: {LABEL[result.verdict]}", result.verdict) + f"  {_headline(result)}")
     if result.explanation:

{pipeverify-0.2.1 → pipeverify-0.2.3}/pipeverify/sources.py

@@ -18,6 +18,7 @@ FILE_READERS = {
     ".json": "read_json_auto",
     ".jsonl": "read_json_auto",
     ".ndjson": "read_json_auto",
+    ".xlsx": "read_xlsx",
 }
 
 
@@ -96,6 +97,17 @@ def _resolve(path: str, base_dir: str) -> str:
     return path if os.path.isabs(path) else os.path.normpath(os.path.join(base_dir, path))
 
 
+COMPRESSION_SUFFIXES = (".gz", ".zst")
+
+
+def _split_extension(path: str) -> tuple[str, str]:
+    """Return (file extension, compression suffix), e.g. 'orders.csv.gz' -> ('.csv', '.gz')."""
+    base, ext = os.path.splitext(path.lower())
+    if ext in COMPRESSION_SUFFIXES:
+        return os.path.splitext(base)[1], ext
+    return ext, ""
+
+
 def load(con: duckdb.DuckDBPyConnection, source: Source, name: str) -> None:
     """Materialize a source as table `name` in the DuckDB connection."""
     if source.kind == "dataframe":
@@ -107,10 +119,24 @@ def load(con: duckdb.DuckDBPyConnection, source: Source, name: str) -> None:
         path = source.path or ""
         if not os.path.exists(path):
             raise SourceError(f"File not found: {path}")
-        ext = os.path.splitext(path)[1].lower()
+        ext, compression = _split_extension(path)
         reader = FILE_READERS.get(ext)
         if reader is None:
-            raise SourceError(f"Unsupported file type '{ext}'. Use csv, parquet or json.")
+            raise SourceError(
+                f"Unsupported file type '{ext}{compression}'. "
+                "Use csv, parquet or json, optionally compressed with .gz or .zst.")
+        if compression and reader == "read_parquet":
+            raise SourceError(
+                f"Compressed Parquet ('{ext}{compression}') is not supported. Parquet is already "
+                f"compressed internally, so use the plain {ext} file.")
+        if ext == ".xlsx":
+            try:
+                con.execute("INSTALL excel")
+                con.execute("LOAD excel")
+            except duckdb.Error as e:
+                raise SourceError(
+                    f"Excel extension could not be installed or loaded: {e} "
+                ) from e
         con.execute(f'CREATE OR REPLACE TABLE "{name}" AS SELECT * FROM {reader}(?)', [path])
         return
     if source.kind == "duckdb":

{pipeverify-0.2.1 → pipeverify-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pipeverify"
-version = "0.2.1"
+version = "0.2.3"
 description = "Check that a pipeline change left the data right, not just the job green."
 readme = "README.md"
 license = "Apache-2.0"
@@ -23,7 +23,7 @@ dependencies = ["duckdb>=1.0", "pyyaml>=6.0", "click>=8.1"]
 [project.optional-dependencies]
 sql = ["sqlalchemy>=2.0", "pandas>=2.0"]
 ai = ["anthropic>=0.40", "openai>=1.40"]
-dev = ["pytest>=8", "pandas>=2.0"]
+dev = ["pytest>=8", "pandas>=2.0", "openpyxl"]
 
 [project.scripts]
 pipeverify = "pipeverify.cli:main"