pipeverify 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. {pipeverify-0.2.1 → pipeverify-0.2.2}/PKG-INFO +36 -12
  2. {pipeverify-0.2.1 → pipeverify-0.2.2}/README.md +35 -11
  3. pipeverify-0.2.2/metrics/README.md +14 -0
  4. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/__init__.py +1 -1
  5. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/sources.py +19 -2
  6. {pipeverify-0.2.1 → pipeverify-0.2.2}/pyproject.toml +1 -1
  7. {pipeverify-0.2.1 → pipeverify-0.2.2}/.gitignore +0 -0
  8. {pipeverify-0.2.1 → pipeverify-0.2.2}/LICENSE +0 -0
  9. {pipeverify-0.2.1 → pipeverify-0.2.2}/NOTICE +0 -0
  10. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/checks.py +0 -0
  11. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/cli.py +0 -0
  12. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/core.py +0 -0
  13. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/explain.py +0 -0
  14. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/report.py +0 -0
  15. {pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/source_checks.py +0 -0
{pipeverify-0.2.1 → pipeverify-0.2.2}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pipeverify
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Check that a pipeline change left the data right, not just the job green.
5
5
  Project-URL: Homepage, https://github.com/akshaysharma1088/pipeverify
6
6
  Project-URL: Issues, https://github.com/akshaysharma1088/pipeverify/issues
@@ -29,16 +29,28 @@ Requires-Dist: pandas>=2.0; extra == 'sql'
29
29
  Requires-Dist: sqlalchemy>=2.0; extra == 'sql'
30
30
  Description-Content-Type: text/markdown
31
31
 
32
- # pipeverify
32
+ <p align="center">
33
+ <img src="docs/assets/banner.svg" alt="pipeverify" width="720">
34
+ </p>
33
35
 
34
- **Check that a pipeline change left the data right, not just the job green.**
36
+ <h3 align="center">The open-source verification step for data pipeline changes.</h3>
37
+
38
+ <p align="center">
39
+ Check that a change left the data right, not just the job green. pipeverify compares tables
40
+ before and after a change, and against the raw source, and returns <b>PASS</b>, <b>REVIEW</b> or <b>BLOCK</b>.
41
+ Works with SQL, Spark, dbt, Airflow, or a fix written by an AI agent.
42
+ </p>
43
+
44
+ <p align="center">
45
+ <a href="https://pypi.org/project/pipeverify/"><img src="https://img.shields.io/pypi/v/pipeverify?label=pypi&color=38bdf8" alt="PyPI"></a>
46
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-4ade80" alt="License"></a>
47
+ <a href="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml"><img src="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
48
+ <a href="https://doi.org/10.5281/zenodo.23542277"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg" alt="DOI"></a>
49
+ <a href="https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open in Colab"></a>
50
+ </p>
51
+
52
+ ---
35
53
 
36
- [![ci](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg)](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml)
37
- [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
38
- ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
39
- ![Status](https://img.shields.io/badge/status-alpha-orange)
40
- [![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
41
- [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
42
54
 
43
55
  A pipeline change can pass every test, finish on time and keep the schema intact while quietly
44
56
  dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -65,6 +77,8 @@ so any team can run it on every change.
65
77
 
66
78
  ## Quick start
67
79
 
80
+ No install needed to try it: [open the 5-minute notebook in Colab](https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb).
81
+
68
82
  ```bash
69
83
  pip install pipeverify
70
84
 
@@ -107,7 +121,7 @@ Every one of these runs successfully and keeps the same schema.
107
121
 
108
122
  | Source | CLI form | Config form |
109
123
  |---|---|---|
110
- | CSV, Parquet, JSON file | `data/orders.parquet` | `path: data/orders.parquet` |
124
+ | CSV, Parquet, JSON file (CSV and JSON can be `.gz` or `.zst` compressed) | `data/orders.parquet` | `path: data/orders.parquet` |
111
125
  | DuckDB table | `duckdb:warehouse.db:gold.orders` | `{duckdb: warehouse.db, table: gold.orders}` |
112
126
  | DuckDB query | `"duckdb:warehouse.db:SELECT ..."` | `{duckdb: warehouse.db, query: "SELECT ..."}` |
113
127
  | Any SQLAlchemy database (Postgres, Snowflake, BigQuery, ...) | | `{url: "postgresql://${PGUSER}@host/db", table: gold.orders}` |
@@ -116,6 +130,8 @@ Every one of these runs successfully and keeps the same schema.
116
130
  SQL databases need `pip install "pipeverify[sql]"` plus the driver for your database. Environment
117
131
  variables in URLs are expanded, so credentials stay out of the config file.
118
132
 
133
+ See [SQL source examples](docs/sources.md) for Postgres, Snowflake and BigQuery.
134
+
119
135
  ## Config
120
136
 
121
137
  Check many tables at once with `pipeverify.yml` (`pipeverify init` writes a starter file):
@@ -215,7 +231,7 @@ jobs:
215
231
  steps:
216
232
  - uses: actions/checkout@v4
217
233
  # ... build the "after" snapshots for this PR here ...
218
- - uses: akshaysharma1088/pipeverify@v0.2.0
234
+ - uses: akshaysharma1088/pipeverify@v0.2.2
219
235
  with:
220
236
  config: pipeverify.yml
221
237
  fail-on: block # or review, or never
@@ -250,6 +266,12 @@ This adds a short plain-English explanation of what probably happened and what t
250
266
  It is off by default. Only the findings are sent (check names, counts, percentages, column names);
251
267
  example values from your data are removed first. The verdict never depends on the LLM.
252
268
 
269
+ ## Who uses pipeverify
270
+
271
+ See [ADOPTERS.md](ADOPTERS.md). Using it at work or in a project?
272
+ [Tell us how](https://github.com/akshaysharma1088/pipeverify/issues/new?template=share_usage.yml), even
273
+ in one line. It helps other teams decide whether to try it and shapes what gets built next.
274
+
253
275
  ## Roadmap
254
276
 
255
277
  - Native adapters for dbt (compare a PR's models against production), Airflow and Spark
@@ -257,7 +279,9 @@ example values from your data are removed first. The verdict never depends on th
257
279
  - Rules learned from history: what "normal" change looks like for each table
258
280
  - Agent integration: a guardrail that coding and data agents call before applying a change
259
281
 
260
- Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
282
+ Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md) and the
283
+ [good first issues](https://github.com/akshaysharma1088/pipeverify/labels/good%20first%20issue).
284
+ Questions go in [Discussions](https://github.com/akshaysharma1088/pipeverify/discussions).
261
285
 
262
286
  ## Status
263
287
 
{pipeverify-0.2.1 → pipeverify-0.2.2}/README.md RENAMED
@@ -1,13 +1,25 @@
1
- # pipeverify
1
+ <p align="center">
2
+ <img src="docs/assets/banner.svg" alt="pipeverify" width="720">
3
+ </p>
2
4
 
3
- **Check that a pipeline change left the data right, not just the job green.**
5
+ <h3 align="center">The open-source verification step for data pipeline changes.</h3>
6
+
7
+ <p align="center">
8
+ Check that a change left the data right, not just the job green. pipeverify compares tables
9
+ before and after a change, and against the raw source, and returns <b>PASS</b>, <b>REVIEW</b> or <b>BLOCK</b>.
10
+ Works with SQL, Spark, dbt, Airflow, or a fix written by an AI agent.
11
+ </p>
12
+
13
+ <p align="center">
14
+ <a href="https://pypi.org/project/pipeverify/"><img src="https://img.shields.io/pypi/v/pipeverify?label=pypi&color=38bdf8" alt="PyPI"></a>
15
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-4ade80" alt="License"></a>
16
+ <a href="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml"><img src="https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
17
+ <a href="https://doi.org/10.5281/zenodo.23542277"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg" alt="DOI"></a>
18
+ <a href="https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open in Colab"></a>
19
+ </p>
20
+
21
+ ---
4
22
 
5
- [![ci](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml/badge.svg)](https://github.com/akshaysharma1088/pipeverify/actions/workflows/ci.yml)
6
- [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
7
- ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
8
- ![Status](https://img.shields.io/badge/status-alpha-orange)
9
- [![PyPI](https://img.shields.io/pypi/v/pipeverify)](https://pypi.org/project/pipeverify/)
10
- [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23542277.svg)](https://doi.org/10.5281/zenodo.23542277)
11
23
 
12
24
  A pipeline change can pass every test, finish on time and keep the schema intact while quietly
13
25
  dropping rows, duplicating keys or changing what a column means. More of these changes are now
@@ -34,6 +46,8 @@ so any team can run it on every change.
34
46
 
35
47
  ## Quick start
36
48
 
49
+ No install needed to try it: [open the 5-minute notebook in Colab](https://colab.research.google.com/github/akshaysharma1088/pipeverify/blob/main/examples/notebooks/quickstart.ipynb).
50
+
37
51
  ```bash
38
52
  pip install pipeverify
39
53
 
@@ -76,7 +90,7 @@ Every one of these runs successfully and keeps the same schema.
76
90
 
77
91
  | Source | CLI form | Config form |
78
92
  |---|---|---|
79
- | CSV, Parquet, JSON file | `data/orders.parquet` | `path: data/orders.parquet` |
93
+ | CSV, Parquet, JSON file (CSV and JSON can be `.gz` or `.zst` compressed) | `data/orders.parquet` | `path: data/orders.parquet` |
80
94
  | DuckDB table | `duckdb:warehouse.db:gold.orders` | `{duckdb: warehouse.db, table: gold.orders}` |
81
95
  | DuckDB query | `"duckdb:warehouse.db:SELECT ..."` | `{duckdb: warehouse.db, query: "SELECT ..."}` |
82
96
  | Any SQLAlchemy database (Postgres, Snowflake, BigQuery, ...) | | `{url: "postgresql://${PGUSER}@host/db", table: gold.orders}` |
@@ -85,6 +99,8 @@ Every one of these runs successfully and keeps the same schema.
85
99
  SQL databases need `pip install "pipeverify[sql]"` plus the driver for your database. Environment
86
100
  variables in URLs are expanded, so credentials stay out of the config file.
87
101
 
102
+ See [SQL source examples](docs/sources.md) for Postgres, Snowflake and BigQuery.
103
+
88
104
  ## Config
89
105
 
90
106
  Check many tables at once with `pipeverify.yml` (`pipeverify init` writes a starter file):
@@ -184,7 +200,7 @@ jobs:
184
200
  steps:
185
201
  - uses: actions/checkout@v4
186
202
  # ... build the "after" snapshots for this PR here ...
187
- - uses: akshaysharma1088/pipeverify@v0.2.0
203
+ - uses: akshaysharma1088/pipeverify@v0.2.2
188
204
  with:
189
205
  config: pipeverify.yml
190
206
  fail-on: block # or review, or never
@@ -219,6 +235,12 @@ This adds a short plain-English explanation of what probably happened and what t
219
235
  It is off by default. Only the findings are sent (check names, counts, percentages, column names);
220
236
  example values from your data are removed first. The verdict never depends on the LLM.
221
237
 
238
+ ## Who uses pipeverify
239
+
240
+ See [ADOPTERS.md](ADOPTERS.md). Using it at work or in a project?
241
+ [Tell us how](https://github.com/akshaysharma1088/pipeverify/issues/new?template=share_usage.yml), even
242
+ in one line. It helps other teams decide whether to try it and shapes what gets built next.
243
+
222
244
  ## Roadmap
223
245
 
224
246
  - Native adapters for dbt (compare a PR's models against production), Airflow and Spark
@@ -226,7 +248,9 @@ example values from your data are removed first. The verdict never depends on th
226
248
  - Rules learned from history: what "normal" change looks like for each table
227
249
  - Agent integration: a guardrail that coding and data agents call before applying a change
228
250
 
229
- Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
251
+ Ideas and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md) and the
252
+ [good first issues](https://github.com/akshaysharma1088/pipeverify/labels/good%20first%20issue).
253
+ Questions go in [Discussions](https://github.com/akshaysharma1088/pipeverify/discussions).
230
254
 
231
255
  ## Status
232
256
 
pipeverify-0.2.2/metrics/README.md ADDED
@@ -0,0 +1,14 @@
1
+ # Usage metrics
2
+
3
+ Updated 2026-10-22 by `scripts/collect_metrics.py`. Raw history is in the CSV files in this folder.
4
+
5
+ | Metric | Value |
6
+ |---|---|
7
+ | GitHub stars | 1 |
8
+ | Forks | 0 |
9
+ | Watchers | 0 |
10
+ | Repositories that depend on it | n/a |
11
+ | Repository views since 2026-10-08 | 0 |
12
+ | Repository clones since 2026-10-08 | 162 |
13
+ | PyPI downloads since 2026-10-21 (without mirrors) | 228 |
14
+ | PyPI downloads, last 30 days | 0 |
{pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/__init__.py RENAMED
@@ -1,6 +1,6 @@
1
1
  """pipeverify: check that a pipeline change left the data right, not just the job green."""
2
2
 
3
- __version__ = "0.2.1"
3
+ __version__ = "0.2.2"
4
4
 
5
5
  from .checks import Finding, Thresholds # noqa: E402
6
6
  from .core import RunResult, TableResult, run_config, verify # noqa: E402
{pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/sources.py RENAMED
@@ -96,6 +96,17 @@ def _resolve(path: str, base_dir: str) -> str:
96
96
  return path if os.path.isabs(path) else os.path.normpath(os.path.join(base_dir, path))
97
97
 
98
98
 
99
+ COMPRESSION_SUFFIXES = (".gz", ".zst")
100
+
101
+
102
+ def _split_extension(path: str) -> tuple[str, str]:
103
+ """Return (file extension, compression suffix), e.g. 'orders.csv.gz' -> ('.csv', '.gz')."""
104
+ base, ext = os.path.splitext(path.lower())
105
+ if ext in COMPRESSION_SUFFIXES:
106
+ return os.path.splitext(base)[1], ext
107
+ return ext, ""
108
+
109
+
99
110
  def load(con: duckdb.DuckDBPyConnection, source: Source, name: str) -> None:
100
111
  """Materialize a source as table `name` in the DuckDB connection."""
101
112
  if source.kind == "dataframe":
@@ -107,10 +118,16 @@ def load(con: duckdb.DuckDBPyConnection, source: Source, name: str) -> None:
107
118
  path = source.path or ""
108
119
  if not os.path.exists(path):
109
120
  raise SourceError(f"File not found: {path}")
110
- ext = os.path.splitext(path)[1].lower()
121
+ ext, compression = _split_extension(path)
111
122
  reader = FILE_READERS.get(ext)
112
123
  if reader is None:
113
- raise SourceError(f"Unsupported file type '{ext}'. Use csv, parquet or json.")
124
+ raise SourceError(
125
+ f"Unsupported file type '{ext}{compression}'. "
126
+ "Use csv, parquet or json, optionally compressed with .gz or .zst.")
127
+ if compression and reader == "read_parquet":
128
+ raise SourceError(
129
+ f"Compressed Parquet ('{ext}{compression}') is not supported. Parquet is already "
130
+ f"compressed internally, so use the plain {ext} file.")
114
131
  con.execute(f'CREATE OR REPLACE TABLE "{name}" AS SELECT * FROM {reader}(?)', [path])
115
132
  return
116
133
  if source.kind == "duckdb":
{pipeverify-0.2.1 → pipeverify-0.2.2}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "pipeverify"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Check that a pipeline change left the data right, not just the job green."
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
{pipeverify-0.2.1 → pipeverify-0.2.2}/.gitignore RENAMED
File without changes
{pipeverify-0.2.1 → pipeverify-0.2.2}/LICENSE RENAMED
File without changes
{pipeverify-0.2.1 → pipeverify-0.2.2}/NOTICE RENAMED
File without changes
{pipeverify-0.2.1 → pipeverify-0.2.2}/pipeverify/cli.py RENAMED
File without changes