rowgate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowgate-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +23 -0
- rowgate-0.1.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
- rowgate-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +15 -0
- rowgate-0.1.0/.github/pull_request_template.md +8 -0
- rowgate-0.1.0/.github/workflows/ci.yml +51 -0
- rowgate-0.1.0/.github/workflows/publish-pypi.yml +20 -0
- rowgate-0.1.0/.github/workflows/release.yml +22 -0
- rowgate-0.1.0/.gitignore +8 -0
- rowgate-0.1.0/CHANGELOG.md +13 -0
- rowgate-0.1.0/CITATION.cff +22 -0
- rowgate-0.1.0/CODE_OF_CONDUCT.md +7 -0
- rowgate-0.1.0/CONTRIBUTING.md +19 -0
- rowgate-0.1.0/LICENSE +21 -0
- rowgate-0.1.0/PKG-INFO +305 -0
- rowgate-0.1.0/README.md +271 -0
- rowgate-0.1.0/SECURITY.md +11 -0
- rowgate-0.1.0/docs/assets/social-preview.png +0 -0
- rowgate-0.1.0/examples/permits_contract.toml +48 -0
- rowgate-0.1.0/examples/seattle_permits.py +49 -0
- rowgate-0.1.0/examples/seattle_permits_sample.ndjson +454 -0
- rowgate-0.1.0/pyproject.toml +56 -0
- rowgate-0.1.0/src/rowgate/__init__.py +9 -0
- rowgate-0.1.0/src/rowgate/__main__.py +3 -0
- rowgate-0.1.0/src/rowgate/checks.py +123 -0
- rowgate-0.1.0/src/rowgate/cli.py +65 -0
- rowgate-0.1.0/src/rowgate/contract.py +185 -0
- rowgate-0.1.0/src/rowgate/gate.py +121 -0
- rowgate-0.1.0/src/rowgate/py.typed +0 -0
- rowgate-0.1.0/tests/test_cli.py +33 -0
- rowgate-0.1.0/tests/test_contract.py +129 -0
- rowgate-0.1.0/tests/test_gate.py +85 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Something in rowgate does not work as documented
|
|
3
|
+
labels: [bug]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: what
|
|
7
|
+
attributes:
|
|
8
|
+
label: What happened?
|
|
9
|
+
description: A short description and what you expected instead.
|
|
10
|
+
validations: {required: true}
|
|
11
|
+
- type: textarea
|
|
12
|
+
id: repro
|
|
13
|
+
attributes:
|
|
14
|
+
label: Minimal example
|
|
15
|
+
description: A small DataFrame and contract that reproduce the problem.
|
|
16
|
+
render: python
|
|
17
|
+
validations: {required: true}
|
|
18
|
+
- type: input
|
|
19
|
+
id: versions
|
|
20
|
+
attributes:
|
|
21
|
+
label: Versions
|
|
22
|
+
placeholder: "rowgate 0.1.0, polars 1.x, Python 3.12, OS"
|
|
23
|
+
validations: {required: true}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Suggest a new check, contract option or report format
|
|
3
|
+
labels: [enhancement]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: problem
|
|
7
|
+
attributes:
|
|
8
|
+
label: What problem are you solving?
|
|
9
|
+
description: Describe the data situation, not just the API you want.
|
|
10
|
+
validations: {required: true}
|
|
11
|
+
- type: textarea
|
|
12
|
+
id: idea
|
|
13
|
+
attributes:
|
|
14
|
+
label: Proposed API or behavior
|
|
15
|
+
render: python
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
name: py${{ matrix.python }} · polars ${{ matrix.polars }}
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
python: ["3.11", "3.12", "3.13"]
|
|
16
|
+
polars: ["latest"]
|
|
17
|
+
include:
|
|
18
|
+
- python: "3.11"
|
|
19
|
+
polars: "1.0.0"
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v5
|
|
22
|
+
- uses: actions/setup-python@v6
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python }}
|
|
25
|
+
- name: Install
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip
|
|
28
|
+
pip install -e ".[dev]"
|
|
29
|
+
if [ "${{ matrix.polars }}" != "latest" ]; then pip install "polars==${{ matrix.polars }}"; fi
|
|
30
|
+
- run: ruff check .
|
|
31
|
+
- run: pytest -q
|
|
32
|
+
- name: Example on real Seattle permit data
|
|
33
|
+
run: python examples/seattle_permits.py
|
|
34
|
+
- name: CLI smoke test
|
|
35
|
+
run: |
|
|
36
|
+
python -c "import polars as pl; pl.DataFrame({'id':[1,2,2],'amount':[1.0,-2.0,3.0]}).write_parquet('/tmp/bad.parquet')"
|
|
37
|
+
printf '[columns.id]\ndtype="int"\nrequired=true\nunique=true\n[columns.amount]\ndtype="float"\nmin=0\n' > /tmp/c.toml
|
|
38
|
+
if rowgate check /tmp/bad.parquet --contract /tmp/c.toml; then echo "expected failure" && exit 1; fi
|
|
39
|
+
|
|
40
|
+
build:
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v5
|
|
44
|
+
- uses: actions/setup-python@v6
|
|
45
|
+
with:
|
|
46
|
+
python-version: "3.12"
|
|
47
|
+
- run: pip install build && python -m build
|
|
48
|
+
- uses: actions/upload-artifact@v4
|
|
49
|
+
with:
|
|
50
|
+
name: dist
|
|
51
|
+
path: dist/
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: publish-pypi
|
|
2
|
+
|
|
3
|
+
# Manual run only. Before the first run, add a trusted publisher on PyPI:
|
|
4
|
+
# owner DTheDataGuy, repository rowgate, workflow publish-pypi.yml, environment pypi.
|
|
5
|
+
on:
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
environment: pypi
|
|
12
|
+
permissions:
|
|
13
|
+
id-token: write
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v5
|
|
16
|
+
- uses: actions/setup-python@v6
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
- run: pip install build && python -m build
|
|
20
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
dist:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v5
|
|
15
|
+
- uses: actions/setup-python@v6
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- run: pip install build && python -m build
|
|
19
|
+
- name: Attach wheel and sdist to the release
|
|
20
|
+
env:
|
|
21
|
+
GH_TOKEN: ${{ github.token }}
|
|
22
|
+
run: gh release upload "${{ github.event.release.tag_name }}" dist/* --clobber
|
rowgate-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-10-23)
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- `Contract` with per-column dtype, required, unique, accepted values, min/max and regex checks, plus row-level rules written as Polars expressions or SQL.
|
|
8
|
+
- `Contract.split()` separates valid rows from a quarantine frame. Each rejected row lists every rule it broke, and nothing is dropped.
|
|
9
|
+
- `Gate` with dataset-level checks: schema drift, uniqueness, nulls, null rate, accepted values, row count, reconciliation, quarantine rate, freshness and custom expressions. Checks are `error` (blocking) or `warn`.
|
|
10
|
+
- Reports as JSON, Markdown and plain text, plus `raise_for_errors()` for pipelines.
|
|
11
|
+
- Contracts can be loaded from TOML.
|
|
12
|
+
- `rowgate check` CLI for Parquet, CSV, NDJSON and JSON files, with exit codes for CI.
|
|
13
|
+
- Tested on Python 3.11 to 3.13 and Polars 1.0 through 2.0.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use rowgate, please cite it with this metadata."
|
|
3
|
+
title: "rowgate: data contracts and quality gates for Polars"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Scisney
|
|
7
|
+
given-names: "Dei'Marlon"
|
|
8
|
+
email: info@thedataguys.org
|
|
9
|
+
affiliation: "The Data Guys"
|
|
10
|
+
version: 0.1.0
|
|
11
|
+
date-released: 2026-10-23
|
|
12
|
+
license: MIT
|
|
13
|
+
repository-code: "https://github.com/DTheDataGuy/rowgate"
|
|
14
|
+
url: "https://github.com/DTheDataGuy/rowgate"
|
|
15
|
+
abstract: "rowgate splits Polars DataFrames into valid and quarantined rows with a reason for every broken rule, then gates the batch with dataset-level checks and writes JSON, Markdown and text reports."
|
|
16
|
+
keywords:
|
|
17
|
+
- polars
|
|
18
|
+
- data quality
|
|
19
|
+
- data contracts
|
|
20
|
+
- data validation
|
|
21
|
+
- data engineering
|
|
22
|
+
- etl
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Code of conduct
|
|
2
|
+
|
|
3
|
+
rowgate follows the [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/).
|
|
4
|
+
|
|
5
|
+
Be respectful, assume good intent, and keep feedback focused on the work. Harassment or discrimination of any kind is not tolerated.
|
|
6
|
+
|
|
7
|
+
Report concerns privately to **info@thedataguys.org**. Reports are handled confidentially.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Bug reports and pull requests are welcome.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
git clone https://github.com/DTheDataGuy/rowgate && cd rowgate
|
|
7
|
+
python -m venv .venv && source .venv/bin/activate
|
|
8
|
+
pip install -e ".[dev]"
|
|
9
|
+
pytest -q && ruff check .
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Before you open a pull request:
|
|
13
|
+
|
|
14
|
+
- Add a test that fails without your change.
|
|
15
|
+
- Keep the core dependency list at `polars` only.
|
|
16
|
+
- New checks need a clear `expectation` string and an `observed` value a person can act on, such as "3 duplicate keys" rather than "False".
|
|
17
|
+
- Add a line to `CHANGELOG.md`.
|
|
18
|
+
|
|
19
|
+
By participating you agree to the [code of conduct](CODE_OF_CONDUCT.md). Report security issues privately, as described in [SECURITY.md](SECURITY.md).
|
rowgate-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dei'Marlon Scisney, The Data Guys
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rowgate-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rowgate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Data contracts and quality gates for Polars: quarantine bad rows with reasons, block bad batches, and write reports people can read.
|
|
5
|
+
Project-URL: Homepage, https://github.com/DTheDataGuy/rowgate
|
|
6
|
+
Project-URL: Documentation, https://github.com/DTheDataGuy/rowgate#readme
|
|
7
|
+
Project-URL: Source, https://github.com/DTheDataGuy/rowgate
|
|
8
|
+
Project-URL: Issues, https://github.com/DTheDataGuy/rowgate/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/DTheDataGuy/rowgate/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: The Data Guys, https://www.thedataguys.org
|
|
11
|
+
Author-email: Dei'Marlon Scisney <info@thedataguys.org>
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: data-contracts,data-engineering,data-pipelines,data-quality,data-testing,data-validation,dataframe,etl,polars,quality-gate,quarantine,schema,validation
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
25
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.11
|
|
28
|
+
Requires-Dist: polars>=1.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pyarrow; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
<div align="center">
|
|
36
|
+
|
|
37
|
+
<img src="docs/assets/social-preview.png" alt="rowgate: data contracts and quality gates for Polars" width="100%">
|
|
38
|
+
|
|
39
|
+
# rowgate
|
|
40
|
+
|
|
41
|
+
**Data contracts and quality gates for [Polars](https://pola.rs). Quarantine bad rows with reasons, block bad batches, and write reports people can read.**
|
|
42
|
+
|
|
43
|
+
[](https://github.com/DTheDataGuy/rowgate/actions/workflows/ci.yml)
|
|
44
|
+
[](https://github.com/DTheDataGuy/rowgate/releases)
|
|
45
|
+

|
|
46
|
+

|
|
47
|
+

|
|
48
|
+
[](LICENSE)
|
|
49
|
+
[](https://github.com/DTheDataGuy/rowgate/stargazers)
|
|
50
|
+
|
|
51
|
+
[Install](#install) · [Quick start](#quick-start) · [Why rowgate](#why-rowgate) · [Real data](#on-real-data) · [Contracts](#contracts) · [Checks](#checks) · [CI and orchestrators](#use-it-in-ci-and-orchestrators) · [CLI](#command-line) · [Contributing](CONTRIBUTING.md) · [Cite](CITATION.cff)
|
|
52
|
+
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
rowgate does two jobs that most pipelines need and usually hand-roll:
|
|
56
|
+
|
|
57
|
+
1. **Split** a batch into rows that meet the contract and rows that don't. Rejected rows go to a quarantine frame that lists every rule each row broke. Nothing is silently dropped, and `valid + quarantined = input` always holds.
|
|
58
|
+
2. **Gate** the batch with dataset-level checks (schema drift, uniqueness, null rates, freshness, reconciliation). `error` checks block the load, `warn` checks are reported. The result is a report you can log as JSON, attach to a run as Markdown or print to a terminal.
|
|
59
|
+
|
|
60
|
+
The only dependency is Polars.
|
|
61
|
+
|
|
62
|
+
## Why rowgate
|
|
63
|
+
|
|
64
|
+
Most validation tools answer one question: did this DataFrame pass? Production pipelines need three more answers. Which rows failed, why did each one fail, and is it safe to load the rest? rowgate is built around those questions.
|
|
65
|
+
|
|
66
|
+
| You need to | rowgate gives you |
|
|
67
|
+
|---|---|
|
|
68
|
+
| Keep loading good rows when a few are bad | `contract.split(df)` returns `valid` and `quarantine` frames, and proves they add up to the input |
|
|
69
|
+
| Fix source data in one pass | Every broken rule is listed per row, not just the first one |
|
|
70
|
+
| Stop a bad batch before it reaches the warehouse | `Gate(...).run(df).raise_for_errors()` or a CLI exit code of `1` |
|
|
71
|
+
| Tell on-call what happened | Readable observed values such as "3 duplicate keys" or "9.62% null" |
|
|
72
|
+
| Keep rules next to the pipeline | Contracts in Python or TOML, with row rules in SQL or Polars expressions |
|
|
73
|
+
| Stay light | One dependency (Polars), typed, single-pass expression evaluation |
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Quick start
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
import polars as pl
|
|
85
|
+
import rowgate as rg
|
|
86
|
+
from rowgate import checks as ck
|
|
87
|
+
|
|
88
|
+
orders = pl.DataFrame({
|
|
89
|
+
"order_id": [1001, 1002, 1002, 1004, 1005],
|
|
90
|
+
"customer": ["acme", "globex", "globex", None, "initech"],
|
|
91
|
+
"amount": [250.0, 99.5, 99.5, -20.0, 1_200.0],
|
|
92
|
+
"currency": ["USD", "USD", "USD", "USD", "usd"],
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
contract = rg.Contract(columns={
|
|
96
|
+
"order_id": rg.Column(pl.Int64, required=True, unique=True),
|
|
97
|
+
"customer": rg.Column(pl.Utf8, required=True),
|
|
98
|
+
"amount": rg.Column(pl.Float64, min=0),
|
|
99
|
+
"currency": rg.Column(pl.Utf8, accepted=["USD", "EUR", "GBP"]),
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
result = contract.split(orders)
|
|
103
|
+
result.quarantine
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
```text
|
|
107
|
+
┌──────────┬──────────┬────────┬──────────┬───────────────────────────────────────┐
|
|
108
|
+
│ order_id ┆ customer ┆ amount ┆ currency ┆ _rowgate_reasons │
|
|
109
|
+
╞══════════╪══════════╪════════╪══════════╪═══════════════════════════════════════╡
|
|
110
|
+
│ 1002 ┆ globex ┆ 99.5 ┆ USD ┆ ["order_id:duplicate"] │
|
|
111
|
+
│ 1004 ┆ null ┆ -20.0 ┆ USD ┆ ["customer:required", "amount:min"] │
|
|
112
|
+
│ 1005 ┆ initech ┆ 1200.0 ┆ usd ┆ ["currency:accepted"] │
|
|
113
|
+
└──────────┴──────────┴────────┴──────────┴───────────────────────────────────────┘
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The first `1002` is kept and the repeat is quarantined. Row `1004` broke two rules, and both are listed.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
report = rg.Gate([
|
|
120
|
+
ck.schema_matches(contract),
|
|
121
|
+
ck.reconciles(result),
|
|
122
|
+
ck.quarantine_rate(result, max_pct=10),
|
|
123
|
+
ck.row_count(min_rows=1),
|
|
124
|
+
]).run(result.valid)
|
|
125
|
+
|
|
126
|
+
print(report)
|
|
127
|
+
report.raise_for_errors() # raises rowgate.GateFailed in a pipeline
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
```text
|
|
131
|
+
rowgate: FAIL (2 rows)
|
|
132
|
+
ok schema_matches no drift
|
|
133
|
+
ok rows_reconcile 2 valid + 3 quarantined = 5 vs 5 input
|
|
134
|
+
FAIL quarantine_rate 60.0%
|
|
135
|
+
ok row_count 2 rows
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## On real data
|
|
139
|
+
|
|
140
|
+
[`examples/seattle_permits.py`](examples/seattle_permits.py) runs a contract over a 454-row sample of the City of Seattle's public [Building Permits](https://data.seattle.gov/Built-Environment/Building-Permits/76t5-zqzr) dataset:
|
|
141
|
+
|
|
142
|
+
```text
|
|
143
|
+
rowgate: PASS (447 rows)
|
|
144
|
+
ok schema_matches no drift
|
|
145
|
+
ok rows_reconcile 447 valid + 7 quarantined = 454 vs 454 input
|
|
146
|
+
ok quarantine_rate 1.54%
|
|
147
|
+
ok unique:permit_num 0 duplicate keys
|
|
148
|
+
ok not_null:permit_num+permit_class 0 nulls
|
|
149
|
+
ok null_rate:applied_date 9.62% null
|
|
150
|
+
ok null_rate:zip 4.7% null
|
|
151
|
+
warn freshness:applied_date newest 2026-07-02 (110 days old)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```text
|
|
155
|
+
reason rows
|
|
156
|
+
issued_on_or_after_applied 3
|
|
157
|
+
permit_num:pattern 3
|
|
158
|
+
est_cost:max 1
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The 7 quarantined rows are real problems in the source data:
|
|
162
|
+
|
|
163
|
+
- 3 permits were issued before they were applied for.
|
|
164
|
+
- 3 sub-permit numbers like `6112685-EX-002` don't match the documented format.
|
|
165
|
+
- 1 permit lists an estimated cost of $25,000,000,000.
|
|
166
|
+
|
|
167
|
+
If sub-permits are valid for your use, widen the pattern in [`permits_contract.toml`](examples/permits_contract.toml). The freshness warning is expected, because the sample is a fixed extract.
|
|
168
|
+
|
|
169
|
+
The same approach runs at full scale in the [Seattle Permits Data Platform](https://github.com/DTheDataGuy/the-data-guys-portfolio/tree/main/data-engineering): 193,747 rows through a Polars pipeline, dbt, PySpark and Scala Spark.
|
|
170
|
+
|
|
171
|
+
## Contracts
|
|
172
|
+
|
|
173
|
+
### Columns
|
|
174
|
+
|
|
175
|
+
| Option | Meaning | Nulls |
|
|
176
|
+
|---|---|---|
|
|
177
|
+
| `dtype` | Expected Polars dtype, checked by `schema_matches` | n/a |
|
|
178
|
+
| `required` | Value must be present. Blank strings count as missing | fail |
|
|
179
|
+
| `unique` | First occurrence kept; later duplicates quarantined | skipped |
|
|
180
|
+
| `accepted` | Value must be in the list | skipped |
|
|
181
|
+
| `min`, `max` | Inclusive bounds | skipped |
|
|
182
|
+
| `pattern` | Regex the value must match | skipped |
|
|
183
|
+
|
|
184
|
+
Nulls only fail `required`, the same way SQL constraints behave. Set `allow_extra_columns=False` to treat unexpected columns as schema drift.
|
|
185
|
+
|
|
186
|
+
### Rules
|
|
187
|
+
|
|
188
|
+
Row-level rules are Polars expressions that are true for valid rows:
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
rg.Rule("issued_on_or_after_applied", pl.col("issued_date") >= pl.col("applied_date"))
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
A rule that evaluates to null passes, so a permit that hasn't been issued yet isn't rejected by an ordering rule.
|
|
195
|
+
|
|
196
|
+
### TOML
|
|
197
|
+
|
|
198
|
+
Contracts can live next to the pipeline as config, with rules written in SQL:
|
|
199
|
+
|
|
200
|
+
```toml
|
|
201
|
+
[columns.permit_num]
|
|
202
|
+
dtype = "str"
|
|
203
|
+
required = true
|
|
204
|
+
unique = true
|
|
205
|
+
pattern = '^\d{6,7}-[A-Z]{2}$'
|
|
206
|
+
|
|
207
|
+
[columns.est_cost]
|
|
208
|
+
dtype = "float"
|
|
209
|
+
min = 0
|
|
210
|
+
max = 5_000_000_000
|
|
211
|
+
|
|
212
|
+
[[rules]]
|
|
213
|
+
name = "issued_on_or_after_applied"
|
|
214
|
+
sql = "issued_date >= applied_date"
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
contract = rg.Contract.from_toml("permits_contract.toml")
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Checks
|
|
222
|
+
|
|
223
|
+
| Check | Default severity | Passes when |
|
|
224
|
+
|---|---|---|
|
|
225
|
+
| `schema_matches(contract)` | error | Every contract column is present with the right dtype |
|
|
226
|
+
| `unique(cols)` | error | No duplicate keys (one column or a composite) |
|
|
227
|
+
| `not_null(cols)` | error | No nulls in the listed columns |
|
|
228
|
+
| `null_rate(col, max_pct)` | warn | Null share is at or below the limit |
|
|
229
|
+
| `accepted_values(col, values)` | error | No values outside the set |
|
|
230
|
+
| `row_count(min_rows, max_rows)` | error | Row count is within bounds |
|
|
231
|
+
| `reconciles(result)` | error | Valid + quarantined equals input |
|
|
232
|
+
| `quarantine_rate(result, max_pct)` | error | Quarantined share is below the limit |
|
|
233
|
+
| `freshness(col, max_age_days, as_of)` | warn | Newest date is recent enough |
|
|
234
|
+
| `expr(name, condition, max_failing)` | error | At most `max_failing` rows break the condition |
|
|
235
|
+
| `custom(name, fn, expectation)` | error | `fn(df)` returns `(True, observed)` |
|
|
236
|
+
|
|
237
|
+
Every check takes `severity="error"` or `severity="warn"`. If a check itself raises, for example because a column is missing, it's recorded as a failure with the exception message, and the other checks still run.
|
|
238
|
+
|
|
239
|
+
## Reports
|
|
240
|
+
|
|
241
|
+
```python
|
|
242
|
+
report.passed # False if any error check failed
|
|
243
|
+
report.errors # failed error checks
|
|
244
|
+
report.warnings # failed warn checks
|
|
245
|
+
report.to_json("quality_report.json")
|
|
246
|
+
report.to_markdown("quality_report.md")
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
The Markdown output is a table that renders in GitHub, GitLab and most CI summaries:
|
|
250
|
+
|
|
251
|
+
| Check | Severity | Result | Observed | Expectation |
|
|
252
|
+
|---|---|---|---|---|
|
|
253
|
+
| `rows_reconcile` | error | pass | 2 valid + 3 quarantined = 5 vs 5 input | valid + quarantined = input |
|
|
254
|
+
| `quarantine_rate` | error | **FAIL** | 60.0% | < 10% of input quarantined |
|
|
255
|
+
|
|
256
|
+
## Use it in CI and orchestrators
|
|
257
|
+
|
|
258
|
+
GitHub Actions, failing the job and posting the report to the run summary:
|
|
259
|
+
|
|
260
|
+
```yaml
|
|
261
|
+
- run: pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
|
|
262
|
+
- run: rowgate check data/orders.parquet --contract contracts/orders.toml --report-dir out/
|
|
263
|
+
- if: always()
|
|
264
|
+
run: cat out/report.md >> "$GITHUB_STEP_SUMMARY"
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
Inside an Airflow, Dagster or Prefect task, or any Python job:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
result = contract.split(df)
|
|
271
|
+
result.quarantine.write_parquet("quarantine/orders.parquet") # keep the evidence
|
|
272
|
+
rg.Gate([ck.reconciles(result), ck.quarantine_rate(result, max_pct=2)]).run(result.valid).raise_for_errors()
|
|
273
|
+
result.valid.write_parquet("silver/orders.parquet") # only reached when the gate passes
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
## Command line
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
rowgate check orders.parquet --contract contract.toml --report-dir out/
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
The command reads Parquet, CSV, NDJSON or JSON files. It prints the report and a count of quarantine reasons. With `--report-dir` it also writes `report.json`, `report.md` and `quarantine.parquet`. It exits with `1` when a blocking check fails, so it can stop a CI job or an orchestrator task. Add `--strict` to fail on warnings too.
|
|
283
|
+
|
|
284
|
+
## Design notes
|
|
285
|
+
|
|
286
|
+
- **Quarantine, don't drop.** A pipeline that deletes bad rows makes its own totals impossible to audit. rowgate keeps every row and records why it was set aside.
|
|
287
|
+
- **Every reason is listed.** A row that breaks three rules lists all three, so one fix doesn't reveal the next problem a day later.
|
|
288
|
+
- **The observed value is readable.** "3 duplicate keys" or "9.62% null" tells the on-call person what to do. A bare `False` doesn't.
|
|
289
|
+
- **Lazy-friendly expressions.** Column checks compile to Polars expressions and evaluate in a single `with_columns` pass.
|
|
290
|
+
|
|
291
|
+
## Development
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
pip install -e ".[dev]"
|
|
295
|
+
pytest -q
|
|
296
|
+
ruff check .
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
CI runs the test suite, the Seattle example and a CLI smoke test on Python 3.11, 3.12 and 3.13. It also runs the suite against Polars 1.0.0, the oldest supported version.
|
|
300
|
+
|
|
301
|
+
## License
|
|
302
|
+
|
|
303
|
+
MIT © Dei'Marlon Scisney · [The Data Guys™](https://www.thedataguys.org)
|
|
304
|
+
|
|
305
|
+
If rowgate saves you a hand-rolled validation script, a star helps other data engineers find it.
|