rowgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rowgate/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """rowgate: data contracts and quality gates for Polars."""
2
+ from . import checks
3
+ from .checks import Check
4
+ from .contract import REASONS, Column, Contract, Rule, SplitResult, columns
5
+ from .gate import CheckResult, Gate, GateFailed, Report
6
+
7
+ __all__ = ["Check", "CheckResult", "Column", "Contract", "Gate", "GateFailed", "REASONS", "Report",
8
+ "Rule", "SplitResult", "checks", "columns"]
9
+ __version__ = "0.1.0"
rowgate/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
rowgate/checks.py ADDED
@@ -0,0 +1,123 @@
1
+ """Dataset-level checks for a :class:`rowgate.Gate`.
2
+
3
+ Each factory returns a :class:`Check`: a name, a severity and a function that looks at
4
+ a DataFrame and returns ``(passed, observed)``. ``error`` checks block the batch,
5
+ ``warn`` checks are reported but don't block it.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Callable, Sequence
10
+ from dataclasses import dataclass
11
+ from datetime import date, datetime
12
+
13
+ import polars as pl
14
+
15
+ from .contract import Contract, SplitResult
16
+
17
+ Severity = str # "error" | "warn"
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class Check:
22
+ name: str
23
+ severity: Severity
24
+ expectation: str
25
+ fn: Callable[[pl.DataFrame], tuple[bool, str]]
26
+
27
+ def __post_init__(self) -> None:
28
+ if self.severity not in ("error", "warn"):
29
+ raise ValueError(f"severity must be 'error' or 'warn', got {self.severity!r}")
30
+
31
+
32
+ def _pct(n: int, d: int) -> float:
33
+ return round(100.0 * n / d, 2) if d else 0.0
34
+
35
+
36
+ def schema_matches(contract: Contract, severity: Severity = "error") -> Check:
37
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
38
+ drift = contract.schema_drift(df)
39
+ if not drift:
40
+ return True, "no drift"
41
+ return False, "; ".join(f"{k}: expected {e}, got {a}" for k, (e, a) in sorted(drift.items()))
42
+ return Check("schema_matches", severity, "columns and dtypes match the contract", fn)
43
+
44
+
45
+ def unique(cols: str | Sequence[str], severity: Severity = "error") -> Check:
46
+ keys = [cols] if isinstance(cols, str) else list(cols)
47
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
48
+ dup = df.height - df.select(keys).unique().height
49
+ return dup == 0, f"{dup:,} duplicate keys"
50
+ return Check(f"unique:{'+'.join(keys)}", severity, "0 duplicate keys", fn)
51
+
52
+
53
+ def not_null(cols: str | Sequence[str], severity: Severity = "error") -> Check:
54
+ names = [cols] if isinstance(cols, str) else list(cols)
55
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
56
+ bad = {c: n for c in names if (n := df[c].null_count())}
57
+ return not bad, ", ".join(f"{c}: {n:,}" for c, n in bad.items()) or "0 nulls"
58
+ return Check(f"not_null:{'+'.join(names)}", severity, f"no nulls in {len(names)} column(s)", fn)
59
+
60
+
61
+ def null_rate(col: str, max_pct: float, severity: Severity = "warn") -> Check:
62
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
63
+ p = _pct(df[col].null_count(), df.height)
64
+ return p <= max_pct, f"{p}% null"
65
+ return Check(f"null_rate:{col}", severity, f"<= {max_pct}% null", fn)
66
+
67
+
68
+ def accepted_values(col: str, values: Sequence[object], severity: Severity = "error") -> Check:
69
+ allowed = set(values)
70
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
71
+ seen = set(df[col].drop_nulls().unique().to_list())
72
+ extra = sorted(map(str, seen - allowed))
73
+ return not extra, "unexpected: " + ", ".join(extra) if extra else "all values accepted"
74
+ return Check(f"accepted_values:{col}", severity, f"subset of {len(allowed)} accepted values", fn)
75
+
76
+
77
+ def row_count(min_rows: int = 1, max_rows: int | None = None, severity: Severity = "error") -> Check:
78
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
79
+ ok = df.height >= min_rows and (max_rows is None or df.height <= max_rows)
80
+ return ok, f"{df.height:,} rows"
81
+ bound = f">= {min_rows:,}" + (f" and <= {max_rows:,}" if max_rows is not None else "")
82
+ return Check("row_count", severity, f"{bound} rows", fn)
83
+
84
+
85
+ def reconciles(result: SplitResult, severity: Severity = "error") -> Check:
86
+ """Every input row is either valid or quarantined."""
87
+ def fn(_: pl.DataFrame) -> tuple[bool, str]:
88
+ total = result.valid.height + result.quarantine.height
89
+ return total == result.input_rows, (
90
+ f"{result.valid.height:,} valid + {result.quarantine.height:,} quarantined = {total:,} "
91
+ f"vs {result.input_rows:,} input")
92
+ return Check("rows_reconcile", severity, "valid + quarantined = input", fn)
93
+
94
+
95
+ def quarantine_rate(result: SplitResult, max_pct: float, severity: Severity = "error") -> Check:
96
+ def fn(_: pl.DataFrame) -> tuple[bool, str]:
97
+ return result.quarantine_rate < max_pct, f"{round(result.quarantine_rate, 2)}%"
98
+ return Check("quarantine_rate", severity, f"< {max_pct}% of input quarantined", fn)
99
+
100
+
101
+ def freshness(col: str, max_age_days: int, as_of: date | None = None, severity: Severity = "warn") -> Check:
102
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
103
+ newest = df[col].max()
104
+ if newest is None:
105
+ return False, "no values"
106
+ ref = as_of or date.today()
107
+ newest_d = newest.date() if isinstance(newest, datetime) else newest
108
+ age = (ref - newest_d).days
109
+ return age <= max_age_days, f"newest {newest_d} ({age} days old)"
110
+ return Check(f"freshness:{col}", severity, f"<= {max_age_days} days old", fn)
111
+
112
+
113
+ def expr(name: str, condition: pl.Expr, severity: Severity = "error", max_failing: int = 0) -> Check:
114
+ """Custom row condition that must hold for (almost) every row."""
115
+ def fn(df: pl.DataFrame) -> tuple[bool, str]:
116
+ failing = df.select(condition.not_().fill_null(False).sum()).item()
117
+ return failing <= max_failing, f"{failing:,} rows fail"
118
+ return Check(name, severity, f"<= {max_failing:,} failing rows", fn)
119
+
120
+
121
+ def custom(name: str, fn: Callable[[pl.DataFrame], tuple[bool, str]], expectation: str,
122
+ severity: Severity = "error") -> Check:
123
+ return Check(name, severity, expectation, fn)
rowgate/cli.py ADDED
@@ -0,0 +1,65 @@
1
+ """Command line: ``rowgate check data.parquet --contract contract.toml``.
2
+
3
+ Exit codes: 0 pass, 1 blocking failure (or any failure with ``--strict``), 2 usage error.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import argparse
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ import polars as pl
12
+
13
+ from . import checks as ck
14
+ from .contract import Contract
15
+ from .gate import Gate
16
+
17
+
18
+ def _read(path: Path) -> pl.DataFrame:
19
+ suffix = path.suffix.lower()
20
+ if suffix == ".parquet":
21
+ return pl.read_parquet(path)
22
+ if suffix == ".csv":
23
+ return pl.read_csv(path, try_parse_dates=True)
24
+ if suffix in (".ndjson", ".jsonl"):
25
+ return pl.read_ndjson(path)
26
+ if suffix == ".json":
27
+ return pl.read_json(path)
28
+ raise SystemExit(f"rowgate: unsupported file type {suffix!r} (use parquet, csv, ndjson or json)")
29
+
30
+
31
+ def main(argv: list[str] | None = None) -> int:
32
+ p = argparse.ArgumentParser(prog="rowgate", description="Data contracts and quality gates for Polars.")
33
+ sub = p.add_subparsers(dest="cmd", required=True)
34
+ c = sub.add_parser("check", help="validate a file against a contract")
35
+ c.add_argument("data", type=Path)
36
+ c.add_argument("--contract", type=Path, required=True)
37
+ c.add_argument("--max-quarantine-pct", type=float, default=1.0)
38
+ c.add_argument("--report-dir", type=Path, help="write report.json, report.md and quarantine.parquet here")
39
+ c.add_argument("--strict", action="store_true", help="treat warnings as failures")
40
+ a = p.parse_args(argv)
41
+
42
+ contract = Contract.from_toml(a.contract)
43
+ df = _read(a.data)
44
+ result = contract.split(df)
45
+ gate_checks = [ck.schema_matches(contract), ck.reconciles(result),
46
+ ck.quarantine_rate(result, a.max_quarantine_pct)]
47
+ gate_checks += [ck.unique(n) for n, col in contract.columns.items() if col.unique and n in df.columns]
48
+ report = Gate(gate_checks).run(result.valid, source=str(a.data), contract=str(a.contract))
49
+
50
+ print(report)
51
+ if not result.quarantine.is_empty():
52
+ print("\nquarantine reasons:")
53
+ for reason, rows in result.reason_counts().iter_rows():
54
+ print(f" {rows:>8,} {reason}")
55
+ if a.report_dir:
56
+ a.report_dir.mkdir(parents=True, exist_ok=True)
57
+ report.to_json(a.report_dir / "report.json")
58
+ report.to_markdown(a.report_dir / "report.md")
59
+ result.quarantine.write_parquet(a.report_dir / "quarantine.parquet")
60
+ failed = not report.passed or (a.strict and report.warnings)
61
+ return 1 if failed else 0
62
+
63
+
64
+ if __name__ == "__main__":
65
+ sys.exit(main())
rowgate/contract.py ADDED
@@ -0,0 +1,185 @@
1
+ """Column contracts and row-level rules.
2
+
3
+ A :class:`Contract` describes what a valid row looks like. :meth:`Contract.split`
4
+ evaluates it against a DataFrame and returns the rows that pass, plus a quarantine
5
+ frame where every rejected row carries the names of the rules it broke.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ import tomllib
11
+ from collections.abc import Iterable, Mapping, Sequence
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ import polars as pl
17
+
18
+ REASONS = "_rowgate_reasons"
19
+
20
+ _DTYPES: dict[str, Any] = {
21
+ "str": pl.Utf8, "string": pl.Utf8, "utf8": pl.Utf8,
22
+ "int": pl.Int64, "int64": pl.Int64, "int32": pl.Int32,
23
+ "float": pl.Float64, "float64": pl.Float64, "float32": pl.Float32,
24
+ "bool": pl.Boolean, "boolean": pl.Boolean,
25
+ "date": pl.Date, "datetime": pl.Datetime, "decimal": pl.Decimal,
26
+ }
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Column:
31
+ """Expectations for a single column.
32
+
33
+ ``None`` values only fail ``required``. Range, pattern and accepted-value checks
34
+ skip nulls, the same way SQL constraints do.
35
+ """
36
+
37
+ dtype: Any = None
38
+ required: bool = False
39
+ unique: bool = False
40
+ accepted: Sequence[Any] | None = None
41
+ min: Any = None
42
+ max: Any = None
43
+ pattern: str | None = None
44
+
45
+ def __post_init__(self) -> None:
46
+ if self.pattern is not None:
47
+ re.compile(self.pattern) # fail early on a bad regex
48
+ if self.min is not None and self.max is not None and self.min > self.max:
49
+ raise ValueError(f"min ({self.min}) is greater than max ({self.max})")
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class Rule:
54
+ """A row-level rule. ``check`` must evaluate to True for valid rows.
55
+
56
+ A null result counts as a pass, so a rule like ``issued >= applied`` does not
57
+ reject rows that simply haven't been issued yet.
58
+ """
59
+
60
+ name: str
61
+ check: pl.Expr
62
+ description: str = ""
63
+
64
+
65
+ @dataclass
66
+ class SplitResult:
67
+ valid: pl.DataFrame
68
+ quarantine: pl.DataFrame
69
+ input_rows: int
70
+
71
+ @property
72
+ def quarantine_rate(self) -> float:
73
+ return 100.0 * self.quarantine.height / self.input_rows if self.input_rows else 0.0
74
+
75
+ def reason_counts(self) -> pl.DataFrame:
76
+ """How many quarantined rows broke each rule, most common first."""
77
+ if self.quarantine.is_empty():
78
+ return pl.DataFrame({"reason": [], "rows": []}, schema={"reason": pl.Utf8, "rows": pl.UInt32})
79
+ return (self.quarantine.select(pl.col(REASONS).explode().alias("reason"))
80
+ .group_by("reason").len(name="rows").sort(["rows", "reason"], descending=[True, False]))
81
+
82
+
83
+ @dataclass
84
+ class Contract:
85
+ columns: Mapping[str, Column]
86
+ rules: Sequence[Rule] = field(default_factory=list)
87
+ allow_extra_columns: bool = True
88
+
89
+ # ------------------------------------------------------------------ loading
90
+ @classmethod
91
+ def from_toml(cls, path: str | Path) -> Contract:
92
+ """Load a contract from TOML. See ``examples/permits_contract.toml``."""
93
+ raw = tomllib.loads(Path(path).read_text())
94
+ return cls.from_dict(raw)
95
+
96
+ @classmethod
97
+ def from_dict(cls, raw: Mapping[str, Any]) -> Contract:
98
+ cols: dict[str, Column] = {}
99
+ for name, spec in raw.get("columns", {}).items():
100
+ spec = dict(spec)
101
+ dt = spec.pop("dtype", None)
102
+ if dt is not None:
103
+ key = str(dt).lower()
104
+ if key not in _DTYPES:
105
+ raise ValueError(f"column {name!r}: unknown dtype {dt!r}; use one of {sorted(_DTYPES)}")
106
+ spec["dtype"] = _DTYPES[key]
107
+ cols[name] = Column(**spec)
108
+ rules = [Rule(r["name"], pl.sql_expr(r["sql"]), r.get("description", ""))
109
+ for r in raw.get("rules", [])]
110
+ return cls(cols, rules, raw.get("allow_extra_columns", True))
111
+
112
+ # ---------------------------------------------------------------- inspection
113
+ def schema_drift(self, df: pl.DataFrame) -> dict[str, tuple[str, str]]:
114
+ """Columns whose presence or dtype differs from the contract: ``{name: (expected, actual)}``."""
115
+ actual = dict(df.schema)
116
+ drift: dict[str, tuple[str, str]] = {}
117
+ for name, col in self.columns.items():
118
+ if name not in actual:
119
+ drift[name] = (str(col.dtype or "any"), "missing")
120
+ elif col.dtype is not None and not _dtype_matches(actual[name], col.dtype):
121
+ drift[name] = (str(col.dtype), str(actual[name]))
122
+ if not self.allow_extra_columns:
123
+ for name in actual.keys() - self.columns.keys():
124
+ drift[name] = ("absent", str(actual[name]))
125
+ return drift
126
+
127
+ def failure_expressions(self, df: pl.DataFrame) -> list[tuple[str, pl.Expr]]:
128
+ """``(reason, expr)`` pairs where ``expr`` is True for rows that fail."""
129
+ out: list[tuple[str, pl.Expr]] = []
130
+ present = set(df.columns)
131
+ for name, col in self.columns.items():
132
+ if name not in present:
133
+ continue
134
+ c = pl.col(name)
135
+ if col.required:
136
+ fail = c.is_null()
137
+ if df.schema[name] == pl.Utf8:
138
+ fail = fail | (c.str.strip_chars() == "")
139
+ out.append((f"{name}:required", fail))
140
+ if col.accepted is not None:
141
+ out.append((f"{name}:accepted", c.is_not_null() & ~c.is_in(list(col.accepted))))
142
+ if col.min is not None:
143
+ out.append((f"{name}:min", c < col.min))
144
+ if col.max is not None:
145
+ out.append((f"{name}:max", c > col.max))
146
+ if col.pattern is not None:
147
+ out.append((f"{name}:pattern", c.is_not_null() & ~c.cast(pl.Utf8).str.contains(col.pattern)))
148
+ if col.unique:
149
+ # keep the first occurrence, quarantine later ones
150
+ out.append((f"{name}:duplicate", c.is_not_null() & c.is_duplicated() & ~c.is_first_distinct()))
151
+ for rule in self.rules:
152
+ out.append((rule.name, rule.check.not_().fill_null(False)))
153
+ return out
154
+
155
+ # -------------------------------------------------------------------- split
156
+ def split(self, df: pl.DataFrame) -> SplitResult:
157
+ """Separate valid rows from rows that break the contract.
158
+
159
+ Nothing is dropped: ``valid.height + quarantine.height == df.height``.
160
+ """
161
+ missing = [n for n, c in self.columns.items() if c.required and n not in df.columns]
162
+ if missing:
163
+ raise KeyError(f"required columns missing from input: {missing}")
164
+ fails = self.failure_expressions(df)
165
+ if not fails:
166
+ return SplitResult(df, df.clear().with_columns(pl.lit([], pl.List(pl.Utf8)).alias(REASONS)), df.height)
167
+ reasons = pl.concat_list(
168
+ [pl.when(expr.fill_null(False)).then(pl.lit(name)).otherwise(pl.lit(None, pl.Utf8)) for name, expr in fails]
169
+ ).list.drop_nulls().alias(REASONS)
170
+ tagged = df.with_columns(reasons)
171
+ bad = tagged[REASONS].list.len() > 0
172
+ return SplitResult(valid=tagged.filter(~bad).drop(REASONS), quarantine=tagged.filter(bad), input_rows=df.height)
173
+
174
+
175
+ def _dtype_matches(actual: Any, expected: Any) -> bool:
176
+ if actual == expected:
177
+ return True
178
+ # Datetime/Decimal declared without parameters match any time unit, zone or precision.
179
+ base = getattr(expected, "base_type", lambda: expected)()
180
+ return actual.base_type() == base if hasattr(actual, "base_type") else False
181
+
182
+
183
+ def columns(names: Iterable[str], **kw: Any) -> dict[str, Column]:
184
+ """Shorthand for several columns with the same expectations."""
185
+ return {n: Column(**kw) for n in names}
rowgate/gate.py ADDED
@@ -0,0 +1,121 @@
1
+ """Run a list of checks and produce a report that machines and people can both read."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import time
6
+ from collections.abc import Sequence
7
+ from dataclasses import asdict, dataclass, field
8
+ from datetime import UTC, datetime
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import polars as pl
13
+
14
+ from .checks import Check
15
+
16
+
17
+ class GateFailed(RuntimeError):
18
+ def __init__(self, report: Report) -> None:
19
+ self.report = report
20
+ names = ", ".join(r.name for r in report.errors)
21
+ super().__init__(f"{len(report.errors)} blocking check(s) failed: {names}")
22
+
23
+
24
+ @dataclass
25
+ class CheckResult:
26
+ name: str
27
+ severity: str
28
+ passed: bool
29
+ observed: str
30
+ expectation: str
31
+ seconds: float
32
+
33
+
34
+ @dataclass
35
+ class Report:
36
+ results: list[CheckResult]
37
+ rows: int
38
+ created_at: str
39
+ meta: dict[str, Any] = field(default_factory=dict)
40
+
41
+ @property
42
+ def errors(self) -> list[CheckResult]:
43
+ return [r for r in self.results if r.severity == "error" and not r.passed]
44
+
45
+ @property
46
+ def warnings(self) -> list[CheckResult]:
47
+ return [r for r in self.results if r.severity == "warn" and not r.passed]
48
+
49
+ @property
50
+ def passed(self) -> bool:
51
+ return not self.errors
52
+
53
+ def raise_for_errors(self) -> Report:
54
+ if self.errors:
55
+ raise GateFailed(self)
56
+ return self
57
+
58
+ def to_dict(self) -> dict[str, Any]:
59
+ return {"passed": self.passed, "rows": self.rows, "created_at": self.created_at,
60
+ "meta": self.meta, "errors": len(self.errors), "warnings": len(self.warnings),
61
+ "results": [asdict(r) for r in self.results]}
62
+
63
+ def to_json(self, path: str | Path | None = None) -> str:
64
+ text = json.dumps(self.to_dict(), indent=2, default=str)
65
+ if path is not None:
66
+ Path(path).write_text(text + "\n")
67
+ return text
68
+
69
+ def to_markdown(self, path: str | Path | None = None, title: str = "Data-quality report") -> str:
70
+ status = "PASS" if self.passed else "FAIL"
71
+ lines = [f"# {title}", "",
72
+ f"**{status}** · {_rows(self.rows)} · {len(self.results)} checks · "
73
+ f"{len(self.errors)} blocking failures · {len(self.warnings)} warnings · {self.created_at}", "",
74
+ "| Check | Severity | Result | Observed | Expectation |", "|---|---|---|---|---|"]
75
+ for r in self.results:
76
+ res = "pass" if r.passed else ("**FAIL**" if r.severity == "error" else "warn")
77
+ lines.append(f"| `{r.name}` | {r.severity} | {res} | {_md(r.observed)} | {_md(r.expectation)} |")
78
+ text = "\n".join(lines) + "\n"
79
+ if path is not None:
80
+ Path(path).write_text(text)
81
+ return text
82
+
83
+ def __str__(self) -> str:
84
+ width = max((len(r.name) for r in self.results), default=10)
85
+ out = [f"rowgate: {'PASS' if self.passed else 'FAIL'} ({_rows(self.rows)})"]
86
+ for r in self.results:
87
+ mark = "ok " if r.passed else ("FAIL" if r.severity == "error" else "warn")
88
+ out.append(f" {mark} {r.name:<{width}} {r.observed}")
89
+ return "\n".join(out)
90
+
91
+
92
+ def _rows(n: int) -> str:
93
+ return f"{n:,} row" + ("" if n == 1 else "s")
94
+
95
+
96
+ def _md(s: str) -> str:
97
+ return s.replace("|", "\\|").replace("\n", " ")
98
+
99
+
100
+ class Gate:
101
+ """An ordered set of checks. A check that raises is recorded as a failure, not a crash."""
102
+
103
+ def __init__(self, checks: Sequence[Check]) -> None:
104
+ names = [c.name for c in checks]
105
+ dupes = {n for n in names if names.count(n) > 1}
106
+ if dupes:
107
+ raise ValueError(f"duplicate check names: {sorted(dupes)}")
108
+ self.checks = list(checks)
109
+
110
+ def run(self, df: pl.DataFrame, **meta: Any) -> Report:
111
+ results = []
112
+ for c in self.checks:
113
+ t0 = time.perf_counter()
114
+ try:
115
+ passed, observed = c.fn(df)
116
+ except Exception as exc: # noqa: BLE001 - a broken check must not hide the others
117
+ passed, observed = False, f"check raised {type(exc).__name__}: {exc}"
118
+ results.append(CheckResult(c.name, c.severity, bool(passed), observed, c.expectation,
119
+ round(time.perf_counter() - t0, 4)))
120
+ created = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
121
+ return Report(results, df.height, created, meta)
rowgate/py.typed ADDED
File without changes
@@ -0,0 +1,305 @@
1
+ Metadata-Version: 2.5
2
+ Name: rowgate
3
+ Version: 0.1.0
4
+ Summary: Data contracts and quality gates for Polars: quarantine bad rows with reasons, block bad batches, and write reports people can read.
5
+ Project-URL: Homepage, https://github.com/DTheDataGuy/rowgate
6
+ Project-URL: Documentation, https://github.com/DTheDataGuy/rowgate#readme
7
+ Project-URL: Source, https://github.com/DTheDataGuy/rowgate
8
+ Project-URL: Issues, https://github.com/DTheDataGuy/rowgate/issues
9
+ Project-URL: Changelog, https://github.com/DTheDataGuy/rowgate/blob/main/CHANGELOG.md
10
+ Project-URL: The Data Guys, https://www.thedataguys.org
11
+ Author-email: Dei'Marlon Scisney <info@thedataguys.org>
12
+ License-Expression: MIT
13
+ License-File: LICENSE
14
+ Keywords: data-contracts,data-engineering,data-pipelines,data-quality,data-testing,data-validation,dataframe,etl,polars,quality-gate,quarantine,schema,validation
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Database
24
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
25
+ Classifier: Topic :: Software Development :: Quality Assurance
26
+ Classifier: Typing :: Typed
27
+ Requires-Python: >=3.11
28
+ Requires-Dist: polars>=1.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pyarrow; extra == 'dev'
31
+ Requires-Dist: pytest>=8; extra == 'dev'
32
+ Requires-Dist: ruff>=0.6; extra == 'dev'
33
+ Description-Content-Type: text/markdown
34
+
35
+ <div align="center">
36
+
37
+ <img src="docs/assets/social-preview.png" alt="rowgate: data contracts and quality gates for Polars" width="100%">
38
+
39
+ # rowgate
40
+
41
+ **Data contracts and quality gates for [Polars](https://pola.rs). Quarantine bad rows with reasons, block bad batches, and write reports people can read.**
42
+
43
+ [![ci](https://github.com/DTheDataGuy/rowgate/actions/workflows/ci.yml/badge.svg)](https://github.com/DTheDataGuy/rowgate/actions/workflows/ci.yml)
44
+ [![release](https://img.shields.io/github/v/release/DTheDataGuy/rowgate?color=3FB8AF)](https://github.com/DTheDataGuy/rowgate/releases)
45
+ ![python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-3776AB)
46
+ ![polars](https://img.shields.io/badge/polars-1.0%20to%202.0-CD792C)
47
+ ![dependencies](https://img.shields.io/badge/dependencies-polars%20only-success)
48
+ [![license](https://img.shields.io/badge/license-MIT-green)](LICENSE)
49
+ [![stars](https://img.shields.io/github/stars/DTheDataGuy/rowgate?style=social)](https://github.com/DTheDataGuy/rowgate/stargazers)
50
+
51
+ [Install](#install) · [Quick start](#quick-start) · [Why rowgate](#why-rowgate) · [Real data](#on-real-data) · [Contracts](#contracts) · [Checks](#checks) · [CI and orchestrators](#use-it-in-ci-and-orchestrators) · [CLI](#command-line) · [Contributing](CONTRIBUTING.md) · [Cite](CITATION.cff)
52
+
53
+ </div>
54
+
55
+ rowgate does two jobs that most pipelines need and usually hand-roll:
56
+
57
+ 1. **Split** a batch into rows that meet the contract and rows that don't. Rejected rows go to a quarantine frame that lists every rule each row broke. Nothing is silently dropped, and `valid + quarantined = input` always holds.
58
+ 2. **Gate** the batch with dataset-level checks (schema drift, uniqueness, null rates, freshness, reconciliation). `error` checks block the load, `warn` checks are reported. The result is a report you can log as JSON, attach to a run as Markdown or print to a terminal.
59
+
60
+ The only dependency is Polars.
61
+
62
+ ## Why rowgate
63
+
64
+ Most validation tools answer one question: did this DataFrame pass? Production pipelines need three more answers. Which rows failed, why did each one fail, and is it safe to load the rest? rowgate is built around those questions.
65
+
66
+ | You need to | rowgate gives you |
67
+ |---|---|
68
+ | Keep loading good rows when a few are bad | `contract.split(df)` returns `valid` and `quarantine` frames, and proves they add up to the input |
69
+ | Fix source data in one pass | Every broken rule is listed per row, not just the first one |
70
+ | Stop a bad batch before it reaches the warehouse | `Gate(...).run(df).raise_for_errors()` or a CLI exit code of `1` |
71
+ | Tell on-call what happened | Readable observed values such as "3 duplicate keys" or "9.62% null" |
72
+ | Keep rules next to the pipeline | Contracts in Python or TOML, with row rules in SQL or Polars expressions |
73
+ | Stay light | One dependency (Polars), typed, single-pass expression evaluation |
74
+
75
+ ## Install
76
+
77
+ ```bash
78
+ pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
79
+ ```
80
+
81
+ ## Quick start
82
+
83
+ ```python
84
+ import polars as pl
85
+ import rowgate as rg
86
+ from rowgate import checks as ck
87
+
88
+ orders = pl.DataFrame({
89
+ "order_id": [1001, 1002, 1002, 1004, 1005],
90
+ "customer": ["acme", "globex", "globex", None, "initech"],
91
+ "amount": [250.0, 99.5, 99.5, -20.0, 1_200.0],
92
+ "currency": ["USD", "USD", "USD", "USD", "usd"],
93
+ })
94
+
95
+ contract = rg.Contract(columns={
96
+ "order_id": rg.Column(pl.Int64, required=True, unique=True),
97
+ "customer": rg.Column(pl.Utf8, required=True),
98
+ "amount": rg.Column(pl.Float64, min=0),
99
+ "currency": rg.Column(pl.Utf8, accepted=["USD", "EUR", "GBP"]),
100
+ })
101
+
102
+ result = contract.split(orders)
103
+ result.quarantine
104
+ ```
105
+
106
+ ```text
107
+ ┌──────────┬──────────┬────────┬──────────┬───────────────────────────────────────┐
108
+ │ order_id ┆ customer ┆ amount ┆ currency ┆ _rowgate_reasons │
109
+ ╞══════════╪══════════╪════════╪══════════╪═══════════════════════════════════════╡
110
+ │ 1002 ┆ globex ┆ 99.5 ┆ USD ┆ ["order_id:duplicate"] │
111
+ │ 1004 ┆ null ┆ -20.0 ┆ USD ┆ ["customer:required", "amount:min"] │
112
+ │ 1005 ┆ initech ┆ 1200.0 ┆ usd ┆ ["currency:accepted"] │
113
+ └──────────┴──────────┴────────┴──────────┴───────────────────────────────────────┘
114
+ ```
115
+
116
+ The first `1002` is kept and the repeat is quarantined. Row `1004` broke two rules, and both are listed.
117
+
118
+ ```python
119
+ report = rg.Gate([
120
+ ck.schema_matches(contract),
121
+ ck.reconciles(result),
122
+ ck.quarantine_rate(result, max_pct=10),
123
+ ck.row_count(min_rows=1),
124
+ ]).run(result.valid)
125
+
126
+ print(report)
127
+ report.raise_for_errors() # raises rowgate.GateFailed in a pipeline
128
+ ```
129
+
130
+ ```text
131
+ rowgate: FAIL (2 rows)
132
+ ok schema_matches no drift
133
+ ok rows_reconcile 2 valid + 3 quarantined = 5 vs 5 input
134
+ FAIL quarantine_rate 60.0%
135
+ ok row_count 2 rows
136
+ ```
137
+
138
+ ## On real data
139
+
140
+ [`examples/seattle_permits.py`](examples/seattle_permits.py) runs a contract over a 454-row sample of the City of Seattle's public [Building Permits](https://data.seattle.gov/Built-Environment/Building-Permits/76t5-zqzr) dataset:
141
+
142
+ ```text
143
+ rowgate: PASS (447 rows)
144
+ ok schema_matches no drift
145
+ ok rows_reconcile 447 valid + 7 quarantined = 454 vs 454 input
146
+ ok quarantine_rate 1.54%
147
+ ok unique:permit_num 0 duplicate keys
148
+ ok not_null:permit_num+permit_class 0 nulls
149
+ ok null_rate:applied_date 9.62% null
150
+ ok null_rate:zip 4.7% null
151
+ warn freshness:applied_date newest 2026-07-02 (110 days old)
152
+ ```
153
+
154
+ ```text
155
+ reason rows
156
+ issued_on_or_after_applied 3
157
+ permit_num:pattern 3
158
+ est_cost:max 1
159
+ ```
160
+
161
+ The 7 quarantined rows are real problems in the source data:
162
+
163
+ - 3 permits were issued before they were applied for.
164
+ - 3 sub-permit numbers like `6112685-EX-002` don't match the documented format.
165
+ - 1 permit lists an estimated cost of $25,000,000,000.
166
+
167
+ If sub-permits are valid for your use, widen the pattern in [`permits_contract.toml`](examples/permits_contract.toml). The freshness warning is expected, because the sample is a fixed extract.
168
+
169
+ The same approach runs at full scale in the [Seattle Permits Data Platform](https://github.com/DTheDataGuy/the-data-guys-portfolio/tree/main/data-engineering): 193,747 rows through a Polars pipeline, dbt, PySpark and Scala Spark.
170
+
171
+ ## Contracts
172
+
173
+ ### Columns
174
+
175
+ | Option | Meaning | Nulls |
176
+ |---|---|---|
177
+ | `dtype` | Expected Polars dtype, checked by `schema_matches` | n/a |
178
+ | `required` | Value must be present. Blank strings count as missing | fail |
179
+ | `unique` | First occurrence kept; later duplicates quarantined | skipped |
180
+ | `accepted` | Value must be in the list | skipped |
181
+ | `min`, `max` | Inclusive bounds | skipped |
182
+ | `pattern` | Regex the value must match | skipped |
183
+
184
+ Nulls only fail `required`, the same way SQL constraints behave. Set `allow_extra_columns=False` to treat unexpected columns as schema drift.
185
+
186
+ ### Rules
187
+
188
+ Row-level rules are Polars expressions that are true for valid rows:
189
+
190
+ ```python
191
+ rg.Rule("issued_on_or_after_applied", pl.col("issued_date") >= pl.col("applied_date"))
192
+ ```
193
+
194
+ A rule that evaluates to null passes, so a permit that hasn't been issued yet isn't rejected by an ordering rule.
195
+
196
+ ### TOML
197
+
198
+ Contracts can live next to the pipeline as config, with rules written in SQL:
199
+
200
+ ```toml
201
+ [columns.permit_num]
202
+ dtype = "str"
203
+ required = true
204
+ unique = true
205
+ pattern = '^\d{6,7}-[A-Z]{2}$'
206
+
207
+ [columns.est_cost]
208
+ dtype = "float"
209
+ min = 0
210
+ max = 5_000_000_000
211
+
212
+ [[rules]]
213
+ name = "issued_on_or_after_applied"
214
+ sql = "issued_date >= applied_date"
215
+ ```
216
+
217
+ ```python
218
+ contract = rg.Contract.from_toml("permits_contract.toml")
219
+ ```
220
+
221
+ ## Checks
222
+
223
+ | Check | Default severity | Passes when |
224
+ |---|---|---|
225
+ | `schema_matches(contract)` | error | Every contract column is present with the right dtype |
226
+ | `unique(cols)` | error | No duplicate keys (one column or a composite) |
227
+ | `not_null(cols)` | error | No nulls in the listed columns |
228
+ | `null_rate(col, max_pct)` | warn | Null share is at or below the limit |
229
+ | `accepted_values(col, values)` | error | No values outside the set |
230
+ | `row_count(min_rows, max_rows)` | error | Row count is within bounds |
231
+ | `reconciles(result)` | error | Valid + quarantined equals input |
232
+ | `quarantine_rate(result, max_pct)` | error | Quarantined share is below the limit |
233
+ | `freshness(col, max_age_days, as_of)` | warn | Newest date is recent enough |
234
+ | `expr(name, condition, max_failing)` | error | At most `max_failing` rows break the condition |
235
+ | `custom(name, fn, expectation)` | error | `fn(df)` returns `(True, observed)` |
236
+
237
+ Every check takes `severity="error"` or `severity="warn"`. If a check itself raises, for example because a column is missing, it's recorded as a failure with the exception message, and the other checks still run.
238
+
239
+ ## Reports
240
+
241
+ ```python
242
+ report.passed # False if any error check failed
243
+ report.errors # failed error checks
244
+ report.warnings # failed warn checks
245
+ report.to_json("quality_report.json")
246
+ report.to_markdown("quality_report.md")
247
+ ```
248
+
249
+ The Markdown output is a table that renders in GitHub, GitLab and most CI summaries:
250
+
251
+ | Check | Severity | Result | Observed | Expectation |
252
+ |---|---|---|---|---|
253
+ | `rows_reconcile` | error | pass | 2 valid + 3 quarantined = 5 vs 5 input | valid + quarantined = input |
254
+ | `quarantine_rate` | error | **FAIL** | 60.0% | < 10% of input quarantined |
255
+
256
+ ## Use it in CI and orchestrators
257
+
258
+ GitHub Actions, failing the job and posting the report to the run summary:
259
+
260
+ ```yaml
261
+ - run: pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
262
+ - run: rowgate check data/orders.parquet --contract contracts/orders.toml --report-dir out/
263
+ - if: always()
264
+ run: cat out/report.md >> "$GITHUB_STEP_SUMMARY"
265
+ ```
266
+
267
+ Inside an Airflow, Dagster or Prefect task, or any Python job:
268
+
269
+ ```python
270
+ result = contract.split(df)
271
+ result.quarantine.write_parquet("quarantine/orders.parquet") # keep the evidence
272
+ rg.Gate([ck.reconciles(result), ck.quarantine_rate(result, max_pct=2)]).run(result.valid).raise_for_errors()
273
+ result.valid.write_parquet("silver/orders.parquet") # only reached when the gate passes
274
+ ```
275
+
276
+ ## Command line
277
+
278
+ ```bash
279
+ rowgate check orders.parquet --contract contract.toml --report-dir out/
280
+ ```
281
+
282
+ The command reads Parquet, CSV, NDJSON or JSON files. It prints the report and a count of quarantine reasons. With `--report-dir` it also writes `report.json`, `report.md` and `quarantine.parquet`. It exits with `1` when a blocking check fails, so it can stop a CI job or an orchestrator task. Add `--strict` to fail on warnings too.
283
+
284
+ ## Design notes
285
+
286
+ - **Quarantine, don't drop.** A pipeline that deletes bad rows makes its own totals impossible to audit. rowgate keeps every row and records why it was set aside.
287
+ - **Every reason is listed.** A row that breaks three rules lists all three, so one fix doesn't reveal the next problem a day later.
288
+ - **The observed value is readable.** "3 duplicate keys" or "9.62% null" tells the on-call person what to do. A bare `False` doesn't.
289
+ - **Lazy-friendly expressions.** Column checks compile to Polars expressions and evaluate in a single `with_columns` pass.
290
+
291
+ ## Development
292
+
293
+ ```bash
294
+ pip install -e ".[dev]"
295
+ pytest -q
296
+ ruff check .
297
+ ```
298
+
299
+ CI runs the test suite, the Seattle example and a CLI smoke test on Python 3.11, 3.12 and 3.13. It also runs the suite against Polars 1.0.0, the oldest supported version.
300
+
301
+ ## License
302
+
303
+ MIT © Dei'Marlon Scisney · [The Data Guys™](https://www.thedataguys.org)
304
+
305
+ If rowgate saves you a hand-rolled validation script, a star helps other data engineers find it.
@@ -0,0 +1,12 @@
1
+ rowgate/__init__.py,sha256=zBaUSe7XYJDtBusZKaAk9oYvWjOYlX6jkSECqvdqV6I,417
2
+ rowgate/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
3
+ rowgate/checks.py,sha256=MbWEKXOH_T-Lk0zk2c70pP3M1uXgU2BhSQ1W7x0q2DY,5432
4
+ rowgate/cli.py,sha256=QEIKFHGGEnnRqLOLGQZPueXCcsbWVSBmSyqSQAPirKk,2554
5
+ rowgate/contract.py,sha256=_Horj2gWR13L14wyxJI31oBsxwxq-ogAqKIKP85SSos,7670
6
+ rowgate/gate.py,sha256=BugMoIyBkHbDeMIhaXgLD_0tGyDGTTDv6eBqgyICOck,4390
7
+ rowgate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ rowgate-0.1.0.dist-info/METADATA,sha256=IE0F5Zz-e1y1xiYYYE8B0pi4Dw_TxVQr7VAjk5N6BTw,13874
9
+ rowgate-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
10
+ rowgate-0.1.0.dist-info/entry_points.txt,sha256=Wph4scCAc9hZSvoMiHzuJalOVVaE-2cBMV1PQPchHmM,45
11
+ rowgate-0.1.0.dist-info/licenses/LICENSE,sha256=mEPEMRah4ptY3kgnGZf_0IcoWBWv-CIeHR4Qj_jFwsk,1090
12
+ rowgate-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ rowgate = rowgate.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Dei'Marlon Scisney, The Data Guys
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.