rowgate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowgate/__init__.py +9 -0
- rowgate/__main__.py +3 -0
- rowgate/checks.py +123 -0
- rowgate/cli.py +65 -0
- rowgate/contract.py +185 -0
- rowgate/gate.py +121 -0
- rowgate/py.typed +0 -0
- rowgate-0.1.0.dist-info/METADATA +305 -0
- rowgate-0.1.0.dist-info/RECORD +12 -0
- rowgate-0.1.0.dist-info/WHEEL +4 -0
- rowgate-0.1.0.dist-info/entry_points.txt +2 -0
- rowgate-0.1.0.dist-info/licenses/LICENSE +21 -0
rowgate/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""rowgate: data contracts and quality gates for Polars."""
|
|
2
|
+
from . import checks
|
|
3
|
+
from .checks import Check
|
|
4
|
+
from .contract import REASONS, Column, Contract, Rule, SplitResult, columns
|
|
5
|
+
from .gate import CheckResult, Gate, GateFailed, Report
|
|
6
|
+
|
|
7
|
+
__all__ = ["Check", "CheckResult", "Column", "Contract", "Gate", "GateFailed", "REASONS", "Report",
|
|
8
|
+
"Rule", "SplitResult", "checks", "columns"]
|
|
9
|
+
__version__ = "0.1.0"
|
rowgate/__main__.py
ADDED
rowgate/checks.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Dataset-level checks for a :class:`rowgate.Gate`.
|
|
2
|
+
|
|
3
|
+
Each factory returns a :class:`Check`: a name, a severity and a function that looks at
|
|
4
|
+
a DataFrame and returns ``(passed, observed)``. ``error`` checks block the batch,
|
|
5
|
+
``warn`` checks are reported but don't block it.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable, Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import date, datetime
|
|
12
|
+
|
|
13
|
+
import polars as pl
|
|
14
|
+
|
|
15
|
+
from .contract import Contract, SplitResult
|
|
16
|
+
|
|
17
|
+
Severity = str # "error" | "warn"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class Check:
|
|
22
|
+
name: str
|
|
23
|
+
severity: Severity
|
|
24
|
+
expectation: str
|
|
25
|
+
fn: Callable[[pl.DataFrame], tuple[bool, str]]
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
if self.severity not in ("error", "warn"):
|
|
29
|
+
raise ValueError(f"severity must be 'error' or 'warn', got {self.severity!r}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _pct(n: int, d: int) -> float:
|
|
33
|
+
return round(100.0 * n / d, 2) if d else 0.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def schema_matches(contract: Contract, severity: Severity = "error") -> Check:
|
|
37
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
38
|
+
drift = contract.schema_drift(df)
|
|
39
|
+
if not drift:
|
|
40
|
+
return True, "no drift"
|
|
41
|
+
return False, "; ".join(f"{k}: expected {e}, got {a}" for k, (e, a) in sorted(drift.items()))
|
|
42
|
+
return Check("schema_matches", severity, "columns and dtypes match the contract", fn)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def unique(cols: str | Sequence[str], severity: Severity = "error") -> Check:
|
|
46
|
+
keys = [cols] if isinstance(cols, str) else list(cols)
|
|
47
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
48
|
+
dup = df.height - df.select(keys).unique().height
|
|
49
|
+
return dup == 0, f"{dup:,} duplicate keys"
|
|
50
|
+
return Check(f"unique:{'+'.join(keys)}", severity, "0 duplicate keys", fn)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def not_null(cols: str | Sequence[str], severity: Severity = "error") -> Check:
|
|
54
|
+
names = [cols] if isinstance(cols, str) else list(cols)
|
|
55
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
56
|
+
bad = {c: n for c in names if (n := df[c].null_count())}
|
|
57
|
+
return not bad, ", ".join(f"{c}: {n:,}" for c, n in bad.items()) or "0 nulls"
|
|
58
|
+
return Check(f"not_null:{'+'.join(names)}", severity, f"no nulls in {len(names)} column(s)", fn)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def null_rate(col: str, max_pct: float, severity: Severity = "warn") -> Check:
|
|
62
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
63
|
+
p = _pct(df[col].null_count(), df.height)
|
|
64
|
+
return p <= max_pct, f"{p}% null"
|
|
65
|
+
return Check(f"null_rate:{col}", severity, f"<= {max_pct}% null", fn)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def accepted_values(col: str, values: Sequence[object], severity: Severity = "error") -> Check:
|
|
69
|
+
allowed = set(values)
|
|
70
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
71
|
+
seen = set(df[col].drop_nulls().unique().to_list())
|
|
72
|
+
extra = sorted(map(str, seen - allowed))
|
|
73
|
+
return not extra, "unexpected: " + ", ".join(extra) if extra else "all values accepted"
|
|
74
|
+
return Check(f"accepted_values:{col}", severity, f"subset of {len(allowed)} accepted values", fn)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def row_count(min_rows: int = 1, max_rows: int | None = None, severity: Severity = "error") -> Check:
|
|
78
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
79
|
+
ok = df.height >= min_rows and (max_rows is None or df.height <= max_rows)
|
|
80
|
+
return ok, f"{df.height:,} rows"
|
|
81
|
+
bound = f">= {min_rows:,}" + (f" and <= {max_rows:,}" if max_rows is not None else "")
|
|
82
|
+
return Check("row_count", severity, f"{bound} rows", fn)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def reconciles(result: SplitResult, severity: Severity = "error") -> Check:
|
|
86
|
+
"""Every input row is either valid or quarantined."""
|
|
87
|
+
def fn(_: pl.DataFrame) -> tuple[bool, str]:
|
|
88
|
+
total = result.valid.height + result.quarantine.height
|
|
89
|
+
return total == result.input_rows, (
|
|
90
|
+
f"{result.valid.height:,} valid + {result.quarantine.height:,} quarantined = {total:,} "
|
|
91
|
+
f"vs {result.input_rows:,} input")
|
|
92
|
+
return Check("rows_reconcile", severity, "valid + quarantined = input", fn)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def quarantine_rate(result: SplitResult, max_pct: float, severity: Severity = "error") -> Check:
|
|
96
|
+
def fn(_: pl.DataFrame) -> tuple[bool, str]:
|
|
97
|
+
return result.quarantine_rate < max_pct, f"{round(result.quarantine_rate, 2)}%"
|
|
98
|
+
return Check("quarantine_rate", severity, f"< {max_pct}% of input quarantined", fn)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def freshness(col: str, max_age_days: int, as_of: date | None = None, severity: Severity = "warn") -> Check:
|
|
102
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
103
|
+
newest = df[col].max()
|
|
104
|
+
if newest is None:
|
|
105
|
+
return False, "no values"
|
|
106
|
+
ref = as_of or date.today()
|
|
107
|
+
newest_d = newest.date() if isinstance(newest, datetime) else newest
|
|
108
|
+
age = (ref - newest_d).days
|
|
109
|
+
return age <= max_age_days, f"newest {newest_d} ({age} days old)"
|
|
110
|
+
return Check(f"freshness:{col}", severity, f"<= {max_age_days} days old", fn)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def expr(name: str, condition: pl.Expr, severity: Severity = "error", max_failing: int = 0) -> Check:
|
|
114
|
+
"""Custom row condition that must hold for (almost) every row."""
|
|
115
|
+
def fn(df: pl.DataFrame) -> tuple[bool, str]:
|
|
116
|
+
failing = df.select(condition.not_().fill_null(False).sum()).item()
|
|
117
|
+
return failing <= max_failing, f"{failing:,} rows fail"
|
|
118
|
+
return Check(name, severity, f"<= {max_failing:,} failing rows", fn)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def custom(name: str, fn: Callable[[pl.DataFrame], tuple[bool, str]], expectation: str,
|
|
122
|
+
severity: Severity = "error") -> Check:
|
|
123
|
+
return Check(name, severity, expectation, fn)
|
rowgate/cli.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Command line: ``rowgate check data.parquet --contract contract.toml``.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 pass, 1 blocking failure (or any failure with ``--strict``), 2 usage error.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import polars as pl
|
|
12
|
+
|
|
13
|
+
from . import checks as ck
|
|
14
|
+
from .contract import Contract
|
|
15
|
+
from .gate import Gate
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _read(path: Path) -> pl.DataFrame:
|
|
19
|
+
suffix = path.suffix.lower()
|
|
20
|
+
if suffix == ".parquet":
|
|
21
|
+
return pl.read_parquet(path)
|
|
22
|
+
if suffix == ".csv":
|
|
23
|
+
return pl.read_csv(path, try_parse_dates=True)
|
|
24
|
+
if suffix in (".ndjson", ".jsonl"):
|
|
25
|
+
return pl.read_ndjson(path)
|
|
26
|
+
if suffix == ".json":
|
|
27
|
+
return pl.read_json(path)
|
|
28
|
+
raise SystemExit(f"rowgate: unsupported file type {suffix!r} (use parquet, csv, ndjson or json)")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def main(argv: list[str] | None = None) -> int:
|
|
32
|
+
p = argparse.ArgumentParser(prog="rowgate", description="Data contracts and quality gates for Polars.")
|
|
33
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
34
|
+
c = sub.add_parser("check", help="validate a file against a contract")
|
|
35
|
+
c.add_argument("data", type=Path)
|
|
36
|
+
c.add_argument("--contract", type=Path, required=True)
|
|
37
|
+
c.add_argument("--max-quarantine-pct", type=float, default=1.0)
|
|
38
|
+
c.add_argument("--report-dir", type=Path, help="write report.json, report.md and quarantine.parquet here")
|
|
39
|
+
c.add_argument("--strict", action="store_true", help="treat warnings as failures")
|
|
40
|
+
a = p.parse_args(argv)
|
|
41
|
+
|
|
42
|
+
contract = Contract.from_toml(a.contract)
|
|
43
|
+
df = _read(a.data)
|
|
44
|
+
result = contract.split(df)
|
|
45
|
+
gate_checks = [ck.schema_matches(contract), ck.reconciles(result),
|
|
46
|
+
ck.quarantine_rate(result, a.max_quarantine_pct)]
|
|
47
|
+
gate_checks += [ck.unique(n) for n, col in contract.columns.items() if col.unique and n in df.columns]
|
|
48
|
+
report = Gate(gate_checks).run(result.valid, source=str(a.data), contract=str(a.contract))
|
|
49
|
+
|
|
50
|
+
print(report)
|
|
51
|
+
if not result.quarantine.is_empty():
|
|
52
|
+
print("\nquarantine reasons:")
|
|
53
|
+
for reason, rows in result.reason_counts().iter_rows():
|
|
54
|
+
print(f" {rows:>8,} {reason}")
|
|
55
|
+
if a.report_dir:
|
|
56
|
+
a.report_dir.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
report.to_json(a.report_dir / "report.json")
|
|
58
|
+
report.to_markdown(a.report_dir / "report.md")
|
|
59
|
+
result.quarantine.write_parquet(a.report_dir / "quarantine.parquet")
|
|
60
|
+
failed = not report.passed or (a.strict and report.warnings)
|
|
61
|
+
return 1 if failed else 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
sys.exit(main())
|
rowgate/contract.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Column contracts and row-level rules.
|
|
2
|
+
|
|
3
|
+
A :class:`Contract` describes what a valid row looks like. :meth:`Contract.split`
|
|
4
|
+
evaluates it against a DataFrame and returns the rows that pass, plus a quarantine
|
|
5
|
+
frame where every rejected row carries the names of the rules it broke.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
import tomllib
|
|
11
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import polars as pl
|
|
17
|
+
|
|
18
|
+
REASONS = "_rowgate_reasons"
|
|
19
|
+
|
|
20
|
+
_DTYPES: dict[str, Any] = {
|
|
21
|
+
"str": pl.Utf8, "string": pl.Utf8, "utf8": pl.Utf8,
|
|
22
|
+
"int": pl.Int64, "int64": pl.Int64, "int32": pl.Int32,
|
|
23
|
+
"float": pl.Float64, "float64": pl.Float64, "float32": pl.Float32,
|
|
24
|
+
"bool": pl.Boolean, "boolean": pl.Boolean,
|
|
25
|
+
"date": pl.Date, "datetime": pl.Datetime, "decimal": pl.Decimal,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Column:
|
|
31
|
+
"""Expectations for a single column.
|
|
32
|
+
|
|
33
|
+
``None`` values only fail ``required``. Range, pattern and accepted-value checks
|
|
34
|
+
skip nulls, the same way SQL constraints do.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
dtype: Any = None
|
|
38
|
+
required: bool = False
|
|
39
|
+
unique: bool = False
|
|
40
|
+
accepted: Sequence[Any] | None = None
|
|
41
|
+
min: Any = None
|
|
42
|
+
max: Any = None
|
|
43
|
+
pattern: str | None = None
|
|
44
|
+
|
|
45
|
+
def __post_init__(self) -> None:
|
|
46
|
+
if self.pattern is not None:
|
|
47
|
+
re.compile(self.pattern) # fail early on a bad regex
|
|
48
|
+
if self.min is not None and self.max is not None and self.min > self.max:
|
|
49
|
+
raise ValueError(f"min ({self.min}) is greater than max ({self.max})")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class Rule:
|
|
54
|
+
"""A row-level rule. ``check`` must evaluate to True for valid rows.
|
|
55
|
+
|
|
56
|
+
A null result counts as a pass, so a rule like ``issued >= applied`` does not
|
|
57
|
+
reject rows that simply haven't been issued yet.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
name: str
|
|
61
|
+
check: pl.Expr
|
|
62
|
+
description: str = ""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class SplitResult:
|
|
67
|
+
valid: pl.DataFrame
|
|
68
|
+
quarantine: pl.DataFrame
|
|
69
|
+
input_rows: int
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def quarantine_rate(self) -> float:
|
|
73
|
+
return 100.0 * self.quarantine.height / self.input_rows if self.input_rows else 0.0
|
|
74
|
+
|
|
75
|
+
def reason_counts(self) -> pl.DataFrame:
|
|
76
|
+
"""How many quarantined rows broke each rule, most common first."""
|
|
77
|
+
if self.quarantine.is_empty():
|
|
78
|
+
return pl.DataFrame({"reason": [], "rows": []}, schema={"reason": pl.Utf8, "rows": pl.UInt32})
|
|
79
|
+
return (self.quarantine.select(pl.col(REASONS).explode().alias("reason"))
|
|
80
|
+
.group_by("reason").len(name="rows").sort(["rows", "reason"], descending=[True, False]))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Contract:
|
|
85
|
+
columns: Mapping[str, Column]
|
|
86
|
+
rules: Sequence[Rule] = field(default_factory=list)
|
|
87
|
+
allow_extra_columns: bool = True
|
|
88
|
+
|
|
89
|
+
# ------------------------------------------------------------------ loading
|
|
90
|
+
@classmethod
|
|
91
|
+
def from_toml(cls, path: str | Path) -> Contract:
|
|
92
|
+
"""Load a contract from TOML. See ``examples/permits_contract.toml``."""
|
|
93
|
+
raw = tomllib.loads(Path(path).read_text())
|
|
94
|
+
return cls.from_dict(raw)
|
|
95
|
+
|
|
96
|
+
@classmethod
|
|
97
|
+
def from_dict(cls, raw: Mapping[str, Any]) -> Contract:
|
|
98
|
+
cols: dict[str, Column] = {}
|
|
99
|
+
for name, spec in raw.get("columns", {}).items():
|
|
100
|
+
spec = dict(spec)
|
|
101
|
+
dt = spec.pop("dtype", None)
|
|
102
|
+
if dt is not None:
|
|
103
|
+
key = str(dt).lower()
|
|
104
|
+
if key not in _DTYPES:
|
|
105
|
+
raise ValueError(f"column {name!r}: unknown dtype {dt!r}; use one of {sorted(_DTYPES)}")
|
|
106
|
+
spec["dtype"] = _DTYPES[key]
|
|
107
|
+
cols[name] = Column(**spec)
|
|
108
|
+
rules = [Rule(r["name"], pl.sql_expr(r["sql"]), r.get("description", ""))
|
|
109
|
+
for r in raw.get("rules", [])]
|
|
110
|
+
return cls(cols, rules, raw.get("allow_extra_columns", True))
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------- inspection
|
|
113
|
+
def schema_drift(self, df: pl.DataFrame) -> dict[str, tuple[str, str]]:
|
|
114
|
+
"""Columns whose presence or dtype differs from the contract: ``{name: (expected, actual)}``."""
|
|
115
|
+
actual = dict(df.schema)
|
|
116
|
+
drift: dict[str, tuple[str, str]] = {}
|
|
117
|
+
for name, col in self.columns.items():
|
|
118
|
+
if name not in actual:
|
|
119
|
+
drift[name] = (str(col.dtype or "any"), "missing")
|
|
120
|
+
elif col.dtype is not None and not _dtype_matches(actual[name], col.dtype):
|
|
121
|
+
drift[name] = (str(col.dtype), str(actual[name]))
|
|
122
|
+
if not self.allow_extra_columns:
|
|
123
|
+
for name in actual.keys() - self.columns.keys():
|
|
124
|
+
drift[name] = ("absent", str(actual[name]))
|
|
125
|
+
return drift
|
|
126
|
+
|
|
127
|
+
def failure_expressions(self, df: pl.DataFrame) -> list[tuple[str, pl.Expr]]:
|
|
128
|
+
"""``(reason, expr)`` pairs where ``expr`` is True for rows that fail."""
|
|
129
|
+
out: list[tuple[str, pl.Expr]] = []
|
|
130
|
+
present = set(df.columns)
|
|
131
|
+
for name, col in self.columns.items():
|
|
132
|
+
if name not in present:
|
|
133
|
+
continue
|
|
134
|
+
c = pl.col(name)
|
|
135
|
+
if col.required:
|
|
136
|
+
fail = c.is_null()
|
|
137
|
+
if df.schema[name] == pl.Utf8:
|
|
138
|
+
fail = fail | (c.str.strip_chars() == "")
|
|
139
|
+
out.append((f"{name}:required", fail))
|
|
140
|
+
if col.accepted is not None:
|
|
141
|
+
out.append((f"{name}:accepted", c.is_not_null() & ~c.is_in(list(col.accepted))))
|
|
142
|
+
if col.min is not None:
|
|
143
|
+
out.append((f"{name}:min", c < col.min))
|
|
144
|
+
if col.max is not None:
|
|
145
|
+
out.append((f"{name}:max", c > col.max))
|
|
146
|
+
if col.pattern is not None:
|
|
147
|
+
out.append((f"{name}:pattern", c.is_not_null() & ~c.cast(pl.Utf8).str.contains(col.pattern)))
|
|
148
|
+
if col.unique:
|
|
149
|
+
# keep the first occurrence, quarantine later ones
|
|
150
|
+
out.append((f"{name}:duplicate", c.is_not_null() & c.is_duplicated() & ~c.is_first_distinct()))
|
|
151
|
+
for rule in self.rules:
|
|
152
|
+
out.append((rule.name, rule.check.not_().fill_null(False)))
|
|
153
|
+
return out
|
|
154
|
+
|
|
155
|
+
# -------------------------------------------------------------------- split
|
|
156
|
+
def split(self, df: pl.DataFrame) -> SplitResult:
|
|
157
|
+
"""Separate valid rows from rows that break the contract.
|
|
158
|
+
|
|
159
|
+
Nothing is dropped: ``valid.height + quarantine.height == df.height``.
|
|
160
|
+
"""
|
|
161
|
+
missing = [n for n, c in self.columns.items() if c.required and n not in df.columns]
|
|
162
|
+
if missing:
|
|
163
|
+
raise KeyError(f"required columns missing from input: {missing}")
|
|
164
|
+
fails = self.failure_expressions(df)
|
|
165
|
+
if not fails:
|
|
166
|
+
return SplitResult(df, df.clear().with_columns(pl.lit([], pl.List(pl.Utf8)).alias(REASONS)), df.height)
|
|
167
|
+
reasons = pl.concat_list(
|
|
168
|
+
[pl.when(expr.fill_null(False)).then(pl.lit(name)).otherwise(pl.lit(None, pl.Utf8)) for name, expr in fails]
|
|
169
|
+
).list.drop_nulls().alias(REASONS)
|
|
170
|
+
tagged = df.with_columns(reasons)
|
|
171
|
+
bad = tagged[REASONS].list.len() > 0
|
|
172
|
+
return SplitResult(valid=tagged.filter(~bad).drop(REASONS), quarantine=tagged.filter(bad), input_rows=df.height)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _dtype_matches(actual: Any, expected: Any) -> bool:
|
|
176
|
+
if actual == expected:
|
|
177
|
+
return True
|
|
178
|
+
# Datetime/Decimal declared without parameters match any time unit, zone or precision.
|
|
179
|
+
base = getattr(expected, "base_type", lambda: expected)()
|
|
180
|
+
return actual.base_type() == base if hasattr(actual, "base_type") else False
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def columns(names: Iterable[str], **kw: Any) -> dict[str, Column]:
|
|
184
|
+
"""Shorthand for several columns with the same expectations."""
|
|
185
|
+
return {n: Column(**kw) for n in names}
|
rowgate/gate.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Run a list of checks and produce a report that machines and people can both read."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import asdict, dataclass, field
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import polars as pl
|
|
13
|
+
|
|
14
|
+
from .checks import Check
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class GateFailed(RuntimeError):
|
|
18
|
+
def __init__(self, report: Report) -> None:
|
|
19
|
+
self.report = report
|
|
20
|
+
names = ", ".join(r.name for r in report.errors)
|
|
21
|
+
super().__init__(f"{len(report.errors)} blocking check(s) failed: {names}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class CheckResult:
|
|
26
|
+
name: str
|
|
27
|
+
severity: str
|
|
28
|
+
passed: bool
|
|
29
|
+
observed: str
|
|
30
|
+
expectation: str
|
|
31
|
+
seconds: float
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Report:
|
|
36
|
+
results: list[CheckResult]
|
|
37
|
+
rows: int
|
|
38
|
+
created_at: str
|
|
39
|
+
meta: dict[str, Any] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def errors(self) -> list[CheckResult]:
|
|
43
|
+
return [r for r in self.results if r.severity == "error" and not r.passed]
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def warnings(self) -> list[CheckResult]:
|
|
47
|
+
return [r for r in self.results if r.severity == "warn" and not r.passed]
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def passed(self) -> bool:
|
|
51
|
+
return not self.errors
|
|
52
|
+
|
|
53
|
+
def raise_for_errors(self) -> Report:
|
|
54
|
+
if self.errors:
|
|
55
|
+
raise GateFailed(self)
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
def to_dict(self) -> dict[str, Any]:
|
|
59
|
+
return {"passed": self.passed, "rows": self.rows, "created_at": self.created_at,
|
|
60
|
+
"meta": self.meta, "errors": len(self.errors), "warnings": len(self.warnings),
|
|
61
|
+
"results": [asdict(r) for r in self.results]}
|
|
62
|
+
|
|
63
|
+
def to_json(self, path: str | Path | None = None) -> str:
|
|
64
|
+
text = json.dumps(self.to_dict(), indent=2, default=str)
|
|
65
|
+
if path is not None:
|
|
66
|
+
Path(path).write_text(text + "\n")
|
|
67
|
+
return text
|
|
68
|
+
|
|
69
|
+
def to_markdown(self, path: str | Path | None = None, title: str = "Data-quality report") -> str:
|
|
70
|
+
status = "PASS" if self.passed else "FAIL"
|
|
71
|
+
lines = [f"# {title}", "",
|
|
72
|
+
f"**{status}** · {_rows(self.rows)} · {len(self.results)} checks · "
|
|
73
|
+
f"{len(self.errors)} blocking failures · {len(self.warnings)} warnings · {self.created_at}", "",
|
|
74
|
+
"| Check | Severity | Result | Observed | Expectation |", "|---|---|---|---|---|"]
|
|
75
|
+
for r in self.results:
|
|
76
|
+
res = "pass" if r.passed else ("**FAIL**" if r.severity == "error" else "warn")
|
|
77
|
+
lines.append(f"| `{r.name}` | {r.severity} | {res} | {_md(r.observed)} | {_md(r.expectation)} |")
|
|
78
|
+
text = "\n".join(lines) + "\n"
|
|
79
|
+
if path is not None:
|
|
80
|
+
Path(path).write_text(text)
|
|
81
|
+
return text
|
|
82
|
+
|
|
83
|
+
def __str__(self) -> str:
|
|
84
|
+
width = max((len(r.name) for r in self.results), default=10)
|
|
85
|
+
out = [f"rowgate: {'PASS' if self.passed else 'FAIL'} ({_rows(self.rows)})"]
|
|
86
|
+
for r in self.results:
|
|
87
|
+
mark = "ok " if r.passed else ("FAIL" if r.severity == "error" else "warn")
|
|
88
|
+
out.append(f" {mark} {r.name:<{width}} {r.observed}")
|
|
89
|
+
return "\n".join(out)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _rows(n: int) -> str:
|
|
93
|
+
return f"{n:,} row" + ("" if n == 1 else "s")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _md(s: str) -> str:
|
|
97
|
+
return s.replace("|", "\\|").replace("\n", " ")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Gate:
|
|
101
|
+
"""An ordered set of checks. A check that raises is recorded as a failure, not a crash."""
|
|
102
|
+
|
|
103
|
+
def __init__(self, checks: Sequence[Check]) -> None:
|
|
104
|
+
names = [c.name for c in checks]
|
|
105
|
+
dupes = {n for n in names if names.count(n) > 1}
|
|
106
|
+
if dupes:
|
|
107
|
+
raise ValueError(f"duplicate check names: {sorted(dupes)}")
|
|
108
|
+
self.checks = list(checks)
|
|
109
|
+
|
|
110
|
+
def run(self, df: pl.DataFrame, **meta: Any) -> Report:
|
|
111
|
+
results = []
|
|
112
|
+
for c in self.checks:
|
|
113
|
+
t0 = time.perf_counter()
|
|
114
|
+
try:
|
|
115
|
+
passed, observed = c.fn(df)
|
|
116
|
+
except Exception as exc: # noqa: BLE001 - a broken check must not hide the others
|
|
117
|
+
passed, observed = False, f"check raised {type(exc).__name__}: {exc}"
|
|
118
|
+
results.append(CheckResult(c.name, c.severity, bool(passed), observed, c.expectation,
|
|
119
|
+
round(time.perf_counter() - t0, 4)))
|
|
120
|
+
created = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
121
|
+
return Report(results, df.height, created, meta)
|
rowgate/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rowgate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Data contracts and quality gates for Polars: quarantine bad rows with reasons, block bad batches, and write reports people can read.
|
|
5
|
+
Project-URL: Homepage, https://github.com/DTheDataGuy/rowgate
|
|
6
|
+
Project-URL: Documentation, https://github.com/DTheDataGuy/rowgate#readme
|
|
7
|
+
Project-URL: Source, https://github.com/DTheDataGuy/rowgate
|
|
8
|
+
Project-URL: Issues, https://github.com/DTheDataGuy/rowgate/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/DTheDataGuy/rowgate/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: The Data Guys, https://www.thedataguys.org
|
|
11
|
+
Author-email: Dei'Marlon Scisney <info@thedataguys.org>
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: data-contracts,data-engineering,data-pipelines,data-quality,data-testing,data-validation,dataframe,etl,polars,quality-gate,quarantine,schema,validation
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
25
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.11
|
|
28
|
+
Requires-Dist: polars>=1.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pyarrow; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
<div align="center">
|
|
36
|
+
|
|
37
|
+
<img src="docs/assets/social-preview.png" alt="rowgate: data contracts and quality gates for Polars" width="100%">
|
|
38
|
+
|
|
39
|
+
# rowgate
|
|
40
|
+
|
|
41
|
+
**Data contracts and quality gates for [Polars](https://pola.rs). Quarantine bad rows with reasons, block bad batches, and write reports people can read.**
|
|
42
|
+
|
|
43
|
+
[](https://github.com/DTheDataGuy/rowgate/actions/workflows/ci.yml)
|
|
44
|
+
[](https://github.com/DTheDataGuy/rowgate/releases)
|
|
45
|
+

|
|
46
|
+

|
|
47
|
+

|
|
48
|
+
[](LICENSE)
|
|
49
|
+
[](https://github.com/DTheDataGuy/rowgate/stargazers)
|
|
50
|
+
|
|
51
|
+
[Install](#install) · [Quick start](#quick-start) · [Why rowgate](#why-rowgate) · [Real data](#on-real-data) · [Contracts](#contracts) · [Checks](#checks) · [CI and orchestrators](#use-it-in-ci-and-orchestrators) · [CLI](#command-line) · [Contributing](CONTRIBUTING.md) · [Cite](CITATION.cff)
|
|
52
|
+
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
rowgate does two jobs that most pipelines need and usually hand-roll:
|
|
56
|
+
|
|
57
|
+
1. **Split** a batch into rows that meet the contract and rows that don't. Rejected rows go to a quarantine frame that lists every rule each row broke. Nothing is silently dropped, and `valid + quarantined = input` always holds.
|
|
58
|
+
2. **Gate** the batch with dataset-level checks (schema drift, uniqueness, null rates, freshness, reconciliation). `error` checks block the load, `warn` checks are reported. The result is a report you can log as JSON, attach to a run as Markdown or print to a terminal.
|
|
59
|
+
|
|
60
|
+
The only dependency is Polars.
|
|
61
|
+
|
|
62
|
+
## Why rowgate
|
|
63
|
+
|
|
64
|
+
Most validation tools answer one question: did this DataFrame pass? Production pipelines need three more answers. Which rows failed, why did each one fail, and is it safe to load the rest? rowgate is built around those questions.
|
|
65
|
+
|
|
66
|
+
| You need to | rowgate gives you |
|
|
67
|
+
|---|---|
|
|
68
|
+
| Keep loading good rows when a few are bad | `contract.split(df)` returns `valid` and `quarantine` frames, and proves they add up to the input |
|
|
69
|
+
| Fix source data in one pass | Every broken rule is listed per row, not just the first one |
|
|
70
|
+
| Stop a bad batch before it reaches the warehouse | `Gate(...).run(df).raise_for_errors()` or a CLI exit code of `1` |
|
|
71
|
+
| Tell on-call what happened | Readable observed values such as "3 duplicate keys" or "9.62% null" |
|
|
72
|
+
| Keep rules next to the pipeline | Contracts in Python or TOML, with row rules in SQL or Polars expressions |
|
|
73
|
+
| Stay light | One dependency (Polars), typed, single-pass expression evaluation |
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Quick start
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
import polars as pl
|
|
85
|
+
import rowgate as rg
|
|
86
|
+
from rowgate import checks as ck
|
|
87
|
+
|
|
88
|
+
orders = pl.DataFrame({
|
|
89
|
+
"order_id": [1001, 1002, 1002, 1004, 1005],
|
|
90
|
+
"customer": ["acme", "globex", "globex", None, "initech"],
|
|
91
|
+
"amount": [250.0, 99.5, 99.5, -20.0, 1_200.0],
|
|
92
|
+
"currency": ["USD", "USD", "USD", "USD", "usd"],
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
contract = rg.Contract(columns={
|
|
96
|
+
"order_id": rg.Column(pl.Int64, required=True, unique=True),
|
|
97
|
+
"customer": rg.Column(pl.Utf8, required=True),
|
|
98
|
+
"amount": rg.Column(pl.Float64, min=0),
|
|
99
|
+
"currency": rg.Column(pl.Utf8, accepted=["USD", "EUR", "GBP"]),
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
result = contract.split(orders)
|
|
103
|
+
result.quarantine
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
```text
|
|
107
|
+
┌──────────┬──────────┬────────┬──────────┬───────────────────────────────────────┐
|
|
108
|
+
│ order_id ┆ customer ┆ amount ┆ currency ┆ _rowgate_reasons │
|
|
109
|
+
╞══════════╪══════════╪════════╪══════════╪═══════════════════════════════════════╡
|
|
110
|
+
│ 1002 ┆ globex ┆ 99.5 ┆ USD ┆ ["order_id:duplicate"] │
|
|
111
|
+
│ 1004 ┆ null ┆ -20.0 ┆ USD ┆ ["customer:required", "amount:min"] │
|
|
112
|
+
│ 1005 ┆ initech ┆ 1200.0 ┆ usd ┆ ["currency:accepted"] │
|
|
113
|
+
└──────────┴──────────┴────────┴──────────┴───────────────────────────────────────┘
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The first `1002` is kept and the repeat is quarantined. Row `1004` broke two rules, and both are listed.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
report = rg.Gate([
|
|
120
|
+
ck.schema_matches(contract),
|
|
121
|
+
ck.reconciles(result),
|
|
122
|
+
ck.quarantine_rate(result, max_pct=10),
|
|
123
|
+
ck.row_count(min_rows=1),
|
|
124
|
+
]).run(result.valid)
|
|
125
|
+
|
|
126
|
+
print(report)
|
|
127
|
+
report.raise_for_errors() # raises rowgate.GateFailed in a pipeline
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
```text
|
|
131
|
+
rowgate: FAIL (2 rows)
|
|
132
|
+
ok schema_matches no drift
|
|
133
|
+
ok rows_reconcile 2 valid + 3 quarantined = 5 vs 5 input
|
|
134
|
+
FAIL quarantine_rate 60.0%
|
|
135
|
+
ok row_count 2 rows
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## On real data
|
|
139
|
+
|
|
140
|
+
[`examples/seattle_permits.py`](examples/seattle_permits.py) runs a contract over a 454-row sample of the City of Seattle's public [Building Permits](https://data.seattle.gov/Built-Environment/Building-Permits/76t5-zqzr) dataset:
|
|
141
|
+
|
|
142
|
+
```text
|
|
143
|
+
rowgate: PASS (447 rows)
|
|
144
|
+
ok schema_matches no drift
|
|
145
|
+
ok rows_reconcile 447 valid + 7 quarantined = 454 vs 454 input
|
|
146
|
+
ok quarantine_rate 1.54%
|
|
147
|
+
ok unique:permit_num 0 duplicate keys
|
|
148
|
+
ok not_null:permit_num+permit_class 0 nulls
|
|
149
|
+
ok null_rate:applied_date 9.62% null
|
|
150
|
+
ok null_rate:zip 4.7% null
|
|
151
|
+
warn freshness:applied_date newest 2026-07-02 (110 days old)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```text
|
|
155
|
+
reason rows
|
|
156
|
+
issued_on_or_after_applied 3
|
|
157
|
+
permit_num:pattern 3
|
|
158
|
+
est_cost:max 1
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The 7 quarantined rows are real problems in the source data:
|
|
162
|
+
|
|
163
|
+
- 3 permits were issued before they were applied for.
|
|
164
|
+
- 3 sub-permit numbers like `6112685-EX-002` don't match the documented format.
|
|
165
|
+
- 1 permit lists an estimated cost of $25,000,000,000.
|
|
166
|
+
|
|
167
|
+
If sub-permits are valid for your use, widen the pattern in [`permits_contract.toml`](examples/permits_contract.toml). The freshness warning is expected, because the sample is a fixed extract.
|
|
168
|
+
|
|
169
|
+
The same approach runs at full scale in the [Seattle Permits Data Platform](https://github.com/DTheDataGuy/the-data-guys-portfolio/tree/main/data-engineering): 193,747 rows through a Polars pipeline, dbt, PySpark and Scala Spark.
|
|
170
|
+
|
|
171
|
+
## Contracts
|
|
172
|
+
|
|
173
|
+
### Columns
|
|
174
|
+
|
|
175
|
+
| Option | Meaning | Nulls |
|
|
176
|
+
|---|---|---|
|
|
177
|
+
| `dtype` | Expected Polars dtype, checked by `schema_matches` | n/a |
|
|
178
|
+
| `required` | Value must be present. Blank strings count as missing | fail |
|
|
179
|
+
| `unique` | First occurrence kept; later duplicates quarantined | skipped |
|
|
180
|
+
| `accepted` | Value must be in the list | skipped |
|
|
181
|
+
| `min`, `max` | Inclusive bounds | skipped |
|
|
182
|
+
| `pattern` | Regex the value must match | skipped |
|
|
183
|
+
|
|
184
|
+
Nulls only fail `required`, the same way SQL constraints behave. Set `allow_extra_columns=False` to treat unexpected columns as schema drift.
|
|
185
|
+
|
|
186
|
+
### Rules
|
|
187
|
+
|
|
188
|
+
Row-level rules are Polars expressions that are true for valid rows:
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
rg.Rule("issued_on_or_after_applied", pl.col("issued_date") >= pl.col("applied_date"))
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
A rule that evaluates to null passes, so a permit that hasn't been issued yet isn't rejected by an ordering rule.
|
|
195
|
+
|
|
196
|
+
### TOML
|
|
197
|
+
|
|
198
|
+
Contracts can live next to the pipeline as config, with rules written in SQL:
|
|
199
|
+
|
|
200
|
+
```toml
|
|
201
|
+
[columns.permit_num]
|
|
202
|
+
dtype = "str"
|
|
203
|
+
required = true
|
|
204
|
+
unique = true
|
|
205
|
+
pattern = '^\d{6,7}-[A-Z]{2}$'
|
|
206
|
+
|
|
207
|
+
[columns.est_cost]
|
|
208
|
+
dtype = "float"
|
|
209
|
+
min = 0
|
|
210
|
+
max = 5_000_000_000
|
|
211
|
+
|
|
212
|
+
[[rules]]
|
|
213
|
+
name = "issued_on_or_after_applied"
|
|
214
|
+
sql = "issued_date >= applied_date"
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
contract = rg.Contract.from_toml("permits_contract.toml")
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Checks
|
|
222
|
+
|
|
223
|
+
| Check | Default severity | Passes when |
|
|
224
|
+
|---|---|---|
|
|
225
|
+
| `schema_matches(contract)` | error | Every contract column is present with the right dtype |
|
|
226
|
+
| `unique(cols)` | error | No duplicate keys (one column or a composite) |
|
|
227
|
+
| `not_null(cols)` | error | No nulls in the listed columns |
|
|
228
|
+
| `null_rate(col, max_pct)` | warn | Null share is at or below the limit |
|
|
229
|
+
| `accepted_values(col, values)` | error | No values outside the set |
|
|
230
|
+
| `row_count(min_rows, max_rows)` | error | Row count is within bounds |
|
|
231
|
+
| `reconciles(result)` | error | Valid + quarantined equals input |
|
|
232
|
+
| `quarantine_rate(result, max_pct)` | error | Quarantined share is below the limit |
|
|
233
|
+
| `freshness(col, max_age_days, as_of)` | warn | Newest date is recent enough |
|
|
234
|
+
| `expr(name, condition, max_failing)` | error | At most `max_failing` rows break the condition |
|
|
235
|
+
| `custom(name, fn, expectation)` | error | `fn(df)` returns `(True, observed)` |
|
|
236
|
+
|
|
237
|
+
Every check takes `severity="error"` or `severity="warn"`. If a check itself raises, for example because a column is missing, it's recorded as a failure with the exception message, and the other checks still run.
|
|
238
|
+
|
|
239
|
+
## Reports
|
|
240
|
+
|
|
241
|
+
```python
|
|
242
|
+
report.passed # False if any error check failed
|
|
243
|
+
report.errors # failed error checks
|
|
244
|
+
report.warnings # failed warn checks
|
|
245
|
+
report.to_json("quality_report.json")
|
|
246
|
+
report.to_markdown("quality_report.md")
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
The Markdown output is a table that renders in GitHub, GitLab and most CI summaries:
|
|
250
|
+
|
|
251
|
+
| Check | Severity | Result | Observed | Expectation |
|
|
252
|
+
|---|---|---|---|---|
|
|
253
|
+
| `rows_reconcile` | error | pass | 2 valid + 3 quarantined = 5 vs 5 input | valid + quarantined = input |
|
|
254
|
+
| `quarantine_rate` | error | **FAIL** | 60.0% | < 10% of input quarantined |
|
|
255
|
+
|
|
256
|
+
## Use it in CI and orchestrators
|
|
257
|
+
|
|
258
|
+
GitHub Actions, failing the job and posting the report to the run summary:
|
|
259
|
+
|
|
260
|
+
```yaml
|
|
261
|
+
- run: pip install git+https://github.com/DTheDataGuy/rowgate@v0.1.0
|
|
262
|
+
- run: rowgate check data/orders.parquet --contract contracts/orders.toml --report-dir out/
|
|
263
|
+
- if: always()
|
|
264
|
+
run: cat out/report.md >> "$GITHUB_STEP_SUMMARY"
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
Inside an Airflow, Dagster or Prefect task, or any Python job:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
result = contract.split(df)
|
|
271
|
+
result.quarantine.write_parquet("quarantine/orders.parquet") # keep the evidence
|
|
272
|
+
rg.Gate([ck.reconciles(result), ck.quarantine_rate(result, max_pct=2)]).run(result.valid).raise_for_errors()
|
|
273
|
+
result.valid.write_parquet("silver/orders.parquet") # only reached when the gate passes
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
## Command line
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
rowgate check orders.parquet --contract contract.toml --report-dir out/
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
The command reads Parquet, CSV, NDJSON or JSON files. It prints the report and a count of quarantine reasons. With `--report-dir` it also writes `report.json`, `report.md` and `quarantine.parquet`. It exits with `1` when a blocking check fails, so it can stop a CI job or an orchestrator task. Add `--strict` to fail on warnings too.
|
|
283
|
+
|
|
284
|
+
## Design notes
|
|
285
|
+
|
|
286
|
+
- **Quarantine, don't drop.** A pipeline that deletes bad rows makes its own totals impossible to audit. rowgate keeps every row and records why it was set aside.
|
|
287
|
+
- **Every reason is listed.** A row that breaks three rules lists all three, so one fix doesn't reveal the next problem a day later.
|
|
288
|
+
- **The observed value is readable.** "3 duplicate keys" or "9.62% null" tells the on-call person what to do. A bare `False` doesn't.
|
|
289
|
+
- **Lazy-friendly expressions.** Column checks compile to Polars expressions and evaluate in a single `with_columns` pass.
|
|
290
|
+
|
|
291
|
+
## Development
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
pip install -e ".[dev]"
|
|
295
|
+
pytest -q
|
|
296
|
+
ruff check .
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
CI runs the test suite, the Seattle example and a CLI smoke test on Python 3.11, 3.12 and 3.13. It also runs the suite against Polars 1.0.0, the oldest supported version.
|
|
300
|
+
|
|
301
|
+
## License
|
|
302
|
+
|
|
303
|
+
MIT © Dei'Marlon Scisney · [The Data Guys™](https://www.thedataguys.org)
|
|
304
|
+
|
|
305
|
+
If rowgate saves you a hand-rolled validation script, a star helps other data engineers find it.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
rowgate/__init__.py,sha256=zBaUSe7XYJDtBusZKaAk9oYvWjOYlX6jkSECqvdqV6I,417
|
|
2
|
+
rowgate/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
+
rowgate/checks.py,sha256=MbWEKXOH_T-Lk0zk2c70pP3M1uXgU2BhSQ1W7x0q2DY,5432
|
|
4
|
+
rowgate/cli.py,sha256=QEIKFHGGEnnRqLOLGQZPueXCcsbWVSBmSyqSQAPirKk,2554
|
|
5
|
+
rowgate/contract.py,sha256=_Horj2gWR13L14wyxJI31oBsxwxq-ogAqKIKP85SSos,7670
|
|
6
|
+
rowgate/gate.py,sha256=BugMoIyBkHbDeMIhaXgLD_0tGyDGTTDv6eBqgyICOck,4390
|
|
7
|
+
rowgate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
rowgate-0.1.0.dist-info/METADATA,sha256=IE0F5Zz-e1y1xiYYYE8B0pi4Dw_TxVQr7VAjk5N6BTw,13874
|
|
9
|
+
rowgate-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
rowgate-0.1.0.dist-info/entry_points.txt,sha256=Wph4scCAc9hZSvoMiHzuJalOVVaE-2cBMV1PQPchHmM,45
|
|
11
|
+
rowgate-0.1.0.dist-info/licenses/LICENSE,sha256=mEPEMRah4ptY3kgnGZf_0IcoWBWv-CIeHR4Qj_jFwsk,1090
|
|
12
|
+
rowgate-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dei'Marlon Scisney, The Data Guys
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|