duckcheck 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- duckcheck/__init__.py +3 -0
- duckcheck/baseline.py +36 -0
- duckcheck/cli.py +114 -0
- duckcheck/runner.py +352 -0
- duckcheck/spec.py +26 -0
- duckcheck-0.4.0.dist-info/METADATA +145 -0
- duckcheck-0.4.0.dist-info/RECORD +10 -0
- duckcheck-0.4.0.dist-info/WHEEL +4 -0
- duckcheck-0.4.0.dist-info/entry_points.txt +2 -0
- duckcheck-0.4.0.dist-info/licenses/LICENSE +21 -0
duckcheck/__init__.py
ADDED
duckcheck/baseline.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""SQLite baseline store for row-count delta checks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class BaselineStore:
|
|
10
|
+
def __init__(self, path: Path) -> None:
|
|
11
|
+
self.path = path
|
|
12
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
13
|
+
self._conn = sqlite3.connect(str(self.path))
|
|
14
|
+
self._conn.execute(
|
|
15
|
+
"CREATE TABLE IF NOT EXISTS row_counts (check_name TEXT PRIMARY KEY, row_count INTEGER NOT NULL)"
|
|
16
|
+
)
|
|
17
|
+
self._conn.commit()
|
|
18
|
+
|
|
19
|
+
def get(self, check_name: str) -> int | None:
|
|
20
|
+
row = self._conn.execute(
|
|
21
|
+
"SELECT row_count FROM row_counts WHERE check_name = ?", (check_name,)
|
|
22
|
+
).fetchone()
|
|
23
|
+
return None if row is None else int(row[0])
|
|
24
|
+
|
|
25
|
+
def set(self, check_name: str, row_count: int) -> None:
|
|
26
|
+
self._conn.execute(
|
|
27
|
+
"""
|
|
28
|
+
INSERT INTO row_counts (check_name, row_count) VALUES (?, ?)
|
|
29
|
+
ON CONFLICT(check_name) DO UPDATE SET row_count = excluded.row_count
|
|
30
|
+
""",
|
|
31
|
+
(check_name, row_count),
|
|
32
|
+
)
|
|
33
|
+
self._conn.commit()
|
|
34
|
+
|
|
35
|
+
def close(self) -> None:
|
|
36
|
+
self._conn.close()
|
duckcheck/cli.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""CLI for duckcheck."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import NoReturn
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
import duckdb
|
|
9
|
+
import yaml
|
|
10
|
+
from pydantic import ValidationError
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
from rich.table import Table
|
|
13
|
+
|
|
14
|
+
from duckcheck import __version__
|
|
15
|
+
from duckcheck.runner import run_suite, to_junit, update_baseline
|
|
16
|
+
from duckcheck.spec import SuiteSpec
|
|
17
|
+
|
|
18
|
+
console = Console()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@click.group()
|
|
22
|
+
@click.version_option(__version__)
|
|
23
|
+
def main() -> None:
|
|
24
|
+
"""Lightweight data quality checks powered by DuckDB."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@main.command("health")
|
|
28
|
+
def health() -> None:
|
|
29
|
+
console.print(f"[green]duckcheck {__version__} OK[/green]")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _load_suite(suite_path: Path) -> SuiteSpec:
|
|
33
|
+
try:
|
|
34
|
+
raw = yaml.safe_load(suite_path.read_text(encoding="utf-8"))
|
|
35
|
+
return SuiteSpec.model_validate(raw)
|
|
36
|
+
except (yaml.YAMLError, ValidationError, ValueError, TypeError) as exc:
|
|
37
|
+
console.print(f"[red]Error:[/red] {exc}")
|
|
38
|
+
raise SystemExit(2)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _fail(exc: BaseException) -> NoReturn:
|
|
42
|
+
console.print(f"[red]Error:[/red] {exc}")
|
|
43
|
+
raise SystemExit(2)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@main.command("run")
|
|
47
|
+
@click.argument("suite_path", type=click.Path(exists=True, path_type=Path))
|
|
48
|
+
@click.option("--junit", type=click.Path(path_type=Path), default=None)
|
|
49
|
+
@click.option("--source-table", default=None, help="Override attached SQL table name")
|
|
50
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json"]), default="text")
|
|
51
|
+
def run_cmd(
|
|
52
|
+
suite_path: Path,
|
|
53
|
+
junit: Path | None,
|
|
54
|
+
source_table: str | None,
|
|
55
|
+
fmt: str,
|
|
56
|
+
) -> None:
|
|
57
|
+
suite = _load_suite(suite_path)
|
|
58
|
+
if source_table:
|
|
59
|
+
suite.source_table = source_table
|
|
60
|
+
try:
|
|
61
|
+
report = run_suite(suite, suite_dir=suite_path.parent)
|
|
62
|
+
except (FileNotFoundError, ValueError, duckdb.Error) as exc:
|
|
63
|
+
_fail(exc)
|
|
64
|
+
|
|
65
|
+
if fmt == "json":
|
|
66
|
+
import json
|
|
67
|
+
|
|
68
|
+
payload = {
|
|
69
|
+
"suite": report.suite,
|
|
70
|
+
"passed": report.passed,
|
|
71
|
+
"results": [
|
|
72
|
+
{
|
|
73
|
+
"name": r.name,
|
|
74
|
+
"passed": r.passed,
|
|
75
|
+
"message": r.message,
|
|
76
|
+
"rows_failed": r.rows_failed,
|
|
77
|
+
}
|
|
78
|
+
for r in report.results
|
|
79
|
+
],
|
|
80
|
+
}
|
|
81
|
+
console.print_json(json.dumps(payload))
|
|
82
|
+
else:
|
|
83
|
+
table = Table(title=f"Results: {report.suite}")
|
|
84
|
+
table.add_column("Check")
|
|
85
|
+
table.add_column("Status")
|
|
86
|
+
table.add_column("Message")
|
|
87
|
+
for r in report.results:
|
|
88
|
+
status = "[green]PASS[/green]" if r.passed else "[red]FAIL[/red]"
|
|
89
|
+
table.add_row(r.name, status, r.message)
|
|
90
|
+
console.print(table)
|
|
91
|
+
|
|
92
|
+
if junit:
|
|
93
|
+
junit.write_text(to_junit(report), encoding="utf-8")
|
|
94
|
+
console.print(f"Wrote JUnit report to {junit}")
|
|
95
|
+
|
|
96
|
+
if not report.passed:
|
|
97
|
+
sys.exit(1)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@main.command("baseline")
|
|
101
|
+
@click.argument("action", type=click.Choice(["update"]))
|
|
102
|
+
@click.argument("suite_path", type=click.Path(exists=True, path_type=Path))
|
|
103
|
+
def baseline_cmd(action: str, suite_path: Path) -> None:
|
|
104
|
+
"""Persist current row counts for row_count_delta checks."""
|
|
105
|
+
suite = _load_suite(suite_path)
|
|
106
|
+
try:
|
|
107
|
+
path = update_baseline(suite, suite_dir=suite_path.parent)
|
|
108
|
+
except (FileNotFoundError, ValueError, duckdb.Error) as exc:
|
|
109
|
+
_fail(exc)
|
|
110
|
+
console.print(f"[green]Updated baseline[/green] {path}")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
main()
|
duckcheck/runner.py
ADDED
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
"""Execute data quality checks via DuckDB."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from xml.etree.ElementTree import Element, SubElement, tostring
|
|
10
|
+
|
|
11
|
+
import duckdb
|
|
12
|
+
import structlog
|
|
13
|
+
|
|
14
|
+
from duckcheck.baseline import BaselineStore
|
|
15
|
+
from duckcheck.spec import CheckSpec, SuiteSpec
|
|
16
|
+
|
|
17
|
+
log = structlog.get_logger()
|
|
18
|
+
|
|
19
|
+
IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
20
|
+
AGE = re.compile(r"^(\d+)([smhd])$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class CheckResult:
|
|
25
|
+
name: str
|
|
26
|
+
passed: bool
|
|
27
|
+
message: str
|
|
28
|
+
rows_failed: int = 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class RunReport:
|
|
33
|
+
suite: str
|
|
34
|
+
results: list[CheckResult]
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def passed(self) -> bool:
|
|
38
|
+
return all(r.passed for r in self.results)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _ident(name: str) -> str:
|
|
42
|
+
if not IDENT.match(name):
|
|
43
|
+
raise ValueError(f"Invalid identifier: {name!r}")
|
|
44
|
+
return name
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _sql_literal(value: str) -> str:
|
|
48
|
+
return "'" + value.replace("'", "''") + "'"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _substitute_env(value: str) -> str:
|
|
52
|
+
def repl(match: re.Match[str]) -> str:
|
|
53
|
+
return os.environ.get(match.group(1), "")
|
|
54
|
+
|
|
55
|
+
return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", repl, value)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def run_suite(suite: SuiteSpec, suite_dir: Path | None = None) -> RunReport:
|
|
59
|
+
conn = duckdb.connect()
|
|
60
|
+
source = _substitute_env(suite.source)
|
|
61
|
+
_register_source(conn, source, suite.source_table, suite_dir)
|
|
62
|
+
baseline = _open_baseline(suite, suite_dir)
|
|
63
|
+
results: list[CheckResult] = []
|
|
64
|
+
for check in suite.checks:
|
|
65
|
+
results.append(_run_check(conn, check, baseline))
|
|
66
|
+
conn.close()
|
|
67
|
+
if baseline is not None:
|
|
68
|
+
baseline.close()
|
|
69
|
+
return RunReport(suite=suite.name, results=results)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def update_baseline(suite: SuiteSpec, suite_dir: Path | None = None) -> Path:
|
|
73
|
+
"""Persist current row counts for row_count_delta checks."""
|
|
74
|
+
conn = duckdb.connect()
|
|
75
|
+
source = _substitute_env(suite.source)
|
|
76
|
+
_register_source(conn, source, suite.source_table, suite_dir)
|
|
77
|
+
path = _baseline_path(suite, suite_dir)
|
|
78
|
+
store = BaselineStore(path)
|
|
79
|
+
count = conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0]
|
|
80
|
+
for check in suite.checks:
|
|
81
|
+
if check.type == "row_count_delta":
|
|
82
|
+
store.set(check.name, int(count))
|
|
83
|
+
store.close()
|
|
84
|
+
conn.close()
|
|
85
|
+
return path
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _register_source(
|
|
89
|
+
conn: duckdb.DuckDBPyConnection,
|
|
90
|
+
source: str,
|
|
91
|
+
source_table: str | None,
|
|
92
|
+
suite_dir: Path | None,
|
|
93
|
+
) -> None:
|
|
94
|
+
if source.startswith(("postgres://", "postgresql://")):
|
|
95
|
+
conn.execute("INSTALL postgres; LOAD postgres;")
|
|
96
|
+
conn.execute(f"ATTACH {_sql_literal(source)} AS remote (TYPE POSTGRES)")
|
|
97
|
+
table = _ident(source_table or "orders")
|
|
98
|
+
conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
|
|
99
|
+
return
|
|
100
|
+
if source.startswith("mysql://"):
|
|
101
|
+
conn.execute("INSTALL mysql; LOAD mysql;")
|
|
102
|
+
conn.execute(f"ATTACH {_sql_literal(source)} AS remote (TYPE MYSQL)")
|
|
103
|
+
table = _ident(source_table or "orders")
|
|
104
|
+
conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
|
|
105
|
+
return
|
|
106
|
+
if source.startswith("sqlite://") or source.endswith((".db", ".sqlite")):
|
|
107
|
+
db_path = source.removeprefix("sqlite://")
|
|
108
|
+
path = _resolve_file_source(db_path, suite_dir)
|
|
109
|
+
table = _ident(source_table or "source_data")
|
|
110
|
+
conn.execute(f"ATTACH {_sql_literal(str(path))} AS remote (TYPE SQLITE)")
|
|
111
|
+
conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
path = _resolve_file_source(source, suite_dir)
|
|
115
|
+
resolved = str(path)
|
|
116
|
+
quoted = _sql_literal(resolved)
|
|
117
|
+
if resolved.endswith(".csv"):
|
|
118
|
+
conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM read_csv_auto({quoted})")
|
|
119
|
+
elif resolved.endswith(".parquet"):
|
|
120
|
+
conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM read_parquet({quoted})")
|
|
121
|
+
else:
|
|
122
|
+
raise ValueError(f"Unsupported source format: {source}")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _resolve_file_source(source: str, suite_dir: Path | None) -> Path:
|
|
126
|
+
path = Path(source)
|
|
127
|
+
if path.is_absolute():
|
|
128
|
+
if not path.exists():
|
|
129
|
+
raise FileNotFoundError(f"Source not found: {path}")
|
|
130
|
+
return path
|
|
131
|
+
candidates: list[Path] = []
|
|
132
|
+
if suite_dir is not None:
|
|
133
|
+
candidates.append(suite_dir / path)
|
|
134
|
+
candidates.append(suite_dir / path.name)
|
|
135
|
+
candidates.append(Path.cwd() / path)
|
|
136
|
+
for candidate in candidates:
|
|
137
|
+
if candidate.exists():
|
|
138
|
+
return candidate
|
|
139
|
+
tried = ", ".join(str(c) for c in candidates)
|
|
140
|
+
raise FileNotFoundError(f"Source not found: {source} (tried {tried})")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _baseline_path(suite: SuiteSpec, suite_dir: Path | None) -> Path:
|
|
144
|
+
raw = suite.baseline or ".duckcheck/baseline.db"
|
|
145
|
+
path = Path(raw)
|
|
146
|
+
if path.is_absolute():
|
|
147
|
+
return path
|
|
148
|
+
root = suite_dir or Path.cwd()
|
|
149
|
+
return root / path
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _open_baseline(suite: SuiteSpec, suite_dir: Path | None) -> BaselineStore | None:
|
|
153
|
+
if not any(c.type == "row_count_delta" for c in suite.checks):
|
|
154
|
+
return None
|
|
155
|
+
return BaselineStore(_baseline_path(suite, suite_dir))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _now_sql() -> str:
|
|
159
|
+
raw = os.environ.get("DUCKCHECK_NOW")
|
|
160
|
+
if raw:
|
|
161
|
+
return f"TIMESTAMP '{raw}'"
|
|
162
|
+
return "now()"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _run_check(
|
|
166
|
+
conn: duckdb.DuckDBPyConnection,
|
|
167
|
+
check: CheckSpec,
|
|
168
|
+
baseline: BaselineStore | None = None,
|
|
169
|
+
) -> CheckResult:
|
|
170
|
+
log.info("running_check", name=check.name, type=check.type)
|
|
171
|
+
dispatch = {
|
|
172
|
+
"not_null": _check_not_null,
|
|
173
|
+
"unique": _check_unique,
|
|
174
|
+
"accepted_values": _check_accepted_values,
|
|
175
|
+
"custom_sql": _check_custom_sql,
|
|
176
|
+
"freshness": _check_freshness,
|
|
177
|
+
"row_count": _check_row_count,
|
|
178
|
+
"row_count_delta": lambda c, spec: _check_row_count_delta(c, spec, baseline),
|
|
179
|
+
}
|
|
180
|
+
handler = dispatch.get(check.type)
|
|
181
|
+
if handler is None:
|
|
182
|
+
return CheckResult(check.name, False, f"Unknown check type: {check.type}")
|
|
183
|
+
return handler(conn, check)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _check_not_null(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
187
|
+
col = _ident(check.column or "")
|
|
188
|
+
count = conn.execute(f"SELECT COUNT(*) FROM source_data WHERE {col} IS NULL").fetchone()[0]
|
|
189
|
+
passed = count == 0
|
|
190
|
+
return CheckResult(
|
|
191
|
+
check.name,
|
|
192
|
+
passed,
|
|
193
|
+
f"{count} null values in {col}" if not passed else f"{col} has no nulls",
|
|
194
|
+
rows_failed=count,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _check_unique(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
199
|
+
col = _ident(check.column or "")
|
|
200
|
+
dupes = conn.execute(
|
|
201
|
+
f"SELECT COUNT(*) - COUNT(DISTINCT {col}) FROM source_data"
|
|
202
|
+
).fetchone()[0]
|
|
203
|
+
passed = dupes == 0
|
|
204
|
+
return CheckResult(
|
|
205
|
+
check.name,
|
|
206
|
+
passed,
|
|
207
|
+
f"{dupes} duplicate values in {col}" if not passed else f"{col} is unique",
|
|
208
|
+
rows_failed=dupes,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _check_accepted_values(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
213
|
+
col = _ident(check.column or "")
|
|
214
|
+
allowed = ", ".join(f"'{v}'" for v in check.values)
|
|
215
|
+
bad = conn.execute(
|
|
216
|
+
f"SELECT COUNT(*) FROM source_data WHERE {col} NOT IN ({allowed})"
|
|
217
|
+
).fetchone()[0]
|
|
218
|
+
passed = bad == 0
|
|
219
|
+
return CheckResult(
|
|
220
|
+
check.name,
|
|
221
|
+
passed,
|
|
222
|
+
f"{bad} rows with invalid {col}" if not passed else f"{col} values accepted",
|
|
223
|
+
rows_failed=bad,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _eval_expect(count: int, expect: str | int | None) -> tuple[bool, str]:
|
|
228
|
+
"""Return (passed, human message) for custom_sql row counts."""
|
|
229
|
+
raw = "0" if expect is None else str(expect).strip()
|
|
230
|
+
if raw.isdigit() or (raw.startswith("=") and raw[1:].strip().isdigit()):
|
|
231
|
+
want = int(raw[1:].strip() if raw.startswith("=") else raw)
|
|
232
|
+
passed = count == want
|
|
233
|
+
return passed, f"got {count} rows, expect ={want}"
|
|
234
|
+
if raw.startswith(">") and raw[1:].strip().isdigit():
|
|
235
|
+
want = int(raw[1:].strip())
|
|
236
|
+
passed = count > want
|
|
237
|
+
return passed, f"got {count} rows, expect >{want}"
|
|
238
|
+
if raw.startswith("<") and raw[1:].strip().isdigit():
|
|
239
|
+
want = int(raw[1:].strip())
|
|
240
|
+
passed = count < want
|
|
241
|
+
return passed, f"got {count} rows, expect <{want}"
|
|
242
|
+
if raw.startswith(">=") and raw[2:].strip().isdigit():
|
|
243
|
+
want = int(raw[2:].strip())
|
|
244
|
+
passed = count >= want
|
|
245
|
+
return passed, f"got {count} rows, expect >={want}"
|
|
246
|
+
if raw.startswith("<=") and raw[2:].strip().isdigit():
|
|
247
|
+
want = int(raw[2:].strip())
|
|
248
|
+
passed = count <= want
|
|
249
|
+
return passed, f"got {count} rows, expect <={want}"
|
|
250
|
+
raise ValueError(f"Invalid expect {expect!r}. Use 0, =N, >N, <N, >=N, or <=N.")
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _check_custom_sql(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
254
|
+
sql = (check.sql or "").strip()
|
|
255
|
+
if not sql.lower().startswith("select"):
|
|
256
|
+
return CheckResult(check.name, False, "custom_sql must be a SELECT statement.")
|
|
257
|
+
rows = conn.execute(sql).fetchall()
|
|
258
|
+
count = len(rows)
|
|
259
|
+
try:
|
|
260
|
+
passed, detail = _eval_expect(count, check.expect)
|
|
261
|
+
except ValueError as exc:
|
|
262
|
+
return CheckResult(check.name, False, str(exc))
|
|
263
|
+
return CheckResult(
|
|
264
|
+
check.name,
|
|
265
|
+
passed,
|
|
266
|
+
detail if not passed else f"custom SQL ok ({detail})",
|
|
267
|
+
rows_failed=0 if passed else count,
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _parse_age(spec: str) -> str:
|
|
272
|
+
match = AGE.match(spec)
|
|
273
|
+
if not match:
|
|
274
|
+
raise ValueError(f"Invalid max_age '{spec}'. Use Ns/Nm/Nh/Nd.")
|
|
275
|
+
value, unit = match.groups()
|
|
276
|
+
mapping = {"s": "SECOND", "m": "MINUTE", "h": "HOUR", "d": "DAY"}
|
|
277
|
+
return f"INTERVAL {int(value)} {mapping[unit]}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _check_freshness(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
281
|
+
col = _ident(check.column or "")
|
|
282
|
+
interval = _parse_age(check.max_age or "24h")
|
|
283
|
+
stale = conn.execute(
|
|
284
|
+
f"SELECT COUNT(*) FROM source_data WHERE {col} < {_now_sql()} - {interval}"
|
|
285
|
+
).fetchone()[0]
|
|
286
|
+
passed = stale == 0
|
|
287
|
+
return CheckResult(
|
|
288
|
+
check.name,
|
|
289
|
+
passed,
|
|
290
|
+
f"{stale} stale rows in {col}" if not passed else f"{col} is fresh",
|
|
291
|
+
rows_failed=stale,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _check_row_count(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
|
|
296
|
+
count = conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0]
|
|
297
|
+
too_few = check.min_rows is not None and count < check.min_rows
|
|
298
|
+
too_many = check.max_rows is not None and count > check.max_rows
|
|
299
|
+
passed = not too_few and not too_many
|
|
300
|
+
return CheckResult(
|
|
301
|
+
check.name,
|
|
302
|
+
passed,
|
|
303
|
+
f"row count {count} outside [{check.min_rows}, {check.max_rows}]"
|
|
304
|
+
if not passed
|
|
305
|
+
else f"row count {count} within bounds",
|
|
306
|
+
rows_failed=0 if passed else 1,
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _check_row_count_delta(
|
|
311
|
+
conn: duckdb.DuckDBPyConnection,
|
|
312
|
+
check: CheckSpec,
|
|
313
|
+
baseline: BaselineStore | None,
|
|
314
|
+
) -> CheckResult:
|
|
315
|
+
count = int(conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0])
|
|
316
|
+
if baseline is None:
|
|
317
|
+
return CheckResult(check.name, False, "row_count_delta requires a baseline store")
|
|
318
|
+
previous = baseline.get(check.name)
|
|
319
|
+
if previous is None:
|
|
320
|
+
return CheckResult(
|
|
321
|
+
check.name,
|
|
322
|
+
False,
|
|
323
|
+
f"no baseline for {check.name}; run duckcheck baseline update",
|
|
324
|
+
)
|
|
325
|
+
if previous == 0:
|
|
326
|
+
drift = 0.0 if count == 0 else 100.0
|
|
327
|
+
else:
|
|
328
|
+
drift = abs(count - previous) / previous * 100.0
|
|
329
|
+
tol = check.tolerance_pct if check.tolerance_pct is not None else 10.0
|
|
330
|
+
passed = drift <= tol
|
|
331
|
+
return CheckResult(
|
|
332
|
+
check.name,
|
|
333
|
+
passed,
|
|
334
|
+
f"row count {count} vs baseline {previous} ({drift:.1f}% drift, tol {tol}%)"
|
|
335
|
+
if not passed
|
|
336
|
+
else f"row count {count} within {tol}% of baseline {previous}",
|
|
337
|
+
rows_failed=0 if passed else 1,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def to_junit(report: RunReport) -> str:
|
|
342
|
+
"""Serialize results as JUnit XML for CI dashboards."""
|
|
343
|
+
suite = Element("testsuite", name=report.suite, tests=str(len(report.results)))
|
|
344
|
+
failures = 0
|
|
345
|
+
for result in report.results:
|
|
346
|
+
case = SubElement(suite, "testcase", name=result.name, classname=report.suite)
|
|
347
|
+
if not result.passed:
|
|
348
|
+
failures += 1
|
|
349
|
+
failure = SubElement(case, "failure", message=result.message)
|
|
350
|
+
failure.text = result.message
|
|
351
|
+
suite.set("failures", str(failures))
|
|
352
|
+
return tostring(suite, encoding="unicode")
|
duckcheck/spec.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Check specification models."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CheckSpec(BaseModel):
|
|
7
|
+
name: str
|
|
8
|
+
type: str
|
|
9
|
+
table: str = "source_data"
|
|
10
|
+
column: str | None = None
|
|
11
|
+
values: list[str] = Field(default_factory=list)
|
|
12
|
+
sql: str | None = None
|
|
13
|
+
# custom_sql: pass when row count matches expect ("0", "=3", ">0", "<10")
|
|
14
|
+
expect: str | int | None = "0"
|
|
15
|
+
max_age: str | None = None
|
|
16
|
+
min_rows: int | None = None
|
|
17
|
+
max_rows: int | None = None
|
|
18
|
+
tolerance_pct: float | None = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SuiteSpec(BaseModel):
|
|
22
|
+
name: str
|
|
23
|
+
source: str
|
|
24
|
+
source_table: str | None = None
|
|
25
|
+
baseline: str | None = None
|
|
26
|
+
checks: list[CheckSpec] = Field(default_factory=list)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: duckcheck
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Lightweight data quality checks powered by DuckDB
|
|
5
|
+
Project-URL: Homepage, https://github.com/yashshah9/duckcheck
|
|
6
|
+
Project-URL: Repository, https://github.com/yashshah9/duckcheck
|
|
7
|
+
Project-URL: Issues, https://github.com/yashshah9/duckcheck/issues
|
|
8
|
+
Author-email: Yash Shah <yash376351@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Requires-Dist: click>=8.1
|
|
13
|
+
Requires-Dist: duckdb>=1.0
|
|
14
|
+
Requires-Dist: pydantic>=2.6
|
|
15
|
+
Requires-Dist: pyyaml>=6.0
|
|
16
|
+
Requires-Dist: rich>=13.7
|
|
17
|
+
Requires-Dist: structlog>=24.1
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: mypy>=1.9; extra == 'dev'
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
22
|
+
Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# duckcheck
|
|
26
|
+
|
|
27
|
+
Lightweight data quality checks powered by **DuckDB** — the anti–Great Expectations for teams who want `pip install`, one YAML file, and one command.
|
|
28
|
+
|
|
29
|
+
[](LICENSE)
|
|
30
|
+
[](https://www.python.org/downloads/)
|
|
31
|
+
[](https://github.com/yashshah9/duckcheck/actions/workflows/ci.yml)
|
|
32
|
+
|
|
33
|
+
> **Status:** v0.4 — CSV/Parquet/SQLite sources, custom SQL with `expect` operators, freshness, baselines, JUnit, and `--format json`.
|
|
34
|
+
|
|
35
|
+
## 60-second try
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
docker compose run --rm run-example # duckcheck run examples/clean.yaml
|
|
39
|
+
docker compose run --rm test # pytest
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Why this vs alternatives
|
|
43
|
+
|
|
44
|
+
| Approach | Strength | Gap |
|
|
45
|
+
|----------|----------|-----|
|
|
46
|
+
| **duckcheck** | One YAML + DuckDB, local files, CI-friendly | Not a full observability platform |
|
|
47
|
+
| Great Expectations | Rich ecosystem | Heavyweight setup for simple column checks |
|
|
48
|
+
| Soda Core | Familiar check DSL | Cloud-oriented workflow |
|
|
49
|
+
| Ad-hoc SQL in CI | Zero new tools | No standard report / JUnit / baselines |
|
|
50
|
+
|
|
51
|
+
## Problem
|
|
52
|
+
|
|
53
|
+
Data teams need to assert column quality in CI, but Great Expectations is heavyweight and Soda Core funnels to cloud. Ad-hoc SQL checks have no reporting standard.
|
|
54
|
+
|
|
55
|
+
## Key features (v0.4)
|
|
56
|
+
|
|
57
|
+
- YAML check definitions
|
|
58
|
+
- DuckDB scans CSV, Parquet, and SQLite locally — no server
|
|
59
|
+
- Checks: `not_null`, `unique`, `accepted_values`, `custom_sql`, `freshness`, `row_count`, `row_count_delta`
|
|
60
|
+
- `custom_sql` `expect` operators: `0`, `=N`, `>N`, `<N`, `>=N`, `<=N` (default `0`)
|
|
61
|
+
- `--format json` and `--junit` for CI dashboards
|
|
62
|
+
- `${ENV}` in source URIs; `--source-table` for SQL ATTACH
|
|
63
|
+
|
|
64
|
+
## Architecture
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
duckcheck run checks.yaml
|
|
68
|
+
└── SuiteSpec (Pydantic)
|
|
69
|
+
└── DuckDB in-process
|
|
70
|
+
└── source_data view from CSV/Parquet
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
| Component | Technology | Why |
|
|
74
|
+
|-----------|------------|-----|
|
|
75
|
+
| Engine | DuckDB | Single dependency, scans files + SQL databases |
|
|
76
|
+
| CLI | Click + Rich | Simple, good terminal UX |
|
|
77
|
+
| Spec | YAML + Pydantic | Version-controllable checks |
|
|
78
|
+
|
|
79
|
+
## Installation
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install duckcheck
|
|
83
|
+
pip install -e ".[dev]"
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Usage
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
duckcheck health
|
|
90
|
+
duckcheck run examples/clean.yaml
|
|
91
|
+
duckcheck run examples/checks.yaml # fixture with known failures
|
|
92
|
+
duckcheck run examples/clean.yaml --junit /tmp/duckcheck.xml
|
|
93
|
+
duckcheck run examples/clean.yaml --format json
|
|
94
|
+
duckcheck baseline update examples/clean.yaml
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Example `checks.yaml`:
|
|
98
|
+
|
|
99
|
+
```yaml
|
|
100
|
+
name: sample-suite
|
|
101
|
+
source: examples/sample.csv
|
|
102
|
+
checks:
|
|
103
|
+
- name: id_not_null
|
|
104
|
+
type: not_null
|
|
105
|
+
column: id
|
|
106
|
+
- name: status_values
|
|
107
|
+
type: accepted_values
|
|
108
|
+
column: status
|
|
109
|
+
values: [active, inactive]
|
|
110
|
+
- name: three_active
|
|
111
|
+
type: custom_sql
|
|
112
|
+
sql: "SELECT * FROM source_data WHERE status = 'active'"
|
|
113
|
+
expect: "=3"
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Docker
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
docker compose run --rm test
|
|
120
|
+
docker compose run --rm run-example
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Running tests
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
pytest tests/ -v
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Roadmap
|
|
130
|
+
|
|
131
|
+
- [x] Freshness + row_count + custom_sql + JUnit
|
|
132
|
+
- [x] Row-count baseline delta store (`duckcheck baseline update`)
|
|
133
|
+
- [x] custom_sql `expect` operators + `--format json`
|
|
134
|
+
- [ ] Live Postgres/MySQL ATTACH integration tests
|
|
135
|
+
- [ ] Airflow/Dagster operators
|
|
136
|
+
|
|
137
|
+
## License
|
|
138
|
+
|
|
139
|
+
MIT
|
|
140
|
+
|
|
141
|
+
## Known limitations (v0.4)
|
|
142
|
+
|
|
143
|
+
- Postgres/MySQL ATTACH is stubbed (`INSTALL/LOAD`) — no live DB in CI yet
|
|
144
|
+
- `examples/checks.yaml` is a failing fixture; `examples/clean.yaml` is the happy path
|
|
145
|
+
- Checks still run against a `source_data` view
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
duckcheck/__init__.py,sha256=kKKgBq_q4-vqb8QA_WvhdqR0QLCACuaQKGcRue2EnI0,65
|
|
2
|
+
duckcheck/baseline.py,sha256=PS5TM_h6bkyE9frgQAPl6ULkQWaoPpc4oYtTeTloBQE,1163
|
|
3
|
+
duckcheck/cli.py,sha256=Fd_-t8apCZn7KyPLysyawfB1KQgSME1f00GgAOELous,3389
|
|
4
|
+
duckcheck/runner.py,sha256=SHKhOjBVtxpkKN50zB5yPYkh4EomwEDen-8-NIPb8aY,12353
|
|
5
|
+
duckcheck/spec.py,sha256=wZx-Vp5mEkFfIuqFIsDZxkt67e-Uozm1TBUTD4AkRX8,704
|
|
6
|
+
duckcheck-0.4.0.dist-info/METADATA,sha256=apb-sNwFgufQXkTKevfzKIwWUx5ywlBxsLqOfAmmp1o,4457
|
|
7
|
+
duckcheck-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
8
|
+
duckcheck-0.4.0.dist-info/entry_points.txt,sha256=pBnGIP-XwvpLepF_R4kij6R0cxneIL2xlmnLwICIJLs,49
|
|
9
|
+
duckcheck-0.4.0.dist-info/licenses/LICENSE,sha256=AvzqSNURip5YZfXN3-QN_nLlFeC8ob_qbLijyMrFP8w,1079
|
|
10
|
+
duckcheck-0.4.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 duckcheck contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|