duckcheck 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
duckcheck/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """DuckDB-powered data quality checks."""
2
+
3
+ __version__ = "0.4.0"
duckcheck/baseline.py ADDED
@@ -0,0 +1,36 @@
1
+ """SQLite baseline store for row-count delta checks."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sqlite3
6
+ from pathlib import Path
7
+
8
+
9
+ class BaselineStore:
10
+ def __init__(self, path: Path) -> None:
11
+ self.path = path
12
+ self.path.parent.mkdir(parents=True, exist_ok=True)
13
+ self._conn = sqlite3.connect(str(self.path))
14
+ self._conn.execute(
15
+ "CREATE TABLE IF NOT EXISTS row_counts (check_name TEXT PRIMARY KEY, row_count INTEGER NOT NULL)"
16
+ )
17
+ self._conn.commit()
18
+
19
+ def get(self, check_name: str) -> int | None:
20
+ row = self._conn.execute(
21
+ "SELECT row_count FROM row_counts WHERE check_name = ?", (check_name,)
22
+ ).fetchone()
23
+ return None if row is None else int(row[0])
24
+
25
+ def set(self, check_name: str, row_count: int) -> None:
26
+ self._conn.execute(
27
+ """
28
+ INSERT INTO row_counts (check_name, row_count) VALUES (?, ?)
29
+ ON CONFLICT(check_name) DO UPDATE SET row_count = excluded.row_count
30
+ """,
31
+ (check_name, row_count),
32
+ )
33
+ self._conn.commit()
34
+
35
+ def close(self) -> None:
36
+ self._conn.close()
duckcheck/cli.py ADDED
@@ -0,0 +1,114 @@
1
+ """CLI for duckcheck."""
2
+
3
+ import sys
4
+ from pathlib import Path
5
+ from typing import NoReturn
6
+
7
+ import click
8
+ import duckdb
9
+ import yaml
10
+ from pydantic import ValidationError
11
+ from rich.console import Console
12
+ from rich.table import Table
13
+
14
+ from duckcheck import __version__
15
+ from duckcheck.runner import run_suite, to_junit, update_baseline
16
+ from duckcheck.spec import SuiteSpec
17
+
18
+ console = Console()
19
+
20
+
21
+ @click.group()
22
+ @click.version_option(__version__)
23
+ def main() -> None:
24
+ """Lightweight data quality checks powered by DuckDB."""
25
+
26
+
27
+ @main.command("health")
28
+ def health() -> None:
29
+ console.print(f"[green]duckcheck {__version__} OK[/green]")
30
+
31
+
32
+ def _load_suite(suite_path: Path) -> SuiteSpec:
33
+ try:
34
+ raw = yaml.safe_load(suite_path.read_text(encoding="utf-8"))
35
+ return SuiteSpec.model_validate(raw)
36
+ except (yaml.YAMLError, ValidationError, ValueError, TypeError) as exc:
37
+ console.print(f"[red]Error:[/red] {exc}")
38
+ raise SystemExit(2)
39
+
40
+
41
+ def _fail(exc: BaseException) -> NoReturn:
42
+ console.print(f"[red]Error:[/red] {exc}")
43
+ raise SystemExit(2)
44
+
45
+
46
+ @main.command("run")
47
+ @click.argument("suite_path", type=click.Path(exists=True, path_type=Path))
48
+ @click.option("--junit", type=click.Path(path_type=Path), default=None)
49
+ @click.option("--source-table", default=None, help="Override attached SQL table name")
50
+ @click.option("--format", "fmt", type=click.Choice(["text", "json"]), default="text")
51
+ def run_cmd(
52
+ suite_path: Path,
53
+ junit: Path | None,
54
+ source_table: str | None,
55
+ fmt: str,
56
+ ) -> None:
57
+ suite = _load_suite(suite_path)
58
+ if source_table:
59
+ suite.source_table = source_table
60
+ try:
61
+ report = run_suite(suite, suite_dir=suite_path.parent)
62
+ except (FileNotFoundError, ValueError, duckdb.Error) as exc:
63
+ _fail(exc)
64
+
65
+ if fmt == "json":
66
+ import json
67
+
68
+ payload = {
69
+ "suite": report.suite,
70
+ "passed": report.passed,
71
+ "results": [
72
+ {
73
+ "name": r.name,
74
+ "passed": r.passed,
75
+ "message": r.message,
76
+ "rows_failed": r.rows_failed,
77
+ }
78
+ for r in report.results
79
+ ],
80
+ }
81
+ console.print_json(json.dumps(payload))
82
+ else:
83
+ table = Table(title=f"Results: {report.suite}")
84
+ table.add_column("Check")
85
+ table.add_column("Status")
86
+ table.add_column("Message")
87
+ for r in report.results:
88
+ status = "[green]PASS[/green]" if r.passed else "[red]FAIL[/red]"
89
+ table.add_row(r.name, status, r.message)
90
+ console.print(table)
91
+
92
+ if junit:
93
+ junit.write_text(to_junit(report), encoding="utf-8")
94
+ console.print(f"Wrote JUnit report to {junit}")
95
+
96
+ if not report.passed:
97
+ sys.exit(1)
98
+
99
+
100
+ @main.command("baseline")
101
+ @click.argument("action", type=click.Choice(["update"]))
102
+ @click.argument("suite_path", type=click.Path(exists=True, path_type=Path))
103
+ def baseline_cmd(action: str, suite_path: Path) -> None:
104
+ """Persist current row counts for row_count_delta checks."""
105
+ suite = _load_suite(suite_path)
106
+ try:
107
+ path = update_baseline(suite, suite_dir=suite_path.parent)
108
+ except (FileNotFoundError, ValueError, duckdb.Error) as exc:
109
+ _fail(exc)
110
+ console.print(f"[green]Updated baseline[/green] {path}")
111
+
112
+
113
+ if __name__ == "__main__":
114
+ main()
duckcheck/runner.py ADDED
@@ -0,0 +1,352 @@
1
+ """Execute data quality checks via DuckDB."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import re
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+ from xml.etree.ElementTree import Element, SubElement, tostring
10
+
11
+ import duckdb
12
+ import structlog
13
+
14
+ from duckcheck.baseline import BaselineStore
15
+ from duckcheck.spec import CheckSpec, SuiteSpec
16
+
17
+ log = structlog.get_logger()
18
+
19
+ IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
20
+ AGE = re.compile(r"^(\d+)([smhd])$")
21
+
22
+
23
+ @dataclass
24
+ class CheckResult:
25
+ name: str
26
+ passed: bool
27
+ message: str
28
+ rows_failed: int = 0
29
+
30
+
31
+ @dataclass
32
+ class RunReport:
33
+ suite: str
34
+ results: list[CheckResult]
35
+
36
+ @property
37
+ def passed(self) -> bool:
38
+ return all(r.passed for r in self.results)
39
+
40
+
41
+ def _ident(name: str) -> str:
42
+ if not IDENT.match(name):
43
+ raise ValueError(f"Invalid identifier: {name!r}")
44
+ return name
45
+
46
+
47
+ def _sql_literal(value: str) -> str:
48
+ return "'" + value.replace("'", "''") + "'"
49
+
50
+
51
+ def _substitute_env(value: str) -> str:
52
+ def repl(match: re.Match[str]) -> str:
53
+ return os.environ.get(match.group(1), "")
54
+
55
+ return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", repl, value)
56
+
57
+
58
+ def run_suite(suite: SuiteSpec, suite_dir: Path | None = None) -> RunReport:
59
+ conn = duckdb.connect()
60
+ source = _substitute_env(suite.source)
61
+ _register_source(conn, source, suite.source_table, suite_dir)
62
+ baseline = _open_baseline(suite, suite_dir)
63
+ results: list[CheckResult] = []
64
+ for check in suite.checks:
65
+ results.append(_run_check(conn, check, baseline))
66
+ conn.close()
67
+ if baseline is not None:
68
+ baseline.close()
69
+ return RunReport(suite=suite.name, results=results)
70
+
71
+
72
+ def update_baseline(suite: SuiteSpec, suite_dir: Path | None = None) -> Path:
73
+ """Persist current row counts for row_count_delta checks."""
74
+ conn = duckdb.connect()
75
+ source = _substitute_env(suite.source)
76
+ _register_source(conn, source, suite.source_table, suite_dir)
77
+ path = _baseline_path(suite, suite_dir)
78
+ store = BaselineStore(path)
79
+ count = conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0]
80
+ for check in suite.checks:
81
+ if check.type == "row_count_delta":
82
+ store.set(check.name, int(count))
83
+ store.close()
84
+ conn.close()
85
+ return path
86
+
87
+
88
+ def _register_source(
89
+ conn: duckdb.DuckDBPyConnection,
90
+ source: str,
91
+ source_table: str | None,
92
+ suite_dir: Path | None,
93
+ ) -> None:
94
+ if source.startswith(("postgres://", "postgresql://")):
95
+ conn.execute("INSTALL postgres; LOAD postgres;")
96
+ conn.execute(f"ATTACH {_sql_literal(source)} AS remote (TYPE POSTGRES)")
97
+ table = _ident(source_table or "orders")
98
+ conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
99
+ return
100
+ if source.startswith("mysql://"):
101
+ conn.execute("INSTALL mysql; LOAD mysql;")
102
+ conn.execute(f"ATTACH {_sql_literal(source)} AS remote (TYPE MYSQL)")
103
+ table = _ident(source_table or "orders")
104
+ conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
105
+ return
106
+ if source.startswith("sqlite://") or source.endswith((".db", ".sqlite")):
107
+ db_path = source.removeprefix("sqlite://")
108
+ path = _resolve_file_source(db_path, suite_dir)
109
+ table = _ident(source_table or "source_data")
110
+ conn.execute(f"ATTACH {_sql_literal(str(path))} AS remote (TYPE SQLITE)")
111
+ conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM remote.{table}")
112
+ return
113
+
114
+ path = _resolve_file_source(source, suite_dir)
115
+ resolved = str(path)
116
+ quoted = _sql_literal(resolved)
117
+ if resolved.endswith(".csv"):
118
+ conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM read_csv_auto({quoted})")
119
+ elif resolved.endswith(".parquet"):
120
+ conn.execute(f"CREATE OR REPLACE VIEW source_data AS SELECT * FROM read_parquet({quoted})")
121
+ else:
122
+ raise ValueError(f"Unsupported source format: {source}")
123
+
124
+
125
+ def _resolve_file_source(source: str, suite_dir: Path | None) -> Path:
126
+ path = Path(source)
127
+ if path.is_absolute():
128
+ if not path.exists():
129
+ raise FileNotFoundError(f"Source not found: {path}")
130
+ return path
131
+ candidates: list[Path] = []
132
+ if suite_dir is not None:
133
+ candidates.append(suite_dir / path)
134
+ candidates.append(suite_dir / path.name)
135
+ candidates.append(Path.cwd() / path)
136
+ for candidate in candidates:
137
+ if candidate.exists():
138
+ return candidate
139
+ tried = ", ".join(str(c) for c in candidates)
140
+ raise FileNotFoundError(f"Source not found: {source} (tried {tried})")
141
+
142
+
143
+ def _baseline_path(suite: SuiteSpec, suite_dir: Path | None) -> Path:
144
+ raw = suite.baseline or ".duckcheck/baseline.db"
145
+ path = Path(raw)
146
+ if path.is_absolute():
147
+ return path
148
+ root = suite_dir or Path.cwd()
149
+ return root / path
150
+
151
+
152
+ def _open_baseline(suite: SuiteSpec, suite_dir: Path | None) -> BaselineStore | None:
153
+ if not any(c.type == "row_count_delta" for c in suite.checks):
154
+ return None
155
+ return BaselineStore(_baseline_path(suite, suite_dir))
156
+
157
+
158
+ def _now_sql() -> str:
159
+ raw = os.environ.get("DUCKCHECK_NOW")
160
+ if raw:
161
+ return f"TIMESTAMP '{raw}'"
162
+ return "now()"
163
+
164
+
165
+ def _run_check(
166
+ conn: duckdb.DuckDBPyConnection,
167
+ check: CheckSpec,
168
+ baseline: BaselineStore | None = None,
169
+ ) -> CheckResult:
170
+ log.info("running_check", name=check.name, type=check.type)
171
+ dispatch = {
172
+ "not_null": _check_not_null,
173
+ "unique": _check_unique,
174
+ "accepted_values": _check_accepted_values,
175
+ "custom_sql": _check_custom_sql,
176
+ "freshness": _check_freshness,
177
+ "row_count": _check_row_count,
178
+ "row_count_delta": lambda c, spec: _check_row_count_delta(c, spec, baseline),
179
+ }
180
+ handler = dispatch.get(check.type)
181
+ if handler is None:
182
+ return CheckResult(check.name, False, f"Unknown check type: {check.type}")
183
+ return handler(conn, check)
184
+
185
+
186
+ def _check_not_null(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
187
+ col = _ident(check.column or "")
188
+ count = conn.execute(f"SELECT COUNT(*) FROM source_data WHERE {col} IS NULL").fetchone()[0]
189
+ passed = count == 0
190
+ return CheckResult(
191
+ check.name,
192
+ passed,
193
+ f"{count} null values in {col}" if not passed else f"{col} has no nulls",
194
+ rows_failed=count,
195
+ )
196
+
197
+
198
+ def _check_unique(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
199
+ col = _ident(check.column or "")
200
+ dupes = conn.execute(
201
+ f"SELECT COUNT(*) - COUNT(DISTINCT {col}) FROM source_data"
202
+ ).fetchone()[0]
203
+ passed = dupes == 0
204
+ return CheckResult(
205
+ check.name,
206
+ passed,
207
+ f"{dupes} duplicate values in {col}" if not passed else f"{col} is unique",
208
+ rows_failed=dupes,
209
+ )
210
+
211
+
212
+ def _check_accepted_values(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
213
+ col = _ident(check.column or "")
214
+ allowed = ", ".join(f"'{v}'" for v in check.values)
215
+ bad = conn.execute(
216
+ f"SELECT COUNT(*) FROM source_data WHERE {col} NOT IN ({allowed})"
217
+ ).fetchone()[0]
218
+ passed = bad == 0
219
+ return CheckResult(
220
+ check.name,
221
+ passed,
222
+ f"{bad} rows with invalid {col}" if not passed else f"{col} values accepted",
223
+ rows_failed=bad,
224
+ )
225
+
226
+
227
+ def _eval_expect(count: int, expect: str | int | None) -> tuple[bool, str]:
228
+ """Return (passed, human message) for custom_sql row counts."""
229
+ raw = "0" if expect is None else str(expect).strip()
230
+ if raw.isdigit() or (raw.startswith("=") and raw[1:].strip().isdigit()):
231
+ want = int(raw[1:].strip() if raw.startswith("=") else raw)
232
+ passed = count == want
233
+ return passed, f"got {count} rows, expect ={want}"
234
+ if raw.startswith(">") and raw[1:].strip().isdigit():
235
+ want = int(raw[1:].strip())
236
+ passed = count > want
237
+ return passed, f"got {count} rows, expect >{want}"
238
+ if raw.startswith("<") and raw[1:].strip().isdigit():
239
+ want = int(raw[1:].strip())
240
+ passed = count < want
241
+ return passed, f"got {count} rows, expect <{want}"
242
+ if raw.startswith(">=") and raw[2:].strip().isdigit():
243
+ want = int(raw[2:].strip())
244
+ passed = count >= want
245
+ return passed, f"got {count} rows, expect >={want}"
246
+ if raw.startswith("<=") and raw[2:].strip().isdigit():
247
+ want = int(raw[2:].strip())
248
+ passed = count <= want
249
+ return passed, f"got {count} rows, expect <={want}"
250
+ raise ValueError(f"Invalid expect {expect!r}. Use 0, =N, >N, <N, >=N, or <=N.")
251
+
252
+
253
+ def _check_custom_sql(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
254
+ sql = (check.sql or "").strip()
255
+ if not sql.lower().startswith("select"):
256
+ return CheckResult(check.name, False, "custom_sql must be a SELECT statement.")
257
+ rows = conn.execute(sql).fetchall()
258
+ count = len(rows)
259
+ try:
260
+ passed, detail = _eval_expect(count, check.expect)
261
+ except ValueError as exc:
262
+ return CheckResult(check.name, False, str(exc))
263
+ return CheckResult(
264
+ check.name,
265
+ passed,
266
+ detail if not passed else f"custom SQL ok ({detail})",
267
+ rows_failed=0 if passed else count,
268
+ )
269
+
270
+
271
+ def _parse_age(spec: str) -> str:
272
+ match = AGE.match(spec)
273
+ if not match:
274
+ raise ValueError(f"Invalid max_age '{spec}'. Use Ns/Nm/Nh/Nd.")
275
+ value, unit = match.groups()
276
+ mapping = {"s": "SECOND", "m": "MINUTE", "h": "HOUR", "d": "DAY"}
277
+ return f"INTERVAL {int(value)} {mapping[unit]}"
278
+
279
+
280
+ def _check_freshness(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
281
+ col = _ident(check.column or "")
282
+ interval = _parse_age(check.max_age or "24h")
283
+ stale = conn.execute(
284
+ f"SELECT COUNT(*) FROM source_data WHERE {col} < {_now_sql()} - {interval}"
285
+ ).fetchone()[0]
286
+ passed = stale == 0
287
+ return CheckResult(
288
+ check.name,
289
+ passed,
290
+ f"{stale} stale rows in {col}" if not passed else f"{col} is fresh",
291
+ rows_failed=stale,
292
+ )
293
+
294
+
295
+ def _check_row_count(conn: duckdb.DuckDBPyConnection, check: CheckSpec) -> CheckResult:
296
+ count = conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0]
297
+ too_few = check.min_rows is not None and count < check.min_rows
298
+ too_many = check.max_rows is not None and count > check.max_rows
299
+ passed = not too_few and not too_many
300
+ return CheckResult(
301
+ check.name,
302
+ passed,
303
+ f"row count {count} outside [{check.min_rows}, {check.max_rows}]"
304
+ if not passed
305
+ else f"row count {count} within bounds",
306
+ rows_failed=0 if passed else 1,
307
+ )
308
+
309
+
310
+ def _check_row_count_delta(
311
+ conn: duckdb.DuckDBPyConnection,
312
+ check: CheckSpec,
313
+ baseline: BaselineStore | None,
314
+ ) -> CheckResult:
315
+ count = int(conn.execute("SELECT COUNT(*) FROM source_data").fetchone()[0])
316
+ if baseline is None:
317
+ return CheckResult(check.name, False, "row_count_delta requires a baseline store")
318
+ previous = baseline.get(check.name)
319
+ if previous is None:
320
+ return CheckResult(
321
+ check.name,
322
+ False,
323
+ f"no baseline for {check.name}; run duckcheck baseline update",
324
+ )
325
+ if previous == 0:
326
+ drift = 0.0 if count == 0 else 100.0
327
+ else:
328
+ drift = abs(count - previous) / previous * 100.0
329
+ tol = check.tolerance_pct if check.tolerance_pct is not None else 10.0
330
+ passed = drift <= tol
331
+ return CheckResult(
332
+ check.name,
333
+ passed,
334
+ f"row count {count} vs baseline {previous} ({drift:.1f}% drift, tol {tol}%)"
335
+ if not passed
336
+ else f"row count {count} within {tol}% of baseline {previous}",
337
+ rows_failed=0 if passed else 1,
338
+ )
339
+
340
+
341
+ def to_junit(report: RunReport) -> str:
342
+ """Serialize results as JUnit XML for CI dashboards."""
343
+ suite = Element("testsuite", name=report.suite, tests=str(len(report.results)))
344
+ failures = 0
345
+ for result in report.results:
346
+ case = SubElement(suite, "testcase", name=result.name, classname=report.suite)
347
+ if not result.passed:
348
+ failures += 1
349
+ failure = SubElement(case, "failure", message=result.message)
350
+ failure.text = result.message
351
+ suite.set("failures", str(failures))
352
+ return tostring(suite, encoding="unicode")
duckcheck/spec.py ADDED
@@ -0,0 +1,26 @@
1
+ """Check specification models."""
2
+
3
+ from pydantic import BaseModel, Field
4
+
5
+
6
+ class CheckSpec(BaseModel):
7
+ name: str
8
+ type: str
9
+ table: str = "source_data"
10
+ column: str | None = None
11
+ values: list[str] = Field(default_factory=list)
12
+ sql: str | None = None
13
+ # custom_sql: pass when row count matches expect ("0", "=3", ">0", "<10")
14
+ expect: str | int | None = "0"
15
+ max_age: str | None = None
16
+ min_rows: int | None = None
17
+ max_rows: int | None = None
18
+ tolerance_pct: float | None = None
19
+
20
+
21
+ class SuiteSpec(BaseModel):
22
+ name: str
23
+ source: str
24
+ source_table: str | None = None
25
+ baseline: str | None = None
26
+ checks: list[CheckSpec] = Field(default_factory=list)
@@ -0,0 +1,145 @@
1
+ Metadata-Version: 2.5
2
+ Name: duckcheck
3
+ Version: 0.4.0
4
+ Summary: Lightweight data quality checks powered by DuckDB
5
+ Project-URL: Homepage, https://github.com/yashshah9/duckcheck
6
+ Project-URL: Repository, https://github.com/yashshah9/duckcheck
7
+ Project-URL: Issues, https://github.com/yashshah9/duckcheck/issues
8
+ Author-email: Yash Shah <yash376351@gmail.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Requires-Python: >=3.11
12
+ Requires-Dist: click>=8.1
13
+ Requires-Dist: duckdb>=1.0
14
+ Requires-Dist: pydantic>=2.6
15
+ Requires-Dist: pyyaml>=6.0
16
+ Requires-Dist: rich>=13.7
17
+ Requires-Dist: structlog>=24.1
18
+ Provides-Extra: dev
19
+ Requires-Dist: mypy>=1.9; extra == 'dev'
20
+ Requires-Dist: pytest>=8.0; extra == 'dev'
21
+ Requires-Dist: ruff>=0.4; extra == 'dev'
22
+ Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # duckcheck
26
+
27
+ Lightweight data quality checks powered by **DuckDB** — the anti–Great Expectations for teams who want `pip install`, one YAML file, and one command.
28
+
29
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
30
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
31
+ [![CI](https://github.com/yashshah9/duckcheck/actions/workflows/ci.yml/badge.svg)](https://github.com/yashshah9/duckcheck/actions/workflows/ci.yml)
32
+
33
+ > **Status:** v0.4 — CSV/Parquet/SQLite sources, custom SQL with `expect` operators, freshness, baselines, JUnit, and `--format json`.
34
+
35
+ ## 60-second try
36
+
37
+ ```bash
38
+ docker compose run --rm run-example # duckcheck run examples/clean.yaml
39
+ docker compose run --rm test # pytest
40
+ ```
41
+
42
+ ## Why this vs alternatives
43
+
44
+ | Approach | Strength | Gap |
45
+ |----------|----------|-----|
46
+ | **duckcheck** | One YAML + DuckDB, local files, CI-friendly | Not a full observability platform |
47
+ | Great Expectations | Rich ecosystem | Heavyweight setup for simple column checks |
48
+ | Soda Core | Familiar check DSL | Cloud-oriented workflow |
49
+ | Ad-hoc SQL in CI | Zero new tools | No standard report / JUnit / baselines |
50
+
51
+ ## Problem
52
+
53
+ Data teams need to assert column quality in CI, but Great Expectations is heavyweight and Soda Core funnels to cloud. Ad-hoc SQL checks have no reporting standard.
54
+
55
+ ## Key features (v0.4)
56
+
57
+ - YAML check definitions
58
+ - DuckDB scans CSV, Parquet, and SQLite locally — no server
59
+ - Checks: `not_null`, `unique`, `accepted_values`, `custom_sql`, `freshness`, `row_count`, `row_count_delta`
60
+ - `custom_sql` `expect` operators: `0`, `=N`, `>N`, `<N`, `>=N`, `<=N` (default `0`)
61
+ - `--format json` and `--junit` for CI dashboards
62
+ - `${ENV}` in source URIs; `--source-table` for SQL ATTACH
63
+
64
+ ## Architecture
65
+
66
+ ```
67
+ duckcheck run checks.yaml
68
+ └── SuiteSpec (Pydantic)
69
+ └── DuckDB in-process
70
+ └── source_data view from CSV/Parquet
71
+ ```
72
+
73
+ | Component | Technology | Why |
74
+ |-----------|------------|-----|
75
+ | Engine | DuckDB | Single dependency, scans files + SQL databases |
76
+ | CLI | Click + Rich | Simple, good terminal UX |
77
+ | Spec | YAML + Pydantic | Version-controllable checks |
78
+
79
+ ## Installation
80
+
81
+ ```bash
82
+ pip install duckcheck
83
+ pip install -e ".[dev]"
84
+ ```
85
+
86
+ ## Usage
87
+
88
+ ```bash
89
+ duckcheck health
90
+ duckcheck run examples/clean.yaml
91
+ duckcheck run examples/checks.yaml # fixture with known failures
92
+ duckcheck run examples/clean.yaml --junit /tmp/duckcheck.xml
93
+ duckcheck run examples/clean.yaml --format json
94
+ duckcheck baseline update examples/clean.yaml
95
+ ```
96
+
97
+ Example `checks.yaml`:
98
+
99
+ ```yaml
100
+ name: sample-suite
101
+ source: examples/sample.csv
102
+ checks:
103
+ - name: id_not_null
104
+ type: not_null
105
+ column: id
106
+ - name: status_values
107
+ type: accepted_values
108
+ column: status
109
+ values: [active, inactive]
110
+ - name: three_active
111
+ type: custom_sql
112
+ sql: "SELECT * FROM source_data WHERE status = 'active'"
113
+ expect: "=3"
114
+ ```
115
+
116
+ ## Docker
117
+
118
+ ```bash
119
+ docker compose run --rm test
120
+ docker compose run --rm run-example
121
+ ```
122
+
123
+ ## Running tests
124
+
125
+ ```bash
126
+ pytest tests/ -v
127
+ ```
128
+
129
+ ## Roadmap
130
+
131
+ - [x] Freshness + row_count + custom_sql + JUnit
132
+ - [x] Row-count baseline delta store (`duckcheck baseline update`)
133
+ - [x] custom_sql `expect` operators + `--format json`
134
+ - [ ] Live Postgres/MySQL ATTACH integration tests
135
+ - [ ] Airflow/Dagster operators
136
+
137
+ ## License
138
+
139
+ MIT
140
+
141
+ ## Known limitations (v0.4)
142
+
143
+ - Postgres/MySQL ATTACH is stubbed (`INSTALL/LOAD`) — no live DB in CI yet
144
+ - `examples/checks.yaml` is a failing fixture; `examples/clean.yaml` is the happy path
145
+ - Checks still run against a `source_data` view
@@ -0,0 +1,10 @@
1
+ duckcheck/__init__.py,sha256=kKKgBq_q4-vqb8QA_WvhdqR0QLCACuaQKGcRue2EnI0,65
2
+ duckcheck/baseline.py,sha256=PS5TM_h6bkyE9frgQAPl6ULkQWaoPpc4oYtTeTloBQE,1163
3
+ duckcheck/cli.py,sha256=Fd_-t8apCZn7KyPLysyawfB1KQgSME1f00GgAOELous,3389
4
+ duckcheck/runner.py,sha256=SHKhOjBVtxpkKN50zB5yPYkh4EomwEDen-8-NIPb8aY,12353
5
+ duckcheck/spec.py,sha256=wZx-Vp5mEkFfIuqFIsDZxkt67e-Uozm1TBUTD4AkRX8,704
6
+ duckcheck-0.4.0.dist-info/METADATA,sha256=apb-sNwFgufQXkTKevfzKIwWUx5ywlBxsLqOfAmmp1o,4457
7
+ duckcheck-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
8
+ duckcheck-0.4.0.dist-info/entry_points.txt,sha256=pBnGIP-XwvpLepF_R4kij6R0cxneIL2xlmnLwICIJLs,49
9
+ duckcheck-0.4.0.dist-info/licenses/LICENSE,sha256=AvzqSNURip5YZfXN3-QN_nLlFeC8ob_qbLijyMrFP8w,1079
10
+ duckcheck-0.4.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ duckcheck = duckcheck.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 duckcheck contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.