dqvouch 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dqvouch-0.2.1/.env.example +15 -0
- dqvouch-0.2.1/.gitattributes +3 -0
- dqvouch-0.2.1/.github/release-notes.md +18 -0
- dqvouch-0.2.1/.github/workflows/ci.yml +51 -0
- dqvouch-0.2.1/.github/workflows/release.yml +75 -0
- dqvouch-0.2.1/.gitignore +9 -0
- dqvouch-0.2.1/LICENSE +21 -0
- dqvouch-0.2.1/PKG-INFO +350 -0
- dqvouch-0.2.1/README.md +315 -0
- dqvouch-0.2.1/docker-compose.yml +17 -0
- dqvouch-0.2.1/docs/local-model.png +0 -0
- dqvouch-0.2.1/docs/review.png +0 -0
- dqvouch-0.2.1/docs/screenshot.png +0 -0
- dqvouch-0.2.1/docs/trend.png +0 -0
- dqvouch-0.2.1/examples/orders/ddl_clickhouse.sql +94 -0
- dqvouch-0.2.1/examples/orders/ddl_postgres.sql +152 -0
- dqvouch-0.2.1/examples/orders/dqvouch.clickhouse.toml +18 -0
- dqvouch-0.2.1/examples/orders/dqvouch.toml +18 -0
- dqvouch-0.2.1/examples/orders/load.py +249 -0
- dqvouch-0.2.1/examples/orders/sql/01_staging_orders_trans.sql +4 -0
- dqvouch-0.2.1/examples/orders/sql/02_staging_customers.sql +9 -0
- dqvouch-0.2.1/examples/orders/sql/03_staging_returns.sql +6 -0
- dqvouch-0.2.1/examples/orders/sql/04_analytic_fact_orders.sql +11 -0
- dqvouch-0.2.1/examples/orders/sql/05_analytic_daily_sales.sql +4 -0
- dqvouch-0.2.1/examples/orders/sql/06_analytic_sales_by_country.sql +4 -0
- dqvouch-0.2.1/frontend/.npmrc +2 -0
- dqvouch-0.2.1/frontend/index.html +14 -0
- dqvouch-0.2.1/frontend/package-lock.json +1732 -0
- dqvouch-0.2.1/frontend/package.json +20 -0
- dqvouch-0.2.1/frontend/public/icon.svg +8 -0
- dqvouch-0.2.1/frontend/src/App.vue +250 -0
- dqvouch-0.2.1/frontend/src/Icon.vue +34 -0
- dqvouch-0.2.1/frontend/src/Logo.vue +35 -0
- dqvouch-0.2.1/frontend/src/Sparkline.vue +27 -0
- dqvouch-0.2.1/frontend/src/StageFrame.vue +110 -0
- dqvouch-0.2.1/frontend/src/TestCard.vue +46 -0
- dqvouch-0.2.1/frontend/src/api.js +58 -0
- dqvouch-0.2.1/frontend/src/logic.js +265 -0
- dqvouch-0.2.1/frontend/src/logic.test.js +323 -0
- dqvouch-0.2.1/frontend/src/main.js +6 -0
- dqvouch-0.2.1/frontend/src/screens/Config.vue +244 -0
- dqvouch-0.2.1/frontend/src/screens/Context.vue +76 -0
- dqvouch-0.2.1/frontend/src/screens/Dimensions.vue +41 -0
- dqvouch-0.2.1/frontend/src/screens/Generic.vue +10 -0
- dqvouch-0.2.1/frontend/src/screens/Profiling.vue +42 -0
- dqvouch-0.2.1/frontend/src/screens/Results.vue +294 -0
- dqvouch-0.2.1/frontend/src/screens/Review.vue +147 -0
- dqvouch-0.2.1/frontend/src/screens/Runs.vue +66 -0
- dqvouch-0.2.1/frontend/src/screens/SqlRules.vue +35 -0
- dqvouch-0.2.1/frontend/src/screens/Tests.vue +40 -0
- dqvouch-0.2.1/frontend/src/style.css +388 -0
- dqvouch-0.2.1/frontend/vite.config.js +10 -0
- dqvouch-0.2.1/install.ps1 +10 -0
- dqvouch-0.2.1/install.sh +10 -0
- dqvouch-0.2.1/pyproject.toml +53 -0
- dqvouch-0.2.1/src/dqvouch/__init__.py +1 -0
- dqvouch-0.2.1/src/dqvouch/alerts.py +129 -0
- dqvouch-0.2.1/src/dqvouch/changes.py +507 -0
- dqvouch-0.2.1/src/dqvouch/cli.py +283 -0
- dqvouch-0.2.1/src/dqvouch/config.py +181 -0
- dqvouch-0.2.1/src/dqvouch/context.py +229 -0
- dqvouch-0.2.1/src/dqvouch/llm.py +231 -0
- dqvouch-0.2.1/src/dqvouch/pipeline.py +350 -0
- dqvouch-0.2.1/src/dqvouch/profiling.py +1018 -0
- dqvouch-0.2.1/src/dqvouch/render_sql.py +711 -0
- dqvouch-0.2.1/src/dqvouch/rules.py +176 -0
- dqvouch-0.2.1/src/dqvouch/selector.py +132 -0
- dqvouch-0.2.1/src/dqvouch/server.py +493 -0
- dqvouch-0.2.1/src/dqvouch/signals.py +542 -0
- dqvouch-0.2.1/src/dqvouch/specifier.py +544 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/index-6eimbKv1.css +1 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/index-CaNRKsuh.js +3 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-cyrillic-ext-wght-normal-BOeWTOD4.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-cyrillic-wght-normal-DqGufNeO.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-greek-ext-wght-normal-DlzME5K_.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-greek-wght-normal-CkhJZR-_.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-latin-ext-wght-normal-DO1Apj_S.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-latin-wght-normal-Dx4kXJAl.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- dqvouch-0.2.1/src/dqvouch/static/icon.svg +8 -0
- dqvouch-0.2.1/src/dqvouch/static/index.html +15 -0
- dqvouch-0.2.1/src/dqvouch/store.py +163 -0
- dqvouch-0.2.1/src/dqvouch/translate.py +114 -0
- dqvouch-0.2.1/src/dqvouch/validator.py +1394 -0
- dqvouch-0.2.1/src/dqvouch/warehouse.py +214 -0
- dqvouch-0.2.1/tests/conftest.py +48 -0
- dqvouch-0.2.1/tests/fake_llm.py +48 -0
- dqvouch-0.2.1/tests/serve_fake.py +43 -0
- dqvouch-0.2.1/tests/test_alerts.py +169 -0
- dqvouch-0.2.1/tests/test_changes.py +531 -0
- dqvouch-0.2.1/tests/test_cli.py +183 -0
- dqvouch-0.2.1/tests/test_clickhouse.py +71 -0
- dqvouch-0.2.1/tests/test_config.py +163 -0
- dqvouch-0.2.1/tests/test_core.py +58 -0
- dqvouch-0.2.1/tests/test_engines.py +193 -0
- dqvouch-0.2.1/tests/test_export.py +88 -0
- dqvouch-0.2.1/tests/test_llm.py +175 -0
- dqvouch-0.2.1/tests/test_pipeline.py +164 -0
- dqvouch-0.2.1/tests/test_postgres.py +85 -0
- dqvouch-0.2.1/tests/test_protections.py +51 -0
- dqvouch-0.2.1/tests/test_server.py +527 -0
- dqvouch-0.2.1/tests/test_sql_safety.py +37 -0
- dqvouch-0.2.1/tests/test_store.py +150 -0
- dqvouch-0.2.1/tests/test_thesis_regression.py +519 -0
- dqvouch-0.2.1/tests/test_translate.py +72 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# dqvouch reads these from the environment or from a .env file in the folder you start it from.
|
|
2
|
+
# `dqvouch ui` saves them there for you (a private file); never commit .env.
|
|
3
|
+
GEMINI_API_KEY=
|
|
4
|
+
# Only for an OpenAI-compatible server that asks for a key ([llm] provider = "openai"):
|
|
5
|
+
OPENAI_API_KEY=
|
|
6
|
+
DQVOUCH_DB_PASSWORD=
|
|
7
|
+
|
|
8
|
+
# Alerts when a test starts failing (both optional; set them on the Results screen or here).
|
|
9
|
+
DQVOUCH_ALERT_WEBHOOK_URL=
|
|
10
|
+
DQVOUCH_SMTP_HOST=
|
|
11
|
+
DQVOUCH_SMTP_PORT=587
|
|
12
|
+
DQVOUCH_SMTP_USER=
|
|
13
|
+
DQVOUCH_SMTP_PASSWORD=
|
|
14
|
+
DQVOUCH_SMTP_FROM=
|
|
15
|
+
DQVOUCH_SMTP_TO=
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
**An LLM proposes data quality tests, code validates them, you vouch for them.**
|
|
2
|
+
|
|
3
|
+
dqvouch reads what your pipeline already says about each table (schema, descriptions, the SQL that builds it, lineage), lets a language model propose data quality tests, checks every proposal in code against the real schema and data, and shows you the result before anything runs. Then it keeps running the tests you confirmed and tells you when one starts failing.
|
|
4
|
+
|
|
5
|
+
**New in this version: it is on PyPI**
|
|
6
|
+
- Install it with `pip install dqvouch`; upgrade with `pip install -U dqvouch`. Your runs are kept.
|
|
7
|
+
- The sidebar shows the version you have installed (0.2.0 still said 0.1.0).
|
|
8
|
+
|
|
9
|
+
What 0.2.0 brought, change detection on scheduled runs, is in its [release notes](https://github.com/FuzzyLaDuzzy/dqvouch/releases/tag/v0.2.0).
|
|
10
|
+
|
|
11
|
+
**Install** (Python 3.11 or newer), then open the step-by-step interface:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install dqvouch
|
|
15
|
+
dqvouch ui
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
To try it on the fictional orders example, with its planted defects, follow the [Quickstart](https://github.com/FuzzyLaDuzzy/dqvouch#quickstart-fictional-orders-example).
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on: [push, pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
services:
|
|
9
|
+
clickhouse:
|
|
10
|
+
image: clickhouse/clickhouse-server:24.8
|
|
11
|
+
ports:
|
|
12
|
+
- 18123:8123
|
|
13
|
+
env:
|
|
14
|
+
CLICKHOUSE_USER: dqvouch
|
|
15
|
+
CLICKHOUSE_PASSWORD: dqvouch
|
|
16
|
+
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 # load.py creates the read-only user dq_reader
|
|
17
|
+
postgres:
|
|
18
|
+
image: postgres:16
|
|
19
|
+
ports:
|
|
20
|
+
- 15432:5432
|
|
21
|
+
env:
|
|
22
|
+
POSTGRES_USER: dqvouch
|
|
23
|
+
POSTGRES_PASSWORD: dqvouch
|
|
24
|
+
POSTGRES_DB: dqvouch
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
- uses: actions/setup-python@v5
|
|
28
|
+
with:
|
|
29
|
+
python-version: "3.11"
|
|
30
|
+
- run: pip install -e ".[dev]"
|
|
31
|
+
- run: python examples/orders/load.py
|
|
32
|
+
- run: python examples/orders/load.py examples/orders/dqvouch.clickhouse.toml
|
|
33
|
+
- run: pytest -q
|
|
34
|
+
env:
|
|
35
|
+
DQVOUCH_REQUIRE_DB: "1"
|
|
36
|
+
|
|
37
|
+
ui:
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
defaults:
|
|
40
|
+
run:
|
|
41
|
+
working-directory: frontend
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
- uses: actions/setup-node@v4
|
|
45
|
+
with:
|
|
46
|
+
node-version: "22"
|
|
47
|
+
- run: npm ci
|
|
48
|
+
- run: npm test
|
|
49
|
+
- run: npm run build
|
|
50
|
+
# The build is committed (pip install needs no Node): it must match the sources.
|
|
51
|
+
- run: git diff --exit-code -- ../src/dqvouch/static
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# A v tag (v0.1.0) builds the package, checks it installs and runs, and publishes it as a GitHub release
|
|
2
|
+
# with the wheel and the source archive, its notes from .github/release-notes.md.
|
|
3
|
+
#
|
|
4
|
+
# PyPI (pip install dqvouch): with trusted publishing, no password or token is stored here. Once, on pypi.org:
|
|
5
|
+
# Account → Publishing → add a pending publisher: owner FuzzyLaDuzzy, repository dqvouch, workflow
|
|
6
|
+
# release.yml, environment pypi. Then in the repository's Settings → Secrets and variables → Actions →
|
|
7
|
+
# Variables, PUBLISH_TO_PYPI = true: every later tag also goes to PyPI. A tag released before that goes to
|
|
8
|
+
# PyPI by running this workflow by hand (Actions → release → Run workflow) on that tag.
|
|
9
|
+
name: release
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
push:
|
|
13
|
+
tags: ["v*"]
|
|
14
|
+
workflow_dispatch:
|
|
15
|
+
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read # only the release job writes (its own permissions, below)
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
build:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.11"
|
|
27
|
+
- name: The tag is the version in pyproject.toml
|
|
28
|
+
if: github.ref_type == 'tag'
|
|
29
|
+
run: |
|
|
30
|
+
version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
|
|
31
|
+
[ "v$version" = "$GITHUB_REF_NAME" ] || { echo "tag $GITHUB_REF_NAME, pyproject.toml $version"; exit 1; }
|
|
32
|
+
- run: pip install build twine
|
|
33
|
+
- run: python -m build
|
|
34
|
+
- run: twine check dist/*
|
|
35
|
+
- name: The wheel installs on its own and starts
|
|
36
|
+
run: |
|
|
37
|
+
python -m venv /tmp/clean
|
|
38
|
+
/tmp/clean/bin/pip install dist/*.whl
|
|
39
|
+
/tmp/clean/bin/dqvouch --help
|
|
40
|
+
- uses: actions/upload-artifact@v4
|
|
41
|
+
with:
|
|
42
|
+
name: dist
|
|
43
|
+
path: dist/
|
|
44
|
+
|
|
45
|
+
release:
|
|
46
|
+
if: github.event_name == 'push'
|
|
47
|
+
needs: build
|
|
48
|
+
runs-on: ubuntu-latest
|
|
49
|
+
permissions:
|
|
50
|
+
contents: write
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- uses: actions/download-artifact@v4
|
|
54
|
+
with:
|
|
55
|
+
name: dist
|
|
56
|
+
path: dist
|
|
57
|
+
- run: gh release create "$GITHUB_REF_NAME" dist/* --title "dqvouch ${GITHUB_REF_NAME#v}" --notes-file .github/release-notes.md
|
|
58
|
+
env:
|
|
59
|
+
GH_TOKEN: ${{ github.token }}
|
|
60
|
+
|
|
61
|
+
pypi:
|
|
62
|
+
if: github.ref_type == 'tag' && (vars.PUBLISH_TO_PYPI == 'true' || github.event_name == 'workflow_dispatch')
|
|
63
|
+
needs: build
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
environment:
|
|
66
|
+
name: pypi
|
|
67
|
+
url: https://pypi.org/project/dqvouch/
|
|
68
|
+
permissions:
|
|
69
|
+
id-token: write # trusted publishing: PyPI checks this workflow's identity, no token needed
|
|
70
|
+
steps:
|
|
71
|
+
- uses: actions/download-artifact@v4
|
|
72
|
+
with:
|
|
73
|
+
name: dist
|
|
74
|
+
path: dist
|
|
75
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
dqvouch-0.2.1/.gitignore
ADDED
dqvouch-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Flávio Silva
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dqvouch-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dqvouch
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: An LLM proposes data quality tests, code validates them, you vouch for them.
|
|
5
|
+
Project-URL: Homepage, https://github.com/FuzzyLaDuzzy/dqvouch
|
|
6
|
+
Project-URL: Issues, https://github.com/FuzzyLaDuzzy/dqvouch/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/FuzzyLaDuzzy/dqvouch/releases
|
|
8
|
+
Author: Flávio Silva
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: clickhouse,data quality,data testing,gemini,llm,postgresql
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Web Environment
|
|
14
|
+
Classifier: Framework :: FastAPI
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Database
|
|
22
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Requires-Dist: clickhouse-connect>=1.6
|
|
25
|
+
Requires-Dist: fastapi>=0.115
|
|
26
|
+
Requires-Dist: google-genai>=2.17
|
|
27
|
+
Requires-Dist: psycopg[binary]>=3.2
|
|
28
|
+
Requires-Dist: python-dotenv>=1.2
|
|
29
|
+
Requires-Dist: sqlglot>=30.30
|
|
30
|
+
Requires-Dist: uvicorn>=0.30
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# dqvouch
|
|
37
|
+
|
|
38
|
+
[](https://github.com/FuzzyLaDuzzy/dqvouch/actions/workflows/ci.yml)
|
|
39
|
+
[](https://pypi.org/project/dqvouch/)
|
|
40
|
+
[](https://github.com/FuzzyLaDuzzy/dqvouch/blob/main/pyproject.toml)
|
|
41
|
+
[](https://github.com/FuzzyLaDuzzy/dqvouch/blob/main/LICENSE)
|
|
42
|
+
|
|
43
|
+
**An LLM proposes data quality tests, code validates them, you vouch for them.**
|
|
44
|
+
|
|
45
|
+
Works on **PostgreSQL** and **ClickHouse**, with Gemini or a model on your own machine, from a step-by-step web
|
|
46
|
+
interface or the command line.
|
|
47
|
+
|
|
48
|
+

|
|
49
|
+
|
|
50
|
+
Writing data quality tests by hand needs someone who knows the data, and it does not scale to hundreds of tables.
|
|
51
|
+
dqvouch reads what your pipeline already tells about each table (schema, descriptions, the SQL that builds it,
|
|
52
|
+
lineage), lets a language model propose tests, checks every proposal in code, and shows you the result before
|
|
53
|
+
anything runs. Then it keeps running the tests you confirmed and tells you when one starts failing.
|
|
54
|
+
|
|
55
|
+
**Contents:** [Features](#features) · [How it works](#how-it-works) · [Quickstart](#quickstart-fictional-orders-example)
|
|
56
|
+
· [Local model](#use-a-local-model) · [Run with dqvouch closed](#run-with-dqvouch-closed) ·
|
|
57
|
+
[When the database changes](#when-the-database-changes) · [Command line](#command-line) · [Configuration](#configuration) · [Results from the thesis](#results-from-the-thesis) ·
|
|
58
|
+
[Limitations](#limitations)
|
|
59
|
+
|
|
60
|
+
## Features
|
|
61
|
+
|
|
62
|
+
- **Tests from context, not from a blank page:** schema, column comments, the transformation SQL, lineage and
|
|
63
|
+
measured profiles go to the model; what comes back is checked against the real schema and data.
|
|
64
|
+
- **The model never writes SQL.** It fills a closed catalog of five assertions; dqvouch renders the SQL, and every
|
|
65
|
+
threshold has to cite evidence that resolves.
|
|
66
|
+
- **You decide.** Each test is confirmed, edited or excluded on the Review screen; rules the data contradicts are
|
|
67
|
+
flagged, not dropped. A new run of the same database starts with your last decisions.
|
|
68
|
+
- **Read-only, enforced twice:** one `SELECT` at a time in the code, read-only sessions in the database, a time
|
|
69
|
+
limit per query, and a warning when the user you gave it could write.
|
|
70
|
+
- **Gemini or a local model:** any OpenAI-compatible server (Ollama, LM Studio, vLLM) keeps your metadata inside
|
|
71
|
+
your network.
|
|
72
|
+
- **Keeps watch:** in-app schedules, or `dqvouch rerun` from cron, Task Scheduler, Airflow or CI (exit code 1 when a
|
|
73
|
+
test fails); every execution is kept, with each test's trend.
|
|
74
|
+
- **Alerts** by webhook (Slack, Teams, Google Chat, Mattermost) or email, once when a test starts failing, not at
|
|
75
|
+
every run.
|
|
76
|
+
- **Take the tests anywhere:** export the confirmed tests as one SQL file in your database's dialect.
|
|
77
|
+
|
|
78
|
+
<table>
|
|
79
|
+
<tr>
|
|
80
|
+
<td width="50%"><img src="https://raw.githubusercontent.com/FuzzyLaDuzzy/dqvouch/main/docs/review.png" alt="Review: a rule the data contradicts is flagged, and the person decides"></td>
|
|
81
|
+
<td width="50%"><img src="https://raw.githubusercontent.com/FuzzyLaDuzzy/dqvouch/main/docs/trend.png" alt="Results: a test's observed value over the last executions, with its limit"></td>
|
|
82
|
+
</tr>
|
|
83
|
+
<tr>
|
|
84
|
+
<td><sub>Review: a rule the data contradicts is flagged; you confirm, exclude or restore it.</sub></td>
|
|
85
|
+
<td><sub>Results: each test's observed value over the last executions, against its limit.</sub></td>
|
|
86
|
+
</tr>
|
|
87
|
+
</table>
|
|
88
|
+
|
|
89
|
+
## How it works
|
|
90
|
+
|
|
91
|
+
```mermaid
|
|
92
|
+
flowchart LR
|
|
93
|
+
A["1 Context<br/>code"] --> B["2 Dimensions<br/>LLM"] --> C["3 Profiling<br/>code"] --> D["4 SQL rules<br/>LLM"]
|
|
94
|
+
D --> E["5 Tests<br/>LLM + validator"] --> F["6 Review<br/>you"] --> G["7 Results<br/>code"]
|
|
95
|
+
G -. "schedule · rerun · alerts" .-> G
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
| # | Stage | Who | What happens |
|
|
99
|
+
|---|---|---|---|
|
|
100
|
+
| 1 | Context | code | One `AssetContext` per table: columns, types, comments, the SQL that produces it, upstream and downstream tables, and facts the code extracts from the SQL (`dq_signals`). |
|
|
101
|
+
| 2 | Dimensions | LLM, 1 call | Which data quality dimensions (completeness, uniqueness, validity, consistency, freshness, accuracy) to check on which columns. The validator drops tables and columns that do not exist. |
|
|
102
|
+
| 3 | Profiling | code | Metric-only SQL for the accepted dimensions (null counts, duplicates, value frequencies...), run read-only. |
|
|
103
|
+
| 4 | SQL rules | LLM, 2 calls per transformation | What each transformation's SQL guarantees about its output and requires of its inputs, each tied to the SQL excerpt that enforces it. |
|
|
104
|
+
| 5 | Tests | LLM, 1 call per table and per table pair | Concrete tests from a closed catalog, checked by the validator (and optionally sent back once to be repaired). |
|
|
105
|
+
| 6 | Review | you | Confirm, edit or exclude each test. |
|
|
106
|
+
| 7 | Results | code | Each test rendered to SQL, run read-only, with a verdict: pass, fail, no threshold, not comparable or error. |
|
|
107
|
+
|
|
108
|
+
Three rules hold everywhere:
|
|
109
|
+
|
|
110
|
+
1. **The LLM proposes, the code validates, a person decides.**
|
|
111
|
+
2. **The LLM never writes SQL nor touches the database.** It returns structured JSON; SQL is generated by code from a
|
|
112
|
+
closed catalog of five assertions (`row_predicate`, `unique`, `cross_asset_row_count`,
|
|
113
|
+
`cross_asset_missing_rows`, `cross_asset_column_equality`).
|
|
114
|
+
3. **No threshold without evidence.** A limit is accepted only if it cites a fact that resolves (a column
|
|
115
|
+
description, the SQL, a measured metric). Without one, the test only observes and reports the value. When the
|
|
116
|
+
data contradicts a well-formed rule, the validator flags it for review instead of deleting it.
|
|
117
|
+
|
|
118
|
+
dqvouch only reads from your database: it sends one `SELECT` at a time and refuses anything else, PostgreSQL
|
|
119
|
+
sessions are opened read-only and ClickHouse queries carry `readonly = 2`, and every query stops after
|
|
120
|
+
`query_timeout_seconds` (600 by default). Still, connect with a read-only user: that is the guarantee that holds
|
|
121
|
+
whatever happens in between, and dqvouch warns you when the user it was given can write. On PostgreSQL grant it
|
|
122
|
+
`SELECT` only; on ClickHouse give it `readonly = 2` (`readonly = 1` also forbids writes, but then no time limit can
|
|
123
|
+
be sent with the queries).
|
|
124
|
+
|
|
125
|
+
Inside dqvouch all SQL and column types use one notation (ClickHouse's, the one the thesis evaluated); each database
|
|
126
|
+
connection translates at the boundary, so the same stages run unchanged on every engine, and the SQL saved with a
|
|
127
|
+
run is the SQL your database actually ran.
|
|
128
|
+
|
|
129
|
+
## Quickstart (fictional orders example)
|
|
130
|
+
|
|
131
|
+
Requires Python 3.11+, Docker and a [Gemini API key](https://aistudio.google.com/apikey).
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
git clone https://github.com/FuzzyLaDuzzy/dqvouch.git && cd dqvouch
|
|
135
|
+
sh install.sh # Windows: powershell -ExecutionPolicy Bypass -File install.ps1
|
|
136
|
+
docker compose up -d postgres # PostgreSQL on localhost:15432
|
|
137
|
+
.venv/bin/python examples/orders/load.py # 10 tables (orders, customers, returns), landing -> staging -> analytic, with planted defects
|
|
138
|
+
.venv/bin/dqvouch ui --config examples/orders/dqvouch.toml
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
On Windows the programs are in `.venv\Scripts\` instead of `.venv/bin/`. `install.sh` only creates `.venv` and runs
|
|
142
|
+
`pip install -e .` in it; do that yourself if you prefer. For your own database you don't need the repository:
|
|
143
|
+
`pip install dqvouch` (Python 3.11 or newer), then `dqvouch ui`.
|
|
144
|
+
|
|
145
|
+
The browser opens on the first screen with the example filled in. Paste your Gemini key, press **Start** and follow
|
|
146
|
+
the stages: each screen says what happens, whether code or the model does it, what was sent to Gemini and what came
|
|
147
|
+
back. Before the **Review** screen dqvouch only reads your tables' metadata and runs read-only measurement queries
|
|
148
|
+
(Context and Profiling); no test runs until you confirm it there and press **Run confirmed tests**. To use your own
|
|
149
|
+
database, press **New run** and fill in the first screen instead.
|
|
150
|
+
|
|
151
|
+
On the **Results** screen you can run the confirmed tests again on a schedule (every n minutes, hours or days).
|
|
152
|
+
The schedule runs while `dqvouch ui` is running on that computer; an execution missed while it was off runs once
|
|
153
|
+
when it starts again. To run them with dqvouch closed, the same screen gives you the line for cron or the Windows
|
|
154
|
+
Task Scheduler (see [Run with dqvouch closed](#run-with-dqvouch-closed)). Every execution is kept in the run's
|
|
155
|
+
`history.jsonl`: the screen lists them with the tests that started failing since the one before, and draws each
|
|
156
|
+
test's observed value over the last 20 executions, so a count creeping towards its limit shows before it fails.
|
|
157
|
+
Metric thresholds are measured again at every execution.
|
|
158
|
+
|
|
159
|
+
**Alerts:** on the same screen, a Slack, Mattermost, Google Chat or Teams webhook and/or an SMTP server get one
|
|
160
|
+
message when a test starts failing or erroring, or when an execution cannot run, and not again while it stays that
|
|
161
|
+
way. They apply to every run and are kept in `.env`; a failed send is noted in the history, never fails the run.
|
|
162
|
+
|
|
163
|
+
**Export SQL** (Review and Results) downloads the confirmed tests as one SQL file in your database's dialect, each
|
|
164
|
+
query with a comment saying what it checks and what it expects, to run them in your own tools without dqvouch.
|
|
165
|
+
|
|
166
|
+
A new run of the same database starts with the decisions of the last review made on the Review screen: tests that
|
|
167
|
+
check the same thing come confirmed or excluded as you left them (with your threshold), and you can show only the
|
|
168
|
+
new ones.
|
|
169
|
+
|
|
170
|
+
The interface only listens on `127.0.0.1` and refuses requests addressed to any other host name. The key and the
|
|
171
|
+
database password are saved in the `.env` file of the folder you start it from (a value already set in the
|
|
172
|
+
environment wins) and are never written to a run's files.
|
|
173
|
+
|
|
174
|
+
The same example on ClickHouse: `docker compose up -d clickhouse`, then
|
|
175
|
+
`python examples/orders/load.py examples/orders/dqvouch.clickhouse.toml` and
|
|
176
|
+
`dqvouch ui --config examples/orders/dqvouch.clickhouse.toml`.
|
|
177
|
+
|
|
178
|
+
## Use a local model
|
|
179
|
+
|
|
180
|
+
The model can run on your own machine, so the metadata and profiling numbers never leave your network. Any server
|
|
181
|
+
with OpenAI's chat completions API works: [Ollama](https://ollama.com), LM Studio, vLLM, llama.cpp, and also
|
|
182
|
+
hosted ones such as OpenAI or Azure. On the Configuration screen choose *OpenAI-compatible server*, or in the config:
|
|
183
|
+
|
|
184
|
+
<img src="https://raw.githubusercontent.com/FuzzyLaDuzzy/dqvouch/main/docs/local-model.png" alt="The Configuration screen with an OpenAI-compatible server (Ollama) as the model" width="720">
|
|
185
|
+
|
|
186
|
+
```toml
|
|
187
|
+
[llm]
|
|
188
|
+
provider = "openai"
|
|
189
|
+
base_url = "http://localhost:11434/v1" # Ollama; LM Studio: http://localhost:1234/v1
|
|
190
|
+
model = "qwen3:14b"
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
A key is only needed if the server asks for one (`OPENAI_API_KEY`). The Dimensions stage sends every table in one
|
|
194
|
+
prompt, and Ollama's default context is small and silently cuts longer prompts: start it with a larger one, e.g.
|
|
195
|
+
`OLLAMA_CONTEXT_LENGTH=32768 ollama serve`. The thesis numbers below were measured with Gemini; a local model may
|
|
196
|
+
propose different (and fewer valid) tests, which the validator still checks the same way.
|
|
197
|
+
|
|
198
|
+
## Run with dqvouch closed
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
dqvouch rerun runs/20261111-150012 --env-file .env
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
`rerun` executes a reviewed run's confirmed tests once, adds the execution to its history and sends the alerts. It
|
|
205
|
+
exits with 0 when no test failed or errored, 1 when one did and 2 when it could not run, so cron, the Windows Task
|
|
206
|
+
Scheduler, Airflow or a CI job can call it and act on the result. The Results screen shows the exact cron and
|
|
207
|
+
`schtasks` lines for your interval. Two executions of one run never overlap (a lock file in the run folder), so the
|
|
208
|
+
in-app schedule and an operating-system one can both be on.
|
|
209
|
+
|
|
210
|
+
`dqvouch export RUN_DIR [-o tests.sql]` writes the confirmed tests as one SQL file, like the Export SQL button.
|
|
211
|
+
|
|
212
|
+
## When the database changes
|
|
213
|
+
|
|
214
|
+
After every scheduled execution (`dqvouch rerun` or the in-app schedule), once its tests ran, dqvouch compares the
|
|
215
|
+
structure of each table with the one the run was reviewed on: its columns (names, types, descriptions), its
|
|
216
|
+
description, the SQL that builds it and the tables it reads from or feeds. This only reads metadata and the SQL
|
|
217
|
+
files, and calls no model. Values never count: more rows or more nulls are what the tests and the trend are for;
|
|
218
|
+
neither does a new dqvouch version.
|
|
219
|
+
|
|
220
|
+
When something changed, dqvouch makes a new run in which the model only proposes tests for the changed and new
|
|
221
|
+
tables. Every confirmed test is kept as it was, on every table; a test whose column or table is gone is set aside
|
|
222
|
+
with the reason. You get one alert (*"structure changed in 3 table(s), 4 new test(s) to review"*), nothing new runs
|
|
223
|
+
until you confirm it on the Review screen, and meanwhile the old run keeps executing every day. The same change seen
|
|
224
|
+
the next day sends nothing; a second change before you reviewed replaces the waiting run.
|
|
225
|
+
|
|
226
|
+
Once you review it, the schedule moves to the new run and the history comes along, so the trend of every kept test
|
|
227
|
+
continues. A cron or Task Scheduler line that names the old run keeps working: `rerun` follows it to the newest
|
|
228
|
+
reviewed run and says so. Proposing needs the model key in the `--env-file`; if the model cannot be reached, the
|
|
229
|
+
tests still run, the exit code still comes from them, and dqvouch alerts once and tries again at the next execution.
|
|
230
|
+
If the transformation SQL folder cannot be found when the tests run (moved, or on another machine), `rerun` says
|
|
231
|
+
so and nothing is treated as a change.
|
|
232
|
+
To only be told about a change, without any model call from a schedule, set `propose_on_change = false` in `[run]`.
|
|
233
|
+
Each run keeps the value it was created with (from the config file, or the one `dqvouch ui --config` was started
|
|
234
|
+
with); to change it for a run that already exists, edit `propose_on_change` in that run's `run.json`. Runs made
|
|
235
|
+
before this feature have no value there and propose.
|
|
236
|
+
|
|
237
|
+
## Command line
|
|
238
|
+
|
|
239
|
+
The command line runs every stage in one go, and is how the thesis experiments are reproduced:
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
echo GEMINI_API_KEY=your-key > .env
|
|
243
|
+
dqvouch run --config examples/orders/dqvouch.toml
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
The run stops before executing anything: edit `review_input.json` in the run folder (exclude a test, change a
|
|
247
|
+
threshold), save it and press Enter. The report lists every test with its verdict and observed value.
|
|
248
|
+
|
|
249
|
+
## Configuration
|
|
250
|
+
|
|
251
|
+
```toml
|
|
252
|
+
[database]
|
|
253
|
+
engine = "postgres" # or "clickhouse"
|
|
254
|
+
host = "localhost"
|
|
255
|
+
port = 5432 # ClickHouse: the HTTP port, usually 8123
|
|
256
|
+
database = "warehouse" # PostgreSQL only: the database to connect to
|
|
257
|
+
user = "dq_reader" # read-only is enough
|
|
258
|
+
password = "" # better: set DQVOUCH_DB_PASSWORD
|
|
259
|
+
schemas = ["landing", "staging", "analytic"] # PostgreSQL schemas or ClickHouse databases
|
|
260
|
+
query_timeout_seconds = 600 # the longest one query may run; 0 for no limit
|
|
261
|
+
|
|
262
|
+
[llm]
|
|
263
|
+
model = "gemini-3.5-flash-lite" # the key comes from GEMINI_API_KEY (environment or .env)
|
|
264
|
+
# provider = "openai" # a local or other model: see "Use a local model"
|
|
265
|
+
# base_url = "http://localhost:11434/v1"
|
|
266
|
+
|
|
267
|
+
[run]
|
|
268
|
+
sql_dir = "sql" # optional: the transformation SQL; each file is matched to the table it produces
|
|
269
|
+
runs_dir = "runs" # one folder per run
|
|
270
|
+
propose_on_change = true # false: a scheduled execution only alerts about a structural change
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Transformation files are matched to tables by what they produce: `INSERT INTO t`, `CREATE TABLE t AS SELECT`,
|
|
274
|
+
or `CREATE [MATERIALIZED] VIEW t [TO t2]`. File names do not matter. Without SQL, dqvouch still works from the
|
|
275
|
+
schema and the comments, but finds fewer tests.
|
|
276
|
+
|
|
277
|
+
Command line options:
|
|
278
|
+
|
|
279
|
+
| Option | Meaning |
|
|
280
|
+
|---|---|
|
|
281
|
+
| `--tables SCHEMA.TABLE ...` | only these tables |
|
|
282
|
+
| `--level C1 C2 C3` | context level: C1 structure only, C2 + descriptions, C3 + SQL and lineage (default) |
|
|
283
|
+
| `--repeat N` | run every level N times (to measure the model's variance) |
|
|
284
|
+
| `--no-signals`, `--no-sql-rules`, `--no-repair` | turn a part off (ablations) |
|
|
285
|
+
| `--no-review` | execute the proposed tests without the review pause |
|
|
286
|
+
|
|
287
|
+
The web interface always runs the final configuration of the thesis (C3, with the SQL rules and the repair pass);
|
|
288
|
+
`dqvouch ui` takes `--config`, `--runs-dir`, `--port` (default 8765), `--env-file` and `--no-browser`.
|
|
289
|
+
|
|
290
|
+
## Developing the interface
|
|
291
|
+
|
|
292
|
+
The interface is Vue 3 + Vite in `frontend/`; its build lives in `src/dqvouch/static/` and is committed, so
|
|
293
|
+
installing dqvouch needs no Node.
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
python tests/serve_fake.py --slow 2 # the API on :8765 with a scripted model: no key, no quota
|
|
297
|
+
cd frontend && npm ci && npm run dev # hot-reloading UI on :5173, proxied to :8765
|
|
298
|
+
npm test # the UI's logic
|
|
299
|
+
npm run build # before every commit that changes the UI
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
## What a run leaves behind
|
|
303
|
+
|
|
304
|
+
Every run is a folder in `runs_dir`: `run.json` (the settings, without secrets) and one file per stage,
|
|
305
|
+
`01_context.json` to `07_results.json`, with the prompts, the raw model answers, what the validator accepted and
|
|
306
|
+
rejected and why, the profiling metrics and the verdicts. `history.jsonl` keeps every execution (verdicts, observed
|
|
307
|
+
values, alerts sent), `schedule.json` the in-app schedule, and `execution.lock` exists only while the tests run.
|
|
308
|
+
Nothing is written to your database.
|
|
309
|
+
|
|
310
|
+
## Results from the thesis
|
|
311
|
+
|
|
312
|
+
dqvouch comes from a master's dissertation that evaluated it on a real six-table pipeline against 18 tests written
|
|
313
|
+
by data engineers (`gemini-3.5-flash-lite`, temperature 0, 3 repetitions):
|
|
314
|
+
|
|
315
|
+
| | C1 | C2 | C3 | C3 + SQL rules |
|
|
316
|
+
|---|---|---|---|---|
|
|
317
|
+
| Engineers' tests found (18) | 33% | 22% | 39% | **67%** |
|
|
318
|
+
| ... on one table (11) | 55% | 36% | 64% | 82% |
|
|
319
|
+
| ... across tables (7) | 0% | 0% | 0% | **43%** |
|
|
320
|
+
| Tests per run | 88 | 87 | 70 | 117–142 |
|
|
321
|
+
| LLM calls | 13 | 13–15 | 15–17 | 47–49 |
|
|
322
|
+
| Duration | 4–5 min | 5–7 min | 5–6 min | 7 min |
|
|
323
|
+
|
|
324
|
+
Of the extra tests the framework proposed, the engineers judged 54% worth keeping, and 91% of those that came
|
|
325
|
+
from the SQL rules. Having the logic in the SQL is not enough: the model finds the rule when the question is the
|
|
326
|
+
right one, but choosing the key, telling a filter from a rule and knowing how data is loaded take operational
|
|
327
|
+
knowledge that is not in the code.
|
|
328
|
+
|
|
329
|
+
> Flávio Silva, *Framework para geração automática de verificações de qualidade de dados*, MSc dissertation,
|
|
330
|
+
> Universidade do Minho, 2026.
|
|
331
|
+
|
|
332
|
+
## Limitations
|
|
333
|
+
|
|
334
|
+
- **Filter or rule:** a condition the SQL uses to select rows is sometimes read as a rule for every row.
|
|
335
|
+
- **Key choice:** tests across tables often join on a weaker key than the right one.
|
|
336
|
+
- **Operational context:** load patterns (per partition, reprocessing, late arrivals) are not in the SQL.
|
|
337
|
+
- **Two tables at most:** checks involving three or more tables are ignored.
|
|
338
|
+
- **Scale:** the dimension stage sees every table in one call and can exceed the context window; use `--tables`.
|
|
339
|
+
- **Privacy:** the profiling sent to the model includes counts and up to 8 frequent values per check (never whole
|
|
340
|
+
rows). With sensitive data, use a [local model](#use-a-local-model) rather than a hosted one.
|
|
341
|
+
- **Closed catalog:** recomputing an expression from the SQL, comparing a full union of sources and comparing
|
|
342
|
+
pipeline re-runs are out of reach.
|
|
343
|
+
- **Evaluation:** one pipeline, 18 reference tests.
|
|
344
|
+
- **PostgreSQL:** the length of an `array` column is not measured (that profiling query, or a test using `length`
|
|
345
|
+
on an array, fails and is reported as an error), and the facts extracted from the SQL (`dq_signals`) were tuned
|
|
346
|
+
on ClickHouse SQL.
|
|
347
|
+
|
|
348
|
+
## License
|
|
349
|
+
|
|
350
|
+
MIT
|