dbdocs 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbdocs-1.2.0/PKG-INFO +93 -0
- dbdocs-1.2.0/README.md +69 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/cli/main.py +16 -4
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/config.py +14 -0
- dbdocs-1.2.0/dbdocs/extract/health/__init__.py +12 -0
- dbdocs-1.2.0/dbdocs/extract/health/dimensions.py +262 -0
- dbdocs-1.2.0/dbdocs/extract/health/extractor.py +299 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/__init__.py +106 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/base.py +43 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/__init__.py +7 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/documentation.py +71 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/governance.py +75 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/modeling.py +241 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/performance.py +65 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/structure.py +79 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/testing.py +51 -0
- dbdocs-1.2.0/dbdocs/extract/health/rules/registry.py +155 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/builder.py +25 -1
- dbdocs-1.2.0/dbdocs/site/bundle/assets/css/icons.css +35 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/css/style.css +148 -3
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/graph/index.js +12 -12
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/data.js +9 -1
- dbdocs-1.2.0/dbdocs/site/bundle/assets/js/service.js +234 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/ui.js +388 -32
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/index.html +7 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/pyproject.toml +3 -1
- dbdocs-1.1.0/PKG-INFO +0 -78
- dbdocs-1.1.0/README.md +0 -56
- dbdocs-1.1.0/dbdocs/site/bundle/assets/js/service.js +0 -111
- {dbdocs-1.1.0 → dbdocs-1.2.0}/.gitignore +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/LICENSE +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/__init__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/__main__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/cli/__init__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/__init__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/artifacts.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/exceptions.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/log.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/__init__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/_sqlglot_lineage.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/column_lineage.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/erd.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/erd_json.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/graph.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/nodes.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/main.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/__init__.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/favicon.svg +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/graph/index.css +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/app.js +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/vendor/marked.min.js +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/vendor/minisearch.min.js +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/deploy.py +0 -0
- {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/inject.py +0 -0
dbdocs-1.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbdocs
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Alternative dbt docs site: Catalog + ERD + column-level lineage
|
|
5
|
+
Project-URL: Homepage, https://github.com/datnguye/dbt-docs
|
|
6
|
+
Project-URL: Repository, https://github.com/datnguye/dbt-docs
|
|
7
|
+
Author-email: Dat Nguyen <datnguyen.it09@gmail.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: dbt,dbterd,documentation,erd,lineage,sqlglot
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Software Development :: Documentation
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: artifact-parser[dbt]>=1.0.0
|
|
18
|
+
Requires-Dist: click>=8.1
|
|
19
|
+
Requires-Dist: dbterd>=1.27
|
|
20
|
+
Requires-Dist: pydantic>=2.13.4
|
|
21
|
+
Requires-Dist: pyyaml>=6.0
|
|
22
|
+
Requires-Dist: sqlglot>=25
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<img src="docs/assets/logo.svg" alt="dbdocs logo" width="220" height="88">
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
<p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage + versioned deploys, all in one CLI.</b></p>
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
33
|
+
<a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
|
|
34
|
+
<a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
|
|
35
|
+
<img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
|
|
36
|
+
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
|
|
37
|
+
<a href="https://www.python.org"><img src="https://img.shields.io/badge/Python-3.10|3.11|3.12|3.13-3776AB.svg?style=flat&logo=python&logoColor=white" alt="python"></a>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
Turn your dbt artifacts into a self-contained docs site: a browsable catalog, an entity-relationship diagram, an interactive lineage DAG, and **column-level lineage** traced from your compiled SQL — all in one `dbdocs generate`. Serve it with `dbdocs serve`, or deploy versioned builds anywhere a static host will take them.
|
|
41
|
+
|
|
42
|
+
| Catalog | Model page | Lineage DAG | Health Check |
|
|
43
|
+
|---|---|---|---|
|
|
44
|
+
|  |  |  |  |
|
|
45
|
+
|
|
46
|
+
## Why dbdocs?
|
|
47
|
+
|
|
48
|
+
dbt's built-in docs are great — right up until you want to know *which upstream column fed this downstream column*, or *which tables relate to each other*, or *what changed between last week's docs and today's*. dbdocs fills all three gaps without asking you to install a documentation framework or maintain a separate ERD tool.
|
|
49
|
+
|
|
50
|
+
**What you get that nothing else bundles together:**
|
|
51
|
+
|
|
52
|
+
- **ERD + column-level lineage, side by side.** The entity-relationship diagram (powered by [dbterd](https://github.com/datnguye/dbterd)) shows table relationships; column lineage (traced by [sqlglot](https://github.com/tobymao/sqlglot) from compiled SQL) shows exactly which column fed which. Most alternatives give you one or the other — dbdocs gives you both.
|
|
53
|
+
- **Column impact analysis.** Select any column and see its downstream dependents across the project, so you know what a schema change will break before you run it.
|
|
54
|
+
- **Deep-link URLs.** Every focused node, column, and filtered DAG view has a shareable URL. Paste it in Slack and your teammate lands on exactly the right model, column, or graph state.
|
|
55
|
+
- **Any sqlglot-supported dialect.** The dialect for column-lineage parsing is auto-detected from your manifest's `adapter_type` (Snowflake, BigQuery, Redshift, DuckDB, PostgreSQL, Databricks/Spark, Trino, and more — anything sqlglot understands). Override it per-project with `dialect:` in `dbdocs.yml` when auto-detection isn't enough.
|
|
56
|
+
- **Scales without freezing.** Column-lineage parsing fans out across CPU cores automatically above 500 models, so large projects finish in roughly the same wall-clock time as small ones. The DAG is windowed by React Flow, so a 1 000-model graph doesn't turn your browser into a space heater. The payload ships as an external gzip (`dbdocs-data.json.gz`, decompressed client-side) so `index.html` stays tiny regardless of project size.
|
|
57
|
+
- **Fail-soft.** One model with SQL sqlglot can't parse gets skipped and logged — it never sinks the whole generate run.
|
|
58
|
+
- **Project Health Check.** A scorecard across the six [dbt-project-evaluator](https://dbt-labs.github.io/dbt-project-evaluator/) dimensions (testing, modeling, documentation, structure, performance, governance), computed straight from your `manifest.json` — no extra dbt package, no warehouse. When a `run_results.json` is also present (any `dbt build`/`dbt test`; default `<target_dir>/run_results.json`, override with `--run-results`), each test additionally shows up as a pass/fail finding grouped by what it checks (integrity, referential, validity, business logic, freshness). dbdocs only reads the artifacts — it never runs dbt or touches your warehouse. Fail-soft: a missing `run_results.json` just drops the per-test detail; the dimensions still render.
|
|
59
|
+
- **Versioned deploys, no plugins.** `dbdocs deploy --version v1.2 --alias latest` generates into a plain directory tree, writes a `versions.json` index, and the SPA renders a version dropdown. No mike, no external tooling, no surprise dependencies.
|
|
60
|
+
- **Catalog navigation + client-side search.** Models, seeds, and snapshots grouped by database and schema; full-text search without a backend.
|
|
61
|
+
- **Dark / light theme.**
|
|
62
|
+
|
|
63
|
+
## Install
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install dbdocs --upgrade
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Requires Python 3.10+.
|
|
70
|
+
|
|
71
|
+
## Quickstart
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
dbt docs generate # writes target/manifest.json + target/catalog.json
|
|
75
|
+
dbdocs generate # builds ./site/ with index.html + dbdocs-data.json.gz
|
|
76
|
+
dbdocs serve # static http server on http://127.0.0.1:8000
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
|
|
80
|
+
|
|
81
|
+
Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
|
|
82
|
+
|
|
83
|
+
## Contributing
|
|
84
|
+
|
|
85
|
+
Contributions are welcome — bugs, features, docs, typos. See the **[Contributing Guide](https://dbdocs.datnguye.me/latest/nav/development/contributing-guide.html)**.
|
|
86
|
+
|
|
87
|
+
If dbdocs saves you some clicks, consider [buying me a coffee](https://www.buymeacoffee.com/datnguye).
|
|
88
|
+
|
|
89
|
+
<a href="https://www.buymeacoffee.com/datnguye"><img src="https://img.shields.io/badge/buy%20me%20a%20coffee-donate-yellow.svg?logo=buy-me-a-coffee&logoColor=white&labelColor=ff813f&style=for-the-badge" alt="buy me a coffee"></a>
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
[MIT](./LICENSE) © Dat Nguyen
|
dbdocs-1.2.0/README.md
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/logo.svg" alt="dbdocs logo" width="220" height="88">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage + versioned deploys, all in one CLI.</b></p>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
9
|
+
<a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
|
|
10
|
+
<a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
|
|
11
|
+
<img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
|
|
12
|
+
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
|
|
13
|
+
<a href="https://www.python.org"><img src="https://img.shields.io/badge/Python-3.10|3.11|3.12|3.13-3776AB.svg?style=flat&logo=python&logoColor=white" alt="python"></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
16
|
+
Turn your dbt artifacts into a self-contained docs site: a browsable catalog, an entity-relationship diagram, an interactive lineage DAG, and **column-level lineage** traced from your compiled SQL — all in one `dbdocs generate`. Serve it with `dbdocs serve`, or deploy versioned builds anywhere a static host will take them.
|
|
17
|
+
|
|
18
|
+
| Catalog | Model page | Lineage DAG | Health Check |
|
|
19
|
+
|---|---|---|---|
|
|
20
|
+
|  |  |  |  |
|
|
21
|
+
|
|
22
|
+
## Why dbdocs?
|
|
23
|
+
|
|
24
|
+
dbt's built-in docs are great — right up until you want to know *which upstream column fed this downstream column*, or *which tables relate to each other*, or *what changed between last week's docs and today's*. dbdocs fills all three gaps without asking you to install a documentation framework or maintain a separate ERD tool.
|
|
25
|
+
|
|
26
|
+
**What you get that nothing else bundles together:**
|
|
27
|
+
|
|
28
|
+
- **ERD + column-level lineage, side by side.** The entity-relationship diagram (powered by [dbterd](https://github.com/datnguye/dbterd)) shows table relationships; column lineage (traced by [sqlglot](https://github.com/tobymao/sqlglot) from compiled SQL) shows exactly which column fed which. Most alternatives give you one or the other — dbdocs gives you both.
|
|
29
|
+
- **Column impact analysis.** Select any column and see its downstream dependents across the project, so you know what a schema change will break before you run it.
|
|
30
|
+
- **Deep-link URLs.** Every focused node, column, and filtered DAG view has a shareable URL. Paste it in Slack and your teammate lands on exactly the right model, column, or graph state.
|
|
31
|
+
- **Any sqlglot-supported dialect.** The dialect for column-lineage parsing is auto-detected from your manifest's `adapter_type` (Snowflake, BigQuery, Redshift, DuckDB, PostgreSQL, Databricks/Spark, Trino, and more — anything sqlglot understands). Override it per-project with `dialect:` in `dbdocs.yml` when auto-detection isn't enough.
|
|
32
|
+
- **Scales without freezing.** Column-lineage parsing fans out across CPU cores automatically above 500 models, so large projects finish in roughly the same wall-clock time as small ones. The DAG is windowed by React Flow, so a 1 000-model graph doesn't turn your browser into a space heater. The payload ships as an external gzip (`dbdocs-data.json.gz`, decompressed client-side) so `index.html` stays tiny regardless of project size.
|
|
33
|
+
- **Fail-soft.** One model with SQL sqlglot can't parse gets skipped and logged — it never sinks the whole generate run.
|
|
34
|
+
- **Project Health Check.** A scorecard across the six [dbt-project-evaluator](https://dbt-labs.github.io/dbt-project-evaluator/) dimensions (testing, modeling, documentation, structure, performance, governance), computed straight from your `manifest.json` — no extra dbt package, no warehouse. When a `run_results.json` is also present (any `dbt build`/`dbt test`; default `<target_dir>/run_results.json`, override with `--run-results`), each test additionally shows up as a pass/fail finding grouped by what it checks (integrity, referential, validity, business logic, freshness). dbdocs only reads the artifacts — it never runs dbt or touches your warehouse. Fail-soft: a missing `run_results.json` just drops the per-test detail; the dimensions still render.
|
|
35
|
+
- **Versioned deploys, no plugins.** `dbdocs deploy --version v1.2 --alias latest` generates into a plain directory tree, writes a `versions.json` index, and the SPA renders a version dropdown. No mike, no external tooling, no surprise dependencies.
|
|
36
|
+
- **Catalog navigation + client-side search.** Models, seeds, and snapshots grouped by database and schema; full-text search without a backend.
|
|
37
|
+
- **Dark / light theme.**
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install dbdocs --upgrade
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Requires Python 3.10+.
|
|
46
|
+
|
|
47
|
+
## Quickstart
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
dbt docs generate # writes target/manifest.json + target/catalog.json
|
|
51
|
+
dbdocs generate # builds ./site/ with index.html + dbdocs-data.json.gz
|
|
52
|
+
dbdocs serve # static http server on http://127.0.0.1:8000
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
|
|
56
|
+
|
|
57
|
+
Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
|
|
58
|
+
|
|
59
|
+
## Contributing
|
|
60
|
+
|
|
61
|
+
Contributions are welcome — bugs, features, docs, typos. See the **[Contributing Guide](https://dbdocs.datnguye.me/latest/nav/development/contributing-guide.html)**.
|
|
62
|
+
|
|
63
|
+
If dbdocs saves you some clicks, consider [buying me a coffee](https://www.buymeacoffee.com/datnguye).
|
|
64
|
+
|
|
65
|
+
<a href="https://www.buymeacoffee.com/datnguye"><img src="https://img.shields.io/badge/buy%20me%20a%20coffee-donate-yellow.svg?logo=buy-me-a-coffee&logoColor=white&labelColor=ff813f&style=for-the-badge" alt="buy me a coffee"></a>
|
|
66
|
+
|
|
67
|
+
## License
|
|
68
|
+
|
|
69
|
+
[MIT](./LICENSE) © Dat Nguyen
|
|
@@ -26,7 +26,10 @@ __version__ = importlib.metadata.version("dbdocs")
|
|
|
26
26
|
@click.pass_context
|
|
27
27
|
def dbdocs(ctx, config_path):
|
|
28
28
|
"""Alternative dbt docs site: dbt docs + ERD + column-level lineage."""
|
|
29
|
-
|
|
29
|
+
# The version banner is useful for the build commands (generate/deploy) but
|
|
30
|
+
# is noise for the long-running `serve`, which prints its own start line.
|
|
31
|
+
if ctx.invoked_subcommand != "serve":
|
|
32
|
+
logger.info("Run with dbdocs==%s", __version__)
|
|
30
33
|
try:
|
|
31
34
|
ctx.obj = DbDocsConfig.load(config_path)
|
|
32
35
|
except DbDocsError as exc:
|
|
@@ -38,16 +41,25 @@ def dbdocs(ctx, config_path):
|
|
|
38
41
|
@click.option(
|
|
39
42
|
"--dialect", default=None, help="SQL dialect for column lineage (default: adapter_type)."
|
|
40
43
|
)
|
|
44
|
+
@click.option(
|
|
45
|
+
"--run-results",
|
|
46
|
+
default=None,
|
|
47
|
+
help=(
|
|
48
|
+
"Path to run_results.json for the Health Check (default: <target_dir>/run_results.json)."
|
|
49
|
+
),
|
|
50
|
+
)
|
|
41
51
|
@click.pass_obj
|
|
42
|
-
def generate(config: DbDocsConfig, output_dir, dialect):
|
|
52
|
+
def generate(config: DbDocsConfig, output_dir, dialect, run_results):
|
|
43
53
|
"""Build the site from dbt artifacts (served over HTTP; data loaded externally)."""
|
|
44
54
|
if dialect is not None:
|
|
45
55
|
config.dialect = dialect
|
|
56
|
+
if run_results is not None:
|
|
57
|
+
config.run_results = run_results
|
|
46
58
|
try:
|
|
47
|
-
|
|
59
|
+
ReportBuilder(config).generate(output_dir=output_dir)
|
|
48
60
|
except DbDocsError as exc:
|
|
49
61
|
raise click.ClickException(str(exc)) from exc
|
|
50
|
-
|
|
62
|
+
# The builder already logs "Generated site at <path> (N nodes, M edges)".
|
|
51
63
|
|
|
52
64
|
|
|
53
65
|
@dbdocs.command(name="serve")
|
|
@@ -69,6 +69,18 @@ class DbDocsConfig:
|
|
|
69
69
|
#: ``resource_type``, …) passed straight to ``DbtErd``. Configured here so the
|
|
70
70
|
#: ERD shape lives in ``dbdocs.yml`` rather than a separate ``.dbterd.yml``.
|
|
71
71
|
dbterd: dict = field(default_factory=dict)
|
|
72
|
+
#: Path to the ``run_results.json`` parsed for the Health Check (always on).
|
|
73
|
+
#: If ``None`` (the default), ``<target_dir>/run_results.json`` is used. A
|
|
74
|
+
#: relative path is resolved against the cwd; a path that escapes the cwd is
|
|
75
|
+
#: silently ignored (fail-soft, like ``readme`` / ``logo``). When the file is
|
|
76
|
+
#: absent the Health Check page is simply empty.
|
|
77
|
+
run_results: "str | None" = None
|
|
78
|
+
#: Health Check rule engine config (all optional):
|
|
79
|
+
#: ``thresholds`` (e.g. ``model_fanout: 5``), ``disable`` (rule names to skip),
|
|
80
|
+
#: ``disable_dimensions`` (whole dimensions to skip), and ``rules_module`` (a
|
|
81
|
+
#: dotted import path whose ``register_rule`` calls add custom rules). Entry
|
|
82
|
+
#: points under ``dbdocs.health_rules`` are also discovered automatically.
|
|
83
|
+
health: dict = field(default_factory=dict)
|
|
72
84
|
|
|
73
85
|
@classmethod
|
|
74
86
|
def load(cls, path: "str | Path | None" = None) -> "DbDocsConfig":
|
|
@@ -110,6 +122,8 @@ class DbDocsConfig:
|
|
|
110
122
|
"readme",
|
|
111
123
|
"logo",
|
|
112
124
|
"favicon",
|
|
125
|
+
"run_results",
|
|
126
|
+
"health",
|
|
113
127
|
)
|
|
114
128
|
|
|
115
129
|
def render_context(self) -> dict:
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Project Health Check — the six dbt-project-evaluator dimensions.
|
|
2
|
+
|
|
3
|
+
The public entry point is :class:`HealthCheckExtractor`, which composes the
|
|
4
|
+
manifest-derived rule engine (:mod:`dbdocs.extract.health.dimensions` +
|
|
5
|
+
:mod:`dbdocs.extract.health.rules`) with the optional ``run_results.json``
|
|
6
|
+
per-test detail. Custom rules can be registered via
|
|
7
|
+
:func:`dbdocs.extract.health.rules.register_rule` (see the plugin docs).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from dbdocs.extract.health.extractor import HealthCheckExtractor
|
|
11
|
+
|
|
12
|
+
__all__ = ["HealthCheckExtractor"]
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""Build the manifest-derived health *dimensions* (the DPE rule engine's host).
|
|
2
|
+
|
|
3
|
+
``ManifestGraph`` indexes the dbt manifest once — parents/children adjacency,
|
|
4
|
+
plus layer / materialization / access / contract helpers — and the rule functions
|
|
5
|
+
in :mod:`dbdocs.extract.health.rules` run against it. ``DimensionAnalyzer.analyze()``
|
|
6
|
+
runs every rule, groups findings by dimension, and returns the ``dimensions``
|
|
7
|
+
block of the health data dict:
|
|
8
|
+
|
|
9
|
+
{
|
|
10
|
+
"modeling": {"issues": N, "checked": M, "findings": [...]},
|
|
11
|
+
"documentation": {...}, "structure": {...}, "performance": {...},
|
|
12
|
+
"governance": {...},
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
``checked`` is how many nodes the dimension's rules could apply to (models for
|
|
16
|
+
most, models+sources for some) — it gives the SPA a denominator for a score.
|
|
17
|
+
Everything is pure + DOM-free; a malformed/empty manifest yields empty dimensions
|
|
18
|
+
rather than raising (the caller, ``HealthCheckExtractor``, is fail-soft).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from dbdocs.core.log import logger
|
|
24
|
+
from dbdocs.extract.health.rules import (
|
|
25
|
+
DEFAULT_THRESHOLDS,
|
|
26
|
+
DIMENSION_RULES,
|
|
27
|
+
NON_PHYSICAL,
|
|
28
|
+
load_entry_point_rules,
|
|
29
|
+
load_rules_module,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ManifestGraph:
|
|
34
|
+
"""Adjacency + node metadata over a dbt manifest, for the health rules.
|
|
35
|
+
|
|
36
|
+
Wraps the dbterd-parsed manifest: ``models``/``sources``/``exposures`` lists,
|
|
37
|
+
a ``parents``/``children`` adjacency built from each node's ``depends_on``, and
|
|
38
|
+
accessors that smooth over where dbt stores a field (``config.materialized``
|
|
39
|
+
vs node, ``config.access`` vs node, ``schema_`` alias, …).
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, manifest: "Any | None", thresholds: "dict | None" = None) -> None:
|
|
43
|
+
nodes = getattr(manifest, "nodes", None)
|
|
44
|
+
self._nodes: dict = nodes if isinstance(nodes, dict) else {}
|
|
45
|
+
sources = getattr(manifest, "sources", None)
|
|
46
|
+
self._sources: dict = sources if isinstance(sources, dict) else {}
|
|
47
|
+
exposures = getattr(manifest, "exposures", None)
|
|
48
|
+
self._exposures: dict = exposures if isinstance(exposures, dict) else {}
|
|
49
|
+
|
|
50
|
+
self.models = [n for uid, n in self._nodes.items() if uid.startswith("model.")]
|
|
51
|
+
self.sources = list(self._sources.values())
|
|
52
|
+
self.exposures = list(self._exposures.values())
|
|
53
|
+
|
|
54
|
+
# Rule thresholds: per-run overrides layered over the DPE defaults.
|
|
55
|
+
self._thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
|
|
56
|
+
|
|
57
|
+
self._parents: dict[str, list[str]] = {}
|
|
58
|
+
self._children: dict[str, list[str]] = {}
|
|
59
|
+
self._build_adjacency()
|
|
60
|
+
|
|
61
|
+
self._tests_by_model: dict[str, set] = {}
|
|
62
|
+
self._index_tests()
|
|
63
|
+
|
|
64
|
+
# Memo for non_physical_chain_depth (O(nodes) across the whole pass).
|
|
65
|
+
self._chain_cache: dict[str, int] = {}
|
|
66
|
+
|
|
67
|
+
def threshold(self, name: str) -> "int | float":
|
|
68
|
+
"""The configured threshold for *name* (DPE default when unset)."""
|
|
69
|
+
return self._thresholds[name]
|
|
70
|
+
|
|
71
|
+
def _build_adjacency(self) -> None:
|
|
72
|
+
"""Index parent/child edges from every node + exposure ``depends_on.nodes``."""
|
|
73
|
+
everything = list(self._nodes.values()) + self.exposures
|
|
74
|
+
for node in everything:
|
|
75
|
+
uid = str(getattr(node, "unique_id", "") or "")
|
|
76
|
+
if not uid:
|
|
77
|
+
continue
|
|
78
|
+
depends_on = getattr(node, "depends_on", None)
|
|
79
|
+
parents = list(getattr(depends_on, "nodes", None) or [])
|
|
80
|
+
self._parents[uid] = parents
|
|
81
|
+
for parent in parents:
|
|
82
|
+
self._children.setdefault(parent, []).append(uid)
|
|
83
|
+
|
|
84
|
+
def _index_tests(self) -> None:
|
|
85
|
+
"""Map each tested node to the set of test types attached to it."""
|
|
86
|
+
for uid, node in self._nodes.items():
|
|
87
|
+
if not uid.startswith("test."):
|
|
88
|
+
continue
|
|
89
|
+
attached = str(getattr(node, "attached_node", "") or "")
|
|
90
|
+
if not attached:
|
|
91
|
+
continue
|
|
92
|
+
metadata = getattr(node, "test_metadata", None)
|
|
93
|
+
test_type = str(getattr(metadata, "name", "") or "") if metadata else "singular"
|
|
94
|
+
self._tests_by_model.setdefault(attached, set()).add(test_type)
|
|
95
|
+
|
|
96
|
+
def tests_for(self, unique_id: str) -> set:
|
|
97
|
+
"""The set of test types attached to *unique_id* (empty if untested)."""
|
|
98
|
+
return self._tests_by_model.get(unique_id, set())
|
|
99
|
+
|
|
100
|
+
# -- adjacency -----------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
def parents(self, unique_id: str) -> "list[str]":
|
|
103
|
+
return self._parents.get(unique_id, [])
|
|
104
|
+
|
|
105
|
+
def children(self, unique_id: str) -> "list[str]":
|
|
106
|
+
return self._children.get(unique_id, [])
|
|
107
|
+
|
|
108
|
+
def node(self, unique_id: str) -> "Any | None":
|
|
109
|
+
"""The node object for *unique_id* (model/seed/snapshot or source)."""
|
|
110
|
+
return self._nodes.get(unique_id) or self._sources.get(unique_id)
|
|
111
|
+
|
|
112
|
+
# -- node metadata -------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def layer(model: Any) -> str:
|
|
116
|
+
"""The model's layer (staging/intermediate/marts/other) from its ``fqn``.
|
|
117
|
+
|
|
118
|
+
dbt's ``fqn`` is ``[package, ...folders, name]`` — the folder segments name
|
|
119
|
+
the layer. We match the known layer folders anywhere in the fqn so a
|
|
120
|
+
``marts/finance/`` nesting still resolves to ``marts``.
|
|
121
|
+
"""
|
|
122
|
+
fqn = [str(p).lower() for p in (getattr(model, "fqn", None) or [])]
|
|
123
|
+
for layer in ("staging", "intermediate", "marts"):
|
|
124
|
+
if layer in fqn:
|
|
125
|
+
return layer
|
|
126
|
+
# dbt also uses "stg"/"int"/"core"/"mart" folder spellings; map the common ones.
|
|
127
|
+
if "stg" in fqn:
|
|
128
|
+
return "staging"
|
|
129
|
+
if "int" in fqn:
|
|
130
|
+
return "intermediate"
|
|
131
|
+
if "mart" in fqn or "marts" in fqn:
|
|
132
|
+
return "marts"
|
|
133
|
+
return "other"
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def materialization(node: Any) -> str:
|
|
137
|
+
"""The node's materialization (``config.materialized``), lowercased."""
|
|
138
|
+
config = getattr(node, "config", None)
|
|
139
|
+
materialized = getattr(config, "materialized", None) if config else None
|
|
140
|
+
return str(materialized or "").lower()
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def access(model: Any) -> str:
|
|
144
|
+
"""The model's access level (``config.access`` or node ``access``)."""
|
|
145
|
+
config = getattr(model, "config", None)
|
|
146
|
+
access = getattr(config, "access", None) if config else None
|
|
147
|
+
access = access or getattr(model, "access", None)
|
|
148
|
+
return str(access or "protected").lower()
|
|
149
|
+
|
|
150
|
+
@staticmethod
|
|
151
|
+
def contract_enforced(model: Any) -> bool:
|
|
152
|
+
"""Whether the model has an enforced contract (``contract.enforced``)."""
|
|
153
|
+
contract = getattr(model, "contract", None)
|
|
154
|
+
return bool(getattr(contract, "enforced", False)) if contract else False
|
|
155
|
+
|
|
156
|
+
def non_physical_chain_depth(self, unique_id: str) -> int:
|
|
157
|
+
"""Longest chain of consecutive non-physical (view/ephemeral) ancestors + self.
|
|
158
|
+
|
|
159
|
+
Counts this model plus the longest run of view/ephemeral models directly
|
|
160
|
+
upstream; a table/source ancestor (or a model boundary) stops the chain.
|
|
161
|
+
Mirrors DPE's ``chained_view_dependencies`` distance.
|
|
162
|
+
|
|
163
|
+
Iterative + memoized: depth is computed once per node and cached on the
|
|
164
|
+
graph (``_chain_cache``), so the whole pass is ``O(nodes)`` even on a
|
|
165
|
+
3000-deep view chain — recursion here would blow Python's stack limit and
|
|
166
|
+
be ``O(nodes²)``.
|
|
167
|
+
"""
|
|
168
|
+
cache = self._chain_cache
|
|
169
|
+
if unique_id in cache:
|
|
170
|
+
return cache[unique_id]
|
|
171
|
+
# Post-order DFS via an explicit stack: push a node, then its unresolved
|
|
172
|
+
# non-physical parents; resolve a node once all its parents are cached.
|
|
173
|
+
stack = [unique_id]
|
|
174
|
+
on_stack = {unique_id}
|
|
175
|
+
while stack:
|
|
176
|
+
uid = stack[-1]
|
|
177
|
+
node = self._nodes.get(uid)
|
|
178
|
+
if node is None or self.materialization(node) not in NON_PHYSICAL:
|
|
179
|
+
cache[uid] = 0
|
|
180
|
+
stack.pop()
|
|
181
|
+
on_stack.discard(uid)
|
|
182
|
+
continue
|
|
183
|
+
best = 0
|
|
184
|
+
pending = None
|
|
185
|
+
for parent in self.parents(uid):
|
|
186
|
+
if parent in cache:
|
|
187
|
+
best = max(best, cache[parent])
|
|
188
|
+
elif parent not in on_stack: # unresolved, not already in progress
|
|
189
|
+
pending = parent
|
|
190
|
+
break
|
|
191
|
+
# parent in on_stack ⇒ a cycle; treat as a stop (contributes 0)
|
|
192
|
+
if pending is not None:
|
|
193
|
+
stack.append(pending)
|
|
194
|
+
on_stack.add(pending)
|
|
195
|
+
continue
|
|
196
|
+
cache[uid] = 1 + best
|
|
197
|
+
stack.pop()
|
|
198
|
+
on_stack.discard(uid)
|
|
199
|
+
return cache[unique_id]
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class DimensionAnalyzer:
|
|
203
|
+
"""Run every manifest-derived health rule and group findings by dimension.
|
|
204
|
+
|
|
205
|
+
*config* is the optional ``health`` block from ``dbdocs.yml``:
|
|
206
|
+
``thresholds`` (per-rule overrides), ``disable`` (rule names to skip),
|
|
207
|
+
``disable_dimensions`` (whole dimensions to skip), and ``rules_module`` (a
|
|
208
|
+
dotted path whose ``register_rule`` calls add custom rules). Entry-point
|
|
209
|
+
plugins under ``dbdocs.health_rules`` are always discovered.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
def __init__(self, manifest: "Any | None", config: "dict | None" = None) -> None:
|
|
213
|
+
config = config or {}
|
|
214
|
+
thresholds = config.get("thresholds") if isinstance(config.get("thresholds"), dict) else {}
|
|
215
|
+
self._graph = ManifestGraph(manifest, thresholds=thresholds)
|
|
216
|
+
self._disabled_rules = set(config.get("disable") or [])
|
|
217
|
+
self._disabled_dimensions = set(config.get("disable_dimensions") or [])
|
|
218
|
+
|
|
219
|
+
# Plugin loading: entry points always; an explicit rules_module on request.
|
|
220
|
+
load_entry_point_rules()
|
|
221
|
+
rules_module = config.get("rules_module")
|
|
222
|
+
if rules_module:
|
|
223
|
+
load_rules_module(str(rules_module))
|
|
224
|
+
|
|
225
|
+
def analyze(self) -> dict:
|
|
226
|
+
"""Return the ``dimensions`` block: one entry per (enabled) dimension."""
|
|
227
|
+
graph = self._graph
|
|
228
|
+
model_count = len(graph.models)
|
|
229
|
+
source_count = len(graph.sources)
|
|
230
|
+
dimensions: dict = {}
|
|
231
|
+
for dimension, rules in DIMENSION_RULES.items():
|
|
232
|
+
if dimension in self._disabled_dimensions:
|
|
233
|
+
continue
|
|
234
|
+
findings: list = []
|
|
235
|
+
for rule in rules:
|
|
236
|
+
if getattr(rule, "__name__", "") in self._disabled_rules:
|
|
237
|
+
continue
|
|
238
|
+
try:
|
|
239
|
+
findings.extend(rule(graph))
|
|
240
|
+
except (AttributeError, TypeError, ValueError, KeyError, RecursionError) as exc:
|
|
241
|
+
# A single malformed node / bad plugin must never sink the report.
|
|
242
|
+
logger.warning(
|
|
243
|
+
"Health dimension %s: rule %s failed: %s — skipping.",
|
|
244
|
+
dimension,
|
|
245
|
+
getattr(rule, "__name__", rule),
|
|
246
|
+
exc,
|
|
247
|
+
)
|
|
248
|
+
dimensions[dimension] = {
|
|
249
|
+
"issues": len(findings),
|
|
250
|
+
"checked": self._checked(dimension, model_count, source_count),
|
|
251
|
+
"findings": findings,
|
|
252
|
+
}
|
|
253
|
+
return dimensions
|
|
254
|
+
|
|
255
|
+
@staticmethod
|
|
256
|
+
def _checked(dimension: str, model_count: int, source_count: int) -> int:
|
|
257
|
+
"""How many nodes a dimension's rules could apply to (the score denominator)."""
|
|
258
|
+
# Modeling/documentation/structure span models + sources; performance and
|
|
259
|
+
# governance are model/exposure-centric — count models as the denominator.
|
|
260
|
+
if dimension in ("modeling", "documentation", "structure"):
|
|
261
|
+
return model_count + source_count
|
|
262
|
+
return model_count
|