dbdocs 1.1.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. dbdocs-1.2.0/PKG-INFO +93 -0
  2. dbdocs-1.2.0/README.md +69 -0
  3. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/cli/main.py +16 -4
  4. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/config.py +14 -0
  5. dbdocs-1.2.0/dbdocs/extract/health/__init__.py +12 -0
  6. dbdocs-1.2.0/dbdocs/extract/health/dimensions.py +262 -0
  7. dbdocs-1.2.0/dbdocs/extract/health/extractor.py +299 -0
  8. dbdocs-1.2.0/dbdocs/extract/health/rules/__init__.py +106 -0
  9. dbdocs-1.2.0/dbdocs/extract/health/rules/base.py +43 -0
  10. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/__init__.py +7 -0
  11. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/documentation.py +71 -0
  12. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/governance.py +75 -0
  13. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/modeling.py +241 -0
  14. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/performance.py +65 -0
  15. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/structure.py +79 -0
  16. dbdocs-1.2.0/dbdocs/extract/health/rules/dimensions/testing.py +51 -0
  17. dbdocs-1.2.0/dbdocs/extract/health/rules/registry.py +155 -0
  18. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/builder.py +25 -1
  19. dbdocs-1.2.0/dbdocs/site/bundle/assets/css/icons.css +35 -0
  20. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/css/style.css +148 -3
  21. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/graph/index.js +12 -12
  22. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/data.js +9 -1
  23. dbdocs-1.2.0/dbdocs/site/bundle/assets/js/service.js +234 -0
  24. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/ui.js +388 -32
  25. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/index.html +7 -0
  26. {dbdocs-1.1.0 → dbdocs-1.2.0}/pyproject.toml +3 -1
  27. dbdocs-1.1.0/PKG-INFO +0 -78
  28. dbdocs-1.1.0/README.md +0 -56
  29. dbdocs-1.1.0/dbdocs/site/bundle/assets/js/service.js +0 -111
  30. {dbdocs-1.1.0 → dbdocs-1.2.0}/.gitignore +0 -0
  31. {dbdocs-1.1.0 → dbdocs-1.2.0}/LICENSE +0 -0
  32. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/__init__.py +0 -0
  33. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/__main__.py +0 -0
  34. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/cli/__init__.py +0 -0
  35. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/__init__.py +0 -0
  36. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/artifacts.py +0 -0
  37. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/exceptions.py +0 -0
  38. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/core/log.py +0 -0
  39. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/__init__.py +0 -0
  40. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/_sqlglot_lineage.py +0 -0
  41. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/column_lineage.py +0 -0
  42. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/erd.py +0 -0
  43. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/erd_json.py +0 -0
  44. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/graph.py +0 -0
  45. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/extract/nodes.py +0 -0
  46. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/main.py +0 -0
  47. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/__init__.py +0 -0
  48. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/favicon.svg +0 -0
  49. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/graph/index.css +0 -0
  50. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/js/app.js +0 -0
  51. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/vendor/marked.min.js +0 -0
  52. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/bundle/assets/vendor/minisearch.min.js +0 -0
  53. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/deploy.py +0 -0
  54. {dbdocs-1.1.0 → dbdocs-1.2.0}/dbdocs/site/inject.py +0 -0
dbdocs-1.2.0/PKG-INFO ADDED
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: dbdocs
3
+ Version: 1.2.0
4
+ Summary: Alternative dbt docs site: Catalog + ERD + column-level lineage
5
+ Project-URL: Homepage, https://github.com/datnguye/dbt-docs
6
+ Project-URL: Repository, https://github.com/datnguye/dbt-docs
7
+ Author-email: Dat Nguyen <datnguyen.it09@gmail.com>
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: dbt,dbterd,documentation,erd,lineage,sqlglot
11
+ Classifier: Environment :: Console
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Topic :: Software Development :: Documentation
14
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
+ Classifier: Topic :: Software Development :: Quality Assurance
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: artifact-parser[dbt]>=1.0.0
18
+ Requires-Dist: click>=8.1
19
+ Requires-Dist: dbterd>=1.27
20
+ Requires-Dist: pydantic>=2.13.4
21
+ Requires-Dist: pyyaml>=6.0
22
+ Requires-Dist: sqlglot>=25
23
+ Description-Content-Type: text/markdown
24
+
25
+ <p align="center">
26
+ <img src="docs/assets/logo.svg" alt="dbdocs logo" width="220" height="88">
27
+ </p>
28
+
29
+ <p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage + versioned deploys, all in one CLI.</b></p>
30
+
31
+ <p align="center">
32
+ <a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
33
+ <a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
34
+ <a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
35
+ <img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
36
+ <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
37
+ <a href="https://www.python.org"><img src="https://img.shields.io/badge/Python-3.10|3.11|3.12|3.13-3776AB.svg?style=flat&logo=python&logoColor=white" alt="python"></a>
38
+ </p>
39
+
40
+ Turn your dbt artifacts into a self-contained docs site: a browsable catalog, an entity-relationship diagram, an interactive lineage DAG, and **column-level lineage** traced from your compiled SQL — all in one `dbdocs generate`. Serve it with `dbdocs serve`, or deploy versioned builds anywhere a static host will take them.
41
+
42
+ | Catalog | Model page | Lineage DAG | Health Check |
43
+ |---|---|---|---|
44
+ | ![catalog](docs/assets/img/demo-catalog.png) | ![model page](docs/assets/img/demo-model-page.png) | ![dag](docs/assets/img/demo-model-dag.png) | ![health check](docs/assets/img/demo-health.png) |
45
+
46
+ ## Why dbdocs?
47
+
48
+ dbt's built-in docs are great — right up until you want to know *which upstream column fed this downstream column*, or *which tables relate to each other*, or *what changed between last week's docs and today's*. dbdocs fills all three gaps without asking you to install a documentation framework or maintain a separate ERD tool.
49
+
50
+ **What you get that nothing else bundles together:**
51
+
52
+ - **ERD + column-level lineage, side by side.** The entity-relationship diagram (powered by [dbterd](https://github.com/datnguye/dbterd)) shows table relationships; column lineage (traced by [sqlglot](https://github.com/tobymao/sqlglot) from compiled SQL) shows exactly which column fed which. Most alternatives give you one or the other — dbdocs gives you both.
53
+ - **Column impact analysis.** Select any column and see its downstream dependents across the project, so you know what a schema change will break before you run it.
54
+ - **Deep-link URLs.** Every focused node, column, and filtered DAG view has a shareable URL. Paste it in Slack and your teammate lands on exactly the right model, column, or graph state.
55
+ - **Any sqlglot-supported dialect.** The dialect for column-lineage parsing is auto-detected from your manifest's `adapter_type` (Snowflake, BigQuery, Redshift, DuckDB, PostgreSQL, Databricks/Spark, Trino, and more — anything sqlglot understands). Override it per-project with `dialect:` in `dbdocs.yml` when auto-detection isn't enough.
56
+ - **Scales without freezing.** Column-lineage parsing fans out across CPU cores automatically above 500 models, so large projects finish in roughly the same wall-clock time as small ones. The DAG is windowed by React Flow, so a 1 000-model graph doesn't turn your browser into a space heater. The payload ships as an external gzip (`dbdocs-data.json.gz`, decompressed client-side) so `index.html` stays tiny regardless of project size.
57
+ - **Fail-soft.** One model with SQL sqlglot can't parse gets skipped and logged — it never sinks the whole generate run.
58
+ - **Project Health Check.** A scorecard across the six [dbt-project-evaluator](https://dbt-labs.github.io/dbt-project-evaluator/) dimensions (testing, modeling, documentation, structure, performance, governance), computed straight from your `manifest.json` — no extra dbt package, no warehouse. When a `run_results.json` is also present (any `dbt build`/`dbt test`; default `<target_dir>/run_results.json`, override with `--run-results`), each test additionally shows up as a pass/fail finding grouped by what it checks (integrity, referential, validity, business logic, freshness). dbdocs only reads the artifacts — it never runs dbt or touches your warehouse. Fail-soft: a missing `run_results.json` just drops the per-test detail; the dimensions still render.
59
+ - **Versioned deploys, no plugins.** `dbdocs deploy --version v1.2 --alias latest` generates into a plain directory tree, writes a `versions.json` index, and the SPA renders a version dropdown. No mike, no external tooling, no surprise dependencies.
60
+ - **Catalog navigation + client-side search.** Models, seeds, and snapshots grouped by database and schema; full-text search without a backend.
61
+ - **Dark / light theme.**
62
+
63
+ ## Install
64
+
65
+ ```bash
66
+ pip install dbdocs --upgrade
67
+ ```
68
+
69
+ Requires Python 3.10+.
70
+
71
+ ## Quickstart
72
+
73
+ ```bash
74
+ dbt docs generate # writes target/manifest.json + target/catalog.json
75
+ dbdocs generate # builds ./site/ with index.html + dbdocs-data.json.gz
76
+ dbdocs serve # static http server on http://127.0.0.1:8000
77
+ ```
78
+
79
+ The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
80
+
81
+ Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
82
+
83
+ ## Contributing
84
+
85
+ Contributions are welcome — bugs, features, docs, typos. See the **[Contributing Guide](https://dbdocs.datnguye.me/latest/nav/development/contributing-guide.html)**.
86
+
87
+ If dbdocs saves you some clicks, consider [buying me a coffee](https://www.buymeacoffee.com/datnguye).
88
+
89
+ <a href="https://www.buymeacoffee.com/datnguye"><img src="https://img.shields.io/badge/buy%20me%20a%20coffee-donate-yellow.svg?logo=buy-me-a-coffee&logoColor=white&labelColor=ff813f&style=for-the-badge" alt="buy me a coffee"></a>
90
+
91
+ ## License
92
+
93
+ [MIT](./LICENSE) © Dat Nguyen
dbdocs-1.2.0/README.md ADDED
@@ -0,0 +1,69 @@
1
+ <p align="center">
2
+ <img src="docs/assets/logo.svg" alt="dbdocs logo" width="220" height="88">
3
+ </p>
4
+
5
+ <p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage + versioned deploys, all in one CLI.</b></p>
6
+
7
+ <p align="center">
8
+ <a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
9
+ <a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
10
+ <a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
11
+ <img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
12
+ <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
13
+ <a href="https://www.python.org"><img src="https://img.shields.io/badge/Python-3.10|3.11|3.12|3.13-3776AB.svg?style=flat&logo=python&logoColor=white" alt="python"></a>
14
+ </p>
15
+
16
+ Turn your dbt artifacts into a self-contained docs site: a browsable catalog, an entity-relationship diagram, an interactive lineage DAG, and **column-level lineage** traced from your compiled SQL — all in one `dbdocs generate`. Serve it with `dbdocs serve`, or deploy versioned builds anywhere a static host will take them.
17
+
18
+ | Catalog | Model page | Lineage DAG | Health Check |
19
+ |---|---|---|---|
20
+ | ![catalog](docs/assets/img/demo-catalog.png) | ![model page](docs/assets/img/demo-model-page.png) | ![dag](docs/assets/img/demo-model-dag.png) | ![health check](docs/assets/img/demo-health.png) |
21
+
22
+ ## Why dbdocs?
23
+
24
+ dbt's built-in docs are great — right up until you want to know *which upstream column fed this downstream column*, or *which tables relate to each other*, or *what changed between last week's docs and today's*. dbdocs fills all three gaps without asking you to install a documentation framework or maintain a separate ERD tool.
25
+
26
+ **What you get that nothing else bundles together:**
27
+
28
+ - **ERD + column-level lineage, side by side.** The entity-relationship diagram (powered by [dbterd](https://github.com/datnguye/dbterd)) shows table relationships; column lineage (traced by [sqlglot](https://github.com/tobymao/sqlglot) from compiled SQL) shows exactly which column fed which. Most alternatives give you one or the other — dbdocs gives you both.
29
+ - **Column impact analysis.** Select any column and see its downstream dependents across the project, so you know what a schema change will break before you run it.
30
+ - **Deep-link URLs.** Every focused node, column, and filtered DAG view has a shareable URL. Paste it in Slack and your teammate lands on exactly the right model, column, or graph state.
31
+ - **Any sqlglot-supported dialect.** The dialect for column-lineage parsing is auto-detected from your manifest's `adapter_type` (Snowflake, BigQuery, Redshift, DuckDB, PostgreSQL, Databricks/Spark, Trino, and more — anything sqlglot understands). Override it per-project with `dialect:` in `dbdocs.yml` when auto-detection isn't enough.
32
+ - **Scales without freezing.** Column-lineage parsing fans out across CPU cores automatically above 500 models, so large projects finish in roughly the same wall-clock time as small ones. The DAG is windowed by React Flow, so a 1 000-model graph doesn't turn your browser into a space heater. The payload ships as an external gzip (`dbdocs-data.json.gz`, decompressed client-side) so `index.html` stays tiny regardless of project size.
33
+ - **Fail-soft.** One model with SQL sqlglot can't parse gets skipped and logged — it never sinks the whole generate run.
34
+ - **Project Health Check.** A scorecard across the six [dbt-project-evaluator](https://dbt-labs.github.io/dbt-project-evaluator/) dimensions (testing, modeling, documentation, structure, performance, governance), computed straight from your `manifest.json` — no extra dbt package, no warehouse. When a `run_results.json` is also present (any `dbt build`/`dbt test`; default `<target_dir>/run_results.json`, override with `--run-results`), each test additionally shows up as a pass/fail finding grouped by what it checks (integrity, referential, validity, business logic, freshness). dbdocs only reads the artifacts — it never runs dbt or touches your warehouse. Fail-soft: a missing `run_results.json` just drops the per-test detail; the dimensions still render.
35
+ - **Versioned deploys, no plugins.** `dbdocs deploy --version v1.2 --alias latest` generates into a plain directory tree, writes a `versions.json` index, and the SPA renders a version dropdown. No mike, no external tooling, no surprise dependencies.
36
+ - **Catalog navigation + client-side search.** Models, seeds, and snapshots grouped by database and schema; full-text search without a backend.
37
+ - **Dark / light theme.**
38
+
39
+ ## Install
40
+
41
+ ```bash
42
+ pip install dbdocs --upgrade
43
+ ```
44
+
45
+ Requires Python 3.10+.
46
+
47
+ ## Quickstart
48
+
49
+ ```bash
50
+ dbt docs generate # writes target/manifest.json + target/catalog.json
51
+ dbdocs generate # builds ./site/ with index.html + dbdocs-data.json.gz
52
+ dbdocs serve # static http server on http://127.0.0.1:8000
53
+ ```
54
+
55
+ The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
56
+
57
+ Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
58
+
59
+ ## Contributing
60
+
61
+ Contributions are welcome — bugs, features, docs, typos. See the **[Contributing Guide](https://dbdocs.datnguye.me/latest/nav/development/contributing-guide.html)**.
62
+
63
+ If dbdocs saves you some clicks, consider [buying me a coffee](https://www.buymeacoffee.com/datnguye).
64
+
65
+ <a href="https://www.buymeacoffee.com/datnguye"><img src="https://img.shields.io/badge/buy%20me%20a%20coffee-donate-yellow.svg?logo=buy-me-a-coffee&logoColor=white&labelColor=ff813f&style=for-the-badge" alt="buy me a coffee"></a>
66
+
67
+ ## License
68
+
69
+ [MIT](./LICENSE) © Dat Nguyen
@@ -26,7 +26,10 @@ __version__ = importlib.metadata.version("dbdocs")
26
26
  @click.pass_context
27
27
  def dbdocs(ctx, config_path):
28
28
  """Alternative dbt docs site: dbt docs + ERD + column-level lineage."""
29
- logger.info("Run with dbdocs==%s", __version__)
29
+ # The version banner is useful for the build commands (generate/deploy) but
30
+ # is noise for the long-running `serve`, which prints its own start line.
31
+ if ctx.invoked_subcommand != "serve":
32
+ logger.info("Run with dbdocs==%s", __version__)
30
33
  try:
31
34
  ctx.obj = DbDocsConfig.load(config_path)
32
35
  except DbDocsError as exc:
@@ -38,16 +41,25 @@ def dbdocs(ctx, config_path):
38
41
  @click.option(
39
42
  "--dialect", default=None, help="SQL dialect for column lineage (default: adapter_type)."
40
43
  )
44
+ @click.option(
45
+ "--run-results",
46
+ default=None,
47
+ help=(
48
+ "Path to run_results.json for the Health Check (default: <target_dir>/run_results.json)."
49
+ ),
50
+ )
41
51
  @click.pass_obj
42
- def generate(config: DbDocsConfig, output_dir, dialect):
52
+ def generate(config: DbDocsConfig, output_dir, dialect, run_results):
43
53
  """Build the site from dbt artifacts (served over HTTP; data loaded externally)."""
44
54
  if dialect is not None:
45
55
  config.dialect = dialect
56
+ if run_results is not None:
57
+ config.run_results = run_results
46
58
  try:
47
- out = ReportBuilder(config).generate(output_dir=output_dir)
59
+ ReportBuilder(config).generate(output_dir=output_dir)
48
60
  except DbDocsError as exc:
49
61
  raise click.ClickException(str(exc)) from exc
50
- click.echo(f"Generated site into {out}")
62
+ # The builder already logs "Generated site at <path> (N nodes, M edges)".
51
63
 
52
64
 
53
65
  @dbdocs.command(name="serve")
@@ -69,6 +69,18 @@ class DbDocsConfig:
69
69
  #: ``resource_type``, …) passed straight to ``DbtErd``. Configured here so the
70
70
  #: ERD shape lives in ``dbdocs.yml`` rather than a separate ``.dbterd.yml``.
71
71
  dbterd: dict = field(default_factory=dict)
72
+ #: Path to the ``run_results.json`` parsed for the Health Check (always on).
73
+ #: If ``None`` (the default), ``<target_dir>/run_results.json`` is used. A
74
+ #: relative path is resolved against the cwd; a path that escapes the cwd is
75
+ #: silently ignored (fail-soft, like ``readme`` / ``logo``). When the file is
76
+ #: absent the Health Check page is simply empty.
77
+ run_results: "str | None" = None
78
+ #: Health Check rule engine config (all optional):
79
+ #: ``thresholds`` (e.g. ``model_fanout: 5``), ``disable`` (rule names to skip),
80
+ #: ``disable_dimensions`` (whole dimensions to skip), and ``rules_module`` (a
81
+ #: dotted import path whose ``register_rule`` calls add custom rules). Entry
82
+ #: points under ``dbdocs.health_rules`` are also discovered automatically.
83
+ health: dict = field(default_factory=dict)
72
84
 
73
85
  @classmethod
74
86
  def load(cls, path: "str | Path | None" = None) -> "DbDocsConfig":
@@ -110,6 +122,8 @@ class DbDocsConfig:
110
122
  "readme",
111
123
  "logo",
112
124
  "favicon",
125
+ "run_results",
126
+ "health",
113
127
  )
114
128
 
115
129
  def render_context(self) -> dict:
@@ -0,0 +1,12 @@
1
+ """Project Health Check — the six dbt-project-evaluator dimensions.
2
+
3
+ The public entry point is :class:`HealthCheckExtractor`, which composes the
4
+ manifest-derived rule engine (:mod:`dbdocs.extract.health.dimensions` +
5
+ :mod:`dbdocs.extract.health.rules`) with the optional ``run_results.json``
6
+ per-test detail. Custom rules can be registered via
7
+ :func:`dbdocs.extract.health.rules.register_rule` (see the plugin docs).
8
+ """
9
+
10
+ from dbdocs.extract.health.extractor import HealthCheckExtractor
11
+
12
+ __all__ = ["HealthCheckExtractor"]
@@ -0,0 +1,262 @@
1
+ """Build the manifest-derived health *dimensions* (the DPE rule engine's host).
2
+
3
+ ``ManifestGraph`` indexes the dbt manifest once — parents/children adjacency,
4
+ plus layer / materialization / access / contract helpers — and the rule functions
5
+ in :mod:`dbdocs.extract.health.rules` run against it. ``DimensionAnalyzer.analyze()``
6
+ runs every rule, groups findings by dimension, and returns the ``dimensions``
7
+ block of the health data dict:
8
+
9
+ {
10
+ "modeling": {"issues": N, "checked": M, "findings": [...]},
11
+ "documentation": {...}, "structure": {...}, "performance": {...},
12
+ "governance": {...},
13
+ }
14
+
15
+ ``checked`` is how many nodes the dimension's rules could apply to (models for
16
+ most, models+sources for some) — it gives the SPA a denominator for a score.
17
+ Everything is pure + DOM-free; a malformed/empty manifest yields empty dimensions
18
+ rather than raising (the caller, ``HealthCheckExtractor``, is fail-soft).
19
+ """
20
+
21
+ from typing import Any
22
+
23
+ from dbdocs.core.log import logger
24
+ from dbdocs.extract.health.rules import (
25
+ DEFAULT_THRESHOLDS,
26
+ DIMENSION_RULES,
27
+ NON_PHYSICAL,
28
+ load_entry_point_rules,
29
+ load_rules_module,
30
+ )
31
+
32
+
33
+ class ManifestGraph:
34
+ """Adjacency + node metadata over a dbt manifest, for the health rules.
35
+
36
+ Wraps the dbterd-parsed manifest: ``models``/``sources``/``exposures`` lists,
37
+ a ``parents``/``children`` adjacency built from each node's ``depends_on``, and
38
+ accessors that smooth over where dbt stores a field (``config.materialized``
39
+ vs node, ``config.access`` vs node, ``schema_`` alias, …).
40
+ """
41
+
42
+ def __init__(self, manifest: "Any | None", thresholds: "dict | None" = None) -> None:
43
+ nodes = getattr(manifest, "nodes", None)
44
+ self._nodes: dict = nodes if isinstance(nodes, dict) else {}
45
+ sources = getattr(manifest, "sources", None)
46
+ self._sources: dict = sources if isinstance(sources, dict) else {}
47
+ exposures = getattr(manifest, "exposures", None)
48
+ self._exposures: dict = exposures if isinstance(exposures, dict) else {}
49
+
50
+ self.models = [n for uid, n in self._nodes.items() if uid.startswith("model.")]
51
+ self.sources = list(self._sources.values())
52
+ self.exposures = list(self._exposures.values())
53
+
54
+ # Rule thresholds: per-run overrides layered over the DPE defaults.
55
+ self._thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
56
+
57
+ self._parents: dict[str, list[str]] = {}
58
+ self._children: dict[str, list[str]] = {}
59
+ self._build_adjacency()
60
+
61
+ self._tests_by_model: dict[str, set] = {}
62
+ self._index_tests()
63
+
64
+ # Memo for non_physical_chain_depth (O(nodes) across the whole pass).
65
+ self._chain_cache: dict[str, int] = {}
66
+
67
+ def threshold(self, name: str) -> "int | float":
68
+ """The configured threshold for *name* (DPE default when unset)."""
69
+ return self._thresholds[name]
70
+
71
+ def _build_adjacency(self) -> None:
72
+ """Index parent/child edges from every node + exposure ``depends_on.nodes``."""
73
+ everything = list(self._nodes.values()) + self.exposures
74
+ for node in everything:
75
+ uid = str(getattr(node, "unique_id", "") or "")
76
+ if not uid:
77
+ continue
78
+ depends_on = getattr(node, "depends_on", None)
79
+ parents = list(getattr(depends_on, "nodes", None) or [])
80
+ self._parents[uid] = parents
81
+ for parent in parents:
82
+ self._children.setdefault(parent, []).append(uid)
83
+
84
+ def _index_tests(self) -> None:
85
+ """Map each tested node to the set of test types attached to it."""
86
+ for uid, node in self._nodes.items():
87
+ if not uid.startswith("test."):
88
+ continue
89
+ attached = str(getattr(node, "attached_node", "") or "")
90
+ if not attached:
91
+ continue
92
+ metadata = getattr(node, "test_metadata", None)
93
+ test_type = str(getattr(metadata, "name", "") or "") if metadata else "singular"
94
+ self._tests_by_model.setdefault(attached, set()).add(test_type)
95
+
96
+ def tests_for(self, unique_id: str) -> set:
97
+ """The set of test types attached to *unique_id* (empty if untested)."""
98
+ return self._tests_by_model.get(unique_id, set())
99
+
100
+ # -- adjacency -----------------------------------------------------------
101
+
102
+ def parents(self, unique_id: str) -> "list[str]":
103
+ return self._parents.get(unique_id, [])
104
+
105
+ def children(self, unique_id: str) -> "list[str]":
106
+ return self._children.get(unique_id, [])
107
+
108
+ def node(self, unique_id: str) -> "Any | None":
109
+ """The node object for *unique_id* (model/seed/snapshot or source)."""
110
+ return self._nodes.get(unique_id) or self._sources.get(unique_id)
111
+
112
+ # -- node metadata -------------------------------------------------------
113
+
114
+ @staticmethod
115
+ def layer(model: Any) -> str:
116
+ """The model's layer (staging/intermediate/marts/other) from its ``fqn``.
117
+
118
+ dbt's ``fqn`` is ``[package, ...folders, name]`` — the folder segments name
119
+ the layer. We match the known layer folders anywhere in the fqn so a
120
+ ``marts/finance/`` nesting still resolves to ``marts``.
121
+ """
122
+ fqn = [str(p).lower() for p in (getattr(model, "fqn", None) or [])]
123
+ for layer in ("staging", "intermediate", "marts"):
124
+ if layer in fqn:
125
+ return layer
126
+ # dbt also uses "stg"/"int"/"core"/"mart" folder spellings; map the common ones.
127
+ if "stg" in fqn:
128
+ return "staging"
129
+ if "int" in fqn:
130
+ return "intermediate"
131
+ if "mart" in fqn or "marts" in fqn:
132
+ return "marts"
133
+ return "other"
134
+
135
+ @staticmethod
136
+ def materialization(node: Any) -> str:
137
+ """The node's materialization (``config.materialized``), lowercased."""
138
+ config = getattr(node, "config", None)
139
+ materialized = getattr(config, "materialized", None) if config else None
140
+ return str(materialized or "").lower()
141
+
142
+ @staticmethod
143
+ def access(model: Any) -> str:
144
+ """The model's access level (``config.access`` or node ``access``)."""
145
+ config = getattr(model, "config", None)
146
+ access = getattr(config, "access", None) if config else None
147
+ access = access or getattr(model, "access", None)
148
+ return str(access or "protected").lower()
149
+
150
+ @staticmethod
151
+ def contract_enforced(model: Any) -> bool:
152
+ """Whether the model has an enforced contract (``contract.enforced``)."""
153
+ contract = getattr(model, "contract", None)
154
+ return bool(getattr(contract, "enforced", False)) if contract else False
155
+
156
+ def non_physical_chain_depth(self, unique_id: str) -> int:
157
+ """Longest chain of consecutive non-physical (view/ephemeral) ancestors + self.
158
+
159
+ Counts this model plus the longest run of view/ephemeral models directly
160
+ upstream; a table/source ancestor (or a model boundary) stops the chain.
161
+ Mirrors DPE's ``chained_view_dependencies`` distance.
162
+
163
+ Iterative + memoized: depth is computed once per node and cached on the
164
+ graph (``_chain_cache``), so the whole pass is ``O(nodes)`` even on a
165
+ 3000-deep view chain — recursion here would blow Python's stack limit and
166
+ be ``O(nodes²)``.
167
+ """
168
+ cache = self._chain_cache
169
+ if unique_id in cache:
170
+ return cache[unique_id]
171
+ # Post-order DFS via an explicit stack: push a node, then its unresolved
172
+ # non-physical parents; resolve a node once all its parents are cached.
173
+ stack = [unique_id]
174
+ on_stack = {unique_id}
175
+ while stack:
176
+ uid = stack[-1]
177
+ node = self._nodes.get(uid)
178
+ if node is None or self.materialization(node) not in NON_PHYSICAL:
179
+ cache[uid] = 0
180
+ stack.pop()
181
+ on_stack.discard(uid)
182
+ continue
183
+ best = 0
184
+ pending = None
185
+ for parent in self.parents(uid):
186
+ if parent in cache:
187
+ best = max(best, cache[parent])
188
+ elif parent not in on_stack: # unresolved, not already in progress
189
+ pending = parent
190
+ break
191
+ # parent in on_stack ⇒ a cycle; treat as a stop (contributes 0)
192
+ if pending is not None:
193
+ stack.append(pending)
194
+ on_stack.add(pending)
195
+ continue
196
+ cache[uid] = 1 + best
197
+ stack.pop()
198
+ on_stack.discard(uid)
199
+ return cache[unique_id]
200
+
201
+
202
+ class DimensionAnalyzer:
203
+ """Run every manifest-derived health rule and group findings by dimension.
204
+
205
+ *config* is the optional ``health`` block from ``dbdocs.yml``:
206
+ ``thresholds`` (per-rule overrides), ``disable`` (rule names to skip),
207
+ ``disable_dimensions`` (whole dimensions to skip), and ``rules_module`` (a
208
+ dotted path whose ``register_rule`` calls add custom rules). Entry-point
209
+ plugins under ``dbdocs.health_rules`` are always discovered.
210
+ """
211
+
212
+ def __init__(self, manifest: "Any | None", config: "dict | None" = None) -> None:
213
+ config = config or {}
214
+ thresholds = config.get("thresholds") if isinstance(config.get("thresholds"), dict) else {}
215
+ self._graph = ManifestGraph(manifest, thresholds=thresholds)
216
+ self._disabled_rules = set(config.get("disable") or [])
217
+ self._disabled_dimensions = set(config.get("disable_dimensions") or [])
218
+
219
+ # Plugin loading: entry points always; an explicit rules_module on request.
220
+ load_entry_point_rules()
221
+ rules_module = config.get("rules_module")
222
+ if rules_module:
223
+ load_rules_module(str(rules_module))
224
+
225
+ def analyze(self) -> dict:
226
+ """Return the ``dimensions`` block: one entry per (enabled) dimension."""
227
+ graph = self._graph
228
+ model_count = len(graph.models)
229
+ source_count = len(graph.sources)
230
+ dimensions: dict = {}
231
+ for dimension, rules in DIMENSION_RULES.items():
232
+ if dimension in self._disabled_dimensions:
233
+ continue
234
+ findings: list = []
235
+ for rule in rules:
236
+ if getattr(rule, "__name__", "") in self._disabled_rules:
237
+ continue
238
+ try:
239
+ findings.extend(rule(graph))
240
+ except (AttributeError, TypeError, ValueError, KeyError, RecursionError) as exc:
241
+ # A single malformed node / bad plugin must never sink the report.
242
+ logger.warning(
243
+ "Health dimension %s: rule %s failed: %s — skipping.",
244
+ dimension,
245
+ getattr(rule, "__name__", rule),
246
+ exc,
247
+ )
248
+ dimensions[dimension] = {
249
+ "issues": len(findings),
250
+ "checked": self._checked(dimension, model_count, source_count),
251
+ "findings": findings,
252
+ }
253
+ return dimensions
254
+
255
+ @staticmethod
256
+ def _checked(dimension: str, model_count: int, source_count: int) -> int:
257
+ """How many nodes a dimension's rules could apply to (the score denominator)."""
258
+ # Modeling/documentation/structure span models + sources; performance and
259
+ # governance are model/exposure-centric — count models as the denominator.
260
+ if dimension in ("modeling", "documentation", "structure"):
261
+ return model_count + source_count
262
+ return model_count