dbdocs 1.0.1__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbdocs-1.0.1 → dbdocs-1.1.0}/.gitignore +3 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/PKG-INFO +2 -2
- {dbdocs-1.0.1 → dbdocs-1.1.0}/README.md +1 -1
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/cli/main.py +1 -1
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/core/config.py +11 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/column_lineage.py +153 -67
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/erd.py +10 -1
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/nodes.py +39 -14
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/builder.py +59 -10
- {dbdocs-1.0.1/dbdocs/site/bundle/assets → dbdocs-1.1.0/dbdocs/site/bundle/assets/css}/style.css +74 -9
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/assets/graph/index.css +1 -1
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/assets/graph/index.js +12 -12
- dbdocs-1.1.0/dbdocs/site/bundle/assets/js/app.js +20 -0
- dbdocs-1.1.0/dbdocs/site/bundle/assets/js/data.js +55 -0
- dbdocs-1.1.0/dbdocs/site/bundle/assets/js/service.js +111 -0
- dbdocs-1.1.0/dbdocs/site/bundle/assets/js/ui.js +510 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/index.html +12 -5
- dbdocs-1.1.0/dbdocs/site/inject.py +16 -0
- dbdocs-1.0.1/dbdocs/site/bundle/assets/app.js +0 -500
- dbdocs-1.0.1/dbdocs/site/inject.py +0 -32
- {dbdocs-1.0.1 → dbdocs-1.1.0}/LICENSE +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/__init__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/__main__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/cli/__init__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/core/__init__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/core/artifacts.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/core/exceptions.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/core/log.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/__init__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/_sqlglot_lineage.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/erd_json.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/extract/graph.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/main.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/__init__.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/assets/favicon.svg +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/assets/vendor/marked.min.js +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/bundle/assets/vendor/minisearch.min.js +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/dbdocs/site/deploy.py +0 -0
- {dbdocs-1.0.1 → dbdocs-1.1.0}/pyproject.toml +0 -0
|
@@ -152,6 +152,8 @@ frontend/dist/
|
|
|
152
152
|
frontend/.vite/
|
|
153
153
|
frontend/coverage/
|
|
154
154
|
frontend/*.tsbuildinfo
|
|
155
|
+
frontend/test-results/
|
|
156
|
+
frontend/playwright-report/
|
|
155
157
|
.eslintcache
|
|
156
158
|
|
|
157
159
|
# npm / yarn / pnpm logs & debug output
|
|
@@ -166,3 +168,4 @@ pnpm-debug.log*
|
|
|
166
168
|
|
|
167
169
|
# dbdocs DEBUG file log
|
|
168
170
|
logs/
|
|
171
|
+
.DS_Store
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbdocs
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Alternative dbt docs site: Catalog + ERD + column-level lineage
|
|
5
5
|
Project-URL: Homepage, https://github.com/datnguye/dbt-docs
|
|
6
6
|
Project-URL: Repository, https://github.com/datnguye/dbt-docs
|
|
@@ -27,7 +27,7 @@ Description-Content-Type: text/markdown
|
|
|
27
27
|
<p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage, baked into one file.</b></p>
|
|
28
28
|
|
|
29
29
|
<p align="center">
|
|
30
|
-
<a href="https://dbdocs.datnguye.me/latest/demo/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
30
|
+
<a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
31
31
|
<a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
|
|
32
32
|
<a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
|
|
33
33
|
<img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<p align="center"><b>An alternative dbt docs site — catalog + ERD + column-level lineage, baked into one file.</b></p>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<a href="https://dbdocs.datnguye.me/latest/demo/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
8
|
+
<a href="https://dbdocs.datnguye.me/latest/demo/latest/"><img src="https://img.shields.io/badge/live-demo-FF694A?style=flat&logo=rocket&logoColor=white" alt="live demo"></a>
|
|
9
9
|
<a href="https://dbdocs.datnguye.me/"><img src="https://img.shields.io/badge/docs-visit%20site-blue?style=flat&logo=gitbook&logoColor=white" alt="docs"></a>
|
|
10
10
|
<a href="https://pypi.org/project/dbdocs/"><img src="https://badge.fury.io/py/dbdocs.svg" alt="PyPI version"></a>
|
|
11
11
|
<img src="https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white" alt="python-cli">
|
|
@@ -40,7 +40,7 @@ def dbdocs(ctx, config_path):
|
|
|
40
40
|
)
|
|
41
41
|
@click.pass_obj
|
|
42
42
|
def generate(config: DbDocsConfig, output_dir, dialect):
|
|
43
|
-
"""Build the
|
|
43
|
+
"""Build the site from dbt artifacts (served over HTTP; data loaded externally)."""
|
|
44
44
|
if dialect is not None:
|
|
45
45
|
config.dialect = dialect
|
|
46
46
|
try:
|
|
@@ -49,6 +49,13 @@ class DbDocsConfig:
|
|
|
49
49
|
#: Project README rendered on the overview (relative to the working dir). Set
|
|
50
50
|
#: empty to omit the README section. Missing file ⇒ section simply absent.
|
|
51
51
|
readme: str = "README.md"
|
|
52
|
+
#: Custom header logo (path relative to the working dir). Empty ⇒ the bundled
|
|
53
|
+
#: default mark. A missing file or a path escaping the cwd is ignored
|
|
54
|
+
#: (fail-soft, like ``readme``) ⇒ the default mark.
|
|
55
|
+
logo: str = ""
|
|
56
|
+
#: Custom favicon (path relative to the working dir). Empty ⇒ the bundled
|
|
57
|
+
#: default favicon. Same fail-soft handling as ``logo``.
|
|
58
|
+
favicon: str = ""
|
|
52
59
|
target_dir: str = "target"
|
|
53
60
|
#: Where the generated site is written. Nested under the dbt ``target/`` by
|
|
54
61
|
#: default so docs sit alongside the artifacts they're built from.
|
|
@@ -92,6 +99,8 @@ class DbDocsConfig:
|
|
|
92
99
|
return cls(**raw)
|
|
93
100
|
|
|
94
101
|
#: Build-control fields that are not part of the site's display metadata.
|
|
102
|
+
#: ``logo``/``favicon`` are excluded too — they're *source* paths; the builder
|
|
103
|
+
#: copies the files into the output and injects their *deployed* asset URLs.
|
|
95
104
|
_NON_METADATA_FIELDS = (
|
|
96
105
|
"target_dir",
|
|
97
106
|
"output_dir",
|
|
@@ -99,6 +108,8 @@ class DbDocsConfig:
|
|
|
99
108
|
"default_version",
|
|
100
109
|
"dbterd",
|
|
101
110
|
"readme",
|
|
111
|
+
"logo",
|
|
112
|
+
"favicon",
|
|
102
113
|
)
|
|
103
114
|
|
|
104
115
|
def render_context(self) -> dict:
|
|
@@ -12,6 +12,7 @@ must never sink the whole ``generate`` — it's caught, logged, and skipped, and
|
|
|
12
12
|
the run reports how many were skipped.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
15
16
|
from typing import Any
|
|
16
17
|
|
|
17
18
|
from sqlglot import exp
|
|
@@ -27,6 +28,14 @@ _DIALECT_ALIASES = {
|
|
|
27
28
|
"databricks": "spark",
|
|
28
29
|
}
|
|
29
30
|
|
|
31
|
+
#: Parse + qualify is CPU-bound and embarrassingly parallel across models. Below
|
|
32
|
+
#: this model count the ProcessPoolExecutor's spawn + pickle overhead outweighs
|
|
33
|
+
#: the win, so we stay serial; at or above it we fan out across cores.
|
|
34
|
+
_PARALLEL_THRESHOLD = 500
|
|
35
|
+
|
|
36
|
+
#: Errors a single model's lineage may raise that must not sink the whole run.
|
|
37
|
+
_MODEL_ERRORS = (SqlglotError, LineageError, KeyError, ValueError, RecursionError)
|
|
38
|
+
|
|
30
39
|
|
|
31
40
|
def _to_dialect(adapter_type: "str | None") -> "str | None":
|
|
32
41
|
if not adapter_type:
|
|
@@ -34,6 +43,95 @@ def _to_dialect(adapter_type: "str | None") -> "str | None":
|
|
|
34
43
|
return _DIALECT_ALIASES.get(adapter_type, adapter_type)
|
|
35
44
|
|
|
36
45
|
|
|
46
|
+
def _map_table(table: exp.Table, relation_to_node: dict) -> "str | None":
|
|
47
|
+
"""Map a sqlglot ``Table`` to a dbt unique_id via the relation index."""
|
|
48
|
+
candidates = [
|
|
49
|
+
f"{table.catalog}.{table.db}.{table.name}",
|
|
50
|
+
f"{table.db}.{table.name}",
|
|
51
|
+
table.name,
|
|
52
|
+
]
|
|
53
|
+
for candidate in candidates:
|
|
54
|
+
mapped = relation_to_node.get(candidate.lower().strip("."))
|
|
55
|
+
if mapped:
|
|
56
|
+
return mapped
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _leaf_columns(root: Node, relation_to_node: dict) -> list:
|
|
61
|
+
"""Collect distinct upstream ``{node, column}`` leaves of a lineage tree.
|
|
62
|
+
|
|
63
|
+
A leaf is a node whose source is a real ``Table`` (not a CTE/subquery scope)
|
|
64
|
+
that maps back to a dbt node. The root itself is skipped.
|
|
65
|
+
"""
|
|
66
|
+
seen = set()
|
|
67
|
+
upstream = []
|
|
68
|
+
for node in root.walk():
|
|
69
|
+
if node is root:
|
|
70
|
+
continue
|
|
71
|
+
source = node.source
|
|
72
|
+
if not isinstance(source, exp.Table):
|
|
73
|
+
continue
|
|
74
|
+
mapped = _map_table(source, relation_to_node)
|
|
75
|
+
if mapped is None:
|
|
76
|
+
continue
|
|
77
|
+
column = node_name(node.name)
|
|
78
|
+
key = (mapped, column)
|
|
79
|
+
if key in seen:
|
|
80
|
+
continue
|
|
81
|
+
seen.add(key)
|
|
82
|
+
upstream.append({"node": mapped, "column": column})
|
|
83
|
+
return upstream
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _extract_model_columns(
|
|
87
|
+
unique_id: str,
|
|
88
|
+
compiled: str,
|
|
89
|
+
output_columns: list,
|
|
90
|
+
schema: dict,
|
|
91
|
+
dialect: "str | None",
|
|
92
|
+
relation_to_node: dict,
|
|
93
|
+
) -> dict:
|
|
94
|
+
"""Trace every output column of one model to its upstream columns.
|
|
95
|
+
|
|
96
|
+
Pure and picklable (plain str/list/dict args, no manifest/catalog objects) so
|
|
97
|
+
it runs unchanged in the serial loop or a ``ProcessPoolExecutor`` worker.
|
|
98
|
+
Builds the scope once per model — qualify is the expensive part and is
|
|
99
|
+
identical for every column — then traces each column against it.
|
|
100
|
+
"""
|
|
101
|
+
out: dict = {}
|
|
102
|
+
scope = prepare_scope(compiled, schema=schema, dialect=dialect)
|
|
103
|
+
for column in output_columns:
|
|
104
|
+
try:
|
|
105
|
+
root = lineage(column, compiled, dialect=dialect, scope=scope)
|
|
106
|
+
except SqlglotError:
|
|
107
|
+
# One unresolvable column shouldn't drop the rest of the model.
|
|
108
|
+
continue
|
|
109
|
+
upstream = _leaf_columns(root, relation_to_node)
|
|
110
|
+
if upstream:
|
|
111
|
+
out[f"{unique_id}.{column}"] = upstream
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _extract_model_task(work: tuple) -> tuple:
|
|
116
|
+
"""Worker wrapper: returns ``(unique_id, columns_map, error_str_or_None)``.
|
|
117
|
+
|
|
118
|
+
Catching here (rather than letting the exception cross the process boundary)
|
|
119
|
+
keeps the fail-soft contract: one unparseable model is reported and skipped,
|
|
120
|
+
never sinking the pool.
|
|
121
|
+
"""
|
|
122
|
+
unique_id, compiled, output_columns, schema, dialect, relation_to_node = work
|
|
123
|
+
try:
|
|
124
|
+
return (
|
|
125
|
+
unique_id,
|
|
126
|
+
_extract_model_columns(
|
|
127
|
+
unique_id, compiled, output_columns, schema, dialect, relation_to_node
|
|
128
|
+
),
|
|
129
|
+
None,
|
|
130
|
+
)
|
|
131
|
+
except _MODEL_ERRORS as exc:
|
|
132
|
+
return unique_id, {}, str(exc)
|
|
133
|
+
|
|
134
|
+
|
|
37
135
|
class ColumnLineageExtractor:
|
|
38
136
|
"""Build the ``columnLineage`` map for a dbt project's models.
|
|
39
137
|
|
|
@@ -54,86 +152,74 @@ class ColumnLineageExtractor:
|
|
|
54
152
|
self.skipped = 0
|
|
55
153
|
|
|
56
154
|
def extract(self) -> dict:
|
|
57
|
-
"""Return the ``columnLineage`` map across all models (fail-soft).
|
|
155
|
+
"""Return the ``columnLineage`` map across all models (fail-soft).
|
|
156
|
+
|
|
157
|
+
Runs serially for small projects; fans the per-model sqlglot work across
|
|
158
|
+
processes once a project has ``_PARALLEL_THRESHOLD`` models or more, where
|
|
159
|
+
the CPU-bound parse/qualify dominates the pool's spawn cost.
|
|
160
|
+
"""
|
|
161
|
+
work = self._work_items()
|
|
162
|
+
if not work:
|
|
163
|
+
return {}
|
|
164
|
+
if len(work) >= _PARALLEL_THRESHOLD:
|
|
165
|
+
results = self._extract_parallel(work)
|
|
166
|
+
else:
|
|
167
|
+
results = (_extract_model_task(item) for item in work)
|
|
168
|
+
|
|
58
169
|
result: dict = {}
|
|
170
|
+
for unique_id, columns, error in results:
|
|
171
|
+
if error is not None:
|
|
172
|
+
self.skipped += 1
|
|
173
|
+
logger.warning("Column lineage skipped for %s: %s", node_name(unique_id), error)
|
|
174
|
+
continue
|
|
175
|
+
result.update(columns)
|
|
176
|
+
if self.skipped:
|
|
177
|
+
logger.info("Column lineage: skipped %s model(s) that failed to parse.", self.skipped)
|
|
178
|
+
return result
|
|
179
|
+
|
|
180
|
+
def _extract_parallel(self, work: list) -> list:
|
|
181
|
+
"""Run the per-model tasks across a process pool, returning their results."""
|
|
182
|
+
with ProcessPoolExecutor() as pool:
|
|
183
|
+
return list(pool.map(_extract_model_task, work))
|
|
184
|
+
|
|
185
|
+
def _work_items(self) -> list:
|
|
186
|
+
"""Build the picklable per-model work tuples for the (serial or pool) run.
|
|
187
|
+
|
|
188
|
+
Each item is ``(unique_id, compiled, output_columns, schema, dialect,
|
|
189
|
+
relation_to_node)`` — all plain str/list/dict, so it crosses a process
|
|
190
|
+
boundary cleanly (the Pydantic manifest/catalog never do).
|
|
191
|
+
"""
|
|
192
|
+
work = []
|
|
59
193
|
for unique_id, model in (getattr(self.manifest, "nodes", {}) or {}).items():
|
|
60
194
|
if not str(unique_id).startswith("model."):
|
|
61
195
|
continue
|
|
62
196
|
compiled = getattr(model, "compiled_code", "") or ""
|
|
63
197
|
if not compiled.strip():
|
|
64
198
|
continue
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
199
|
+
output_columns = self._output_columns(unique_id, model)
|
|
200
|
+
if not output_columns:
|
|
201
|
+
continue
|
|
202
|
+
work.append(
|
|
203
|
+
(
|
|
204
|
+
unique_id,
|
|
205
|
+
compiled,
|
|
206
|
+
output_columns,
|
|
207
|
+
self.schema,
|
|
208
|
+
self.dialect,
|
|
209
|
+
self._relation_to_node,
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
return work
|
|
73
213
|
|
|
74
|
-
def
|
|
75
|
-
|
|
76
|
-
output_columns = [c for c in (getattr(
|
|
77
|
-
# Fall back to the catalog's column list when the manifest has none.
|
|
214
|
+
def _output_columns(self, unique_id: str, model: Any) -> list:
|
|
215
|
+
"""The model's documented columns, falling back to the catalog's list."""
|
|
216
|
+
output_columns = [c for c in (getattr(model, "columns", {}) or {})]
|
|
78
217
|
if not output_columns:
|
|
79
218
|
catalog_node = (getattr(self.catalog, "nodes", {}) or {}).get(unique_id)
|
|
80
219
|
output_columns = (
|
|
81
220
|
list(getattr(catalog_node, "columns", {}) or {}) if catalog_node else []
|
|
82
221
|
)
|
|
83
|
-
|
|
84
|
-
return
|
|
85
|
-
# Build the scope once per model, then trace each column against it —
|
|
86
|
-
# qualify is the expensive part and is identical for every column.
|
|
87
|
-
scope = prepare_scope(compiled, schema=self.schema, dialect=self.dialect)
|
|
88
|
-
for column in output_columns:
|
|
89
|
-
try:
|
|
90
|
-
root = lineage(column, compiled, dialect=self.dialect, scope=scope)
|
|
91
|
-
except SqlglotError:
|
|
92
|
-
# One unresolvable column shouldn't drop the rest of the model.
|
|
93
|
-
continue
|
|
94
|
-
upstream = self._leaf_columns(root)
|
|
95
|
-
if upstream:
|
|
96
|
-
result[f"{unique_id}.{column}"] = upstream
|
|
97
|
-
|
|
98
|
-
def _leaf_columns(self, root: Node) -> list:
|
|
99
|
-
"""Collect distinct upstream ``{node, column}`` leaves of a lineage tree.
|
|
100
|
-
|
|
101
|
-
A leaf is a node whose source is a real ``Table`` (not a CTE/subquery
|
|
102
|
-
scope) that we can map back to a dbt node. The root itself is skipped.
|
|
103
|
-
"""
|
|
104
|
-
seen = set()
|
|
105
|
-
upstream = []
|
|
106
|
-
for node in root.walk():
|
|
107
|
-
if node is root:
|
|
108
|
-
continue
|
|
109
|
-
source = node.source
|
|
110
|
-
if not isinstance(source, exp.Table):
|
|
111
|
-
continue
|
|
112
|
-
mapped = self._map_table(source)
|
|
113
|
-
if mapped is None:
|
|
114
|
-
continue
|
|
115
|
-
column = node_name(node.name)
|
|
116
|
-
key = (mapped, column)
|
|
117
|
-
if key in seen:
|
|
118
|
-
continue
|
|
119
|
-
seen.add(key)
|
|
120
|
-
upstream.append({"node": mapped, "column": column})
|
|
121
|
-
return upstream
|
|
122
|
-
|
|
123
|
-
def _map_table(self, table: exp.Table) -> "str | None":
|
|
124
|
-
catalog = table.catalog
|
|
125
|
-
db = table.db
|
|
126
|
-
name = table.name
|
|
127
|
-
candidates = [
|
|
128
|
-
f"{catalog}.{db}.{name}",
|
|
129
|
-
f"{db}.{name}",
|
|
130
|
-
name,
|
|
131
|
-
]
|
|
132
|
-
for candidate in candidates:
|
|
133
|
-
mapped = self._relation_to_node.get(candidate.lower().strip("."))
|
|
134
|
-
if mapped:
|
|
135
|
-
return mapped
|
|
136
|
-
return None
|
|
222
|
+
return output_columns
|
|
137
223
|
|
|
138
224
|
def _relation_index(self) -> dict:
|
|
139
225
|
"""Map ``db.schema.table`` (and shorter forms) → dbt unique_id."""
|
|
@@ -9,12 +9,21 @@ foreign-key edges between them, all keyed by dbt unique_id.
|
|
|
9
9
|
|
|
10
10
|
import json
|
|
11
11
|
|
|
12
|
-
from dbterd.api import DbtErd
|
|
12
|
+
from dbterd.api import DbtErd, default
|
|
13
13
|
|
|
14
14
|
# Importing the module registers the "json" target with dbterd's PluginRegistry.
|
|
15
15
|
from dbdocs.extract import erd_json # noqa: F401
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
def erd_algo(dbterd_options: "dict | None" = None) -> str:
|
|
19
|
+
"""The dbterd algorithm that detected the ERD relationships.
|
|
20
|
+
|
|
21
|
+
Reads the configured ``algo`` from the ``dbterd:`` block, falling back to
|
|
22
|
+
dbterd's own default. Surfaced into the SPA so it can explain an empty ERD.
|
|
23
|
+
"""
|
|
24
|
+
return (dbterd_options or {}).get("algo") or default.default_algo()
|
|
25
|
+
|
|
26
|
+
|
|
18
27
|
def build_erd(dbterd_options: "dict | None" = None, artifacts_dir: "str | None" = None) -> DbtErd:
|
|
19
28
|
"""Build the ERD generator (json target) from dbdocs' ``dbterd`` options.
|
|
20
29
|
|
|
@@ -12,27 +12,52 @@ from typing import Any
|
|
|
12
12
|
from dbdocs.core.artifacts import NODE_PREFIXES, db_schema, node_name
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _column_entry(name: str, manifest_column: Any, col_type: str) -> dict:
|
|
16
|
+
"""One column record. Newlines in the description become ``<br>`` for HTML."""
|
|
17
|
+
description = getattr(manifest_column, "description", "") or "" if manifest_column else ""
|
|
18
|
+
return {
|
|
19
|
+
"name": name,
|
|
20
|
+
"type": col_type or "",
|
|
21
|
+
"tags": (getattr(manifest_column, "tags", []) or []) if manifest_column else [],
|
|
22
|
+
"description": description.replace("\n", "<br>"),
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
15
26
|
def _columns(model: Any, catalog_node: Any) -> list:
|
|
16
|
-
"""
|
|
27
|
+
"""Build a node's columns: the manifest is the source of truth, the catalog
|
|
28
|
+
enriches it.
|
|
29
|
+
|
|
30
|
+
The **manifest** decides which columns are documented and carries their
|
|
31
|
+
metadata (description, tags, ``data_type``); the **catalog** *enriches* — it
|
|
32
|
+
supplies the warehouse-confirmed type and any columns the manifest didn't
|
|
33
|
+
document. It never replaces the manifest, so a model absent from a
|
|
34
|
+
stale/partial ``catalog.json`` still shows every documented column.
|
|
17
35
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
36
|
+
Manifest columns come first, in manifest order, each with its type enriched
|
|
37
|
+
from the catalog when present (falling back to the manifest ``data_type``).
|
|
38
|
+
The catalog keys columns by the warehouse's casing (Snowflake upper-cases
|
|
39
|
+
them) while the manifest keeps the modeled casing, so the catalog type lookup
|
|
40
|
+
is case-insensitive. Catalog-only columns are appended afterwards.
|
|
21
41
|
"""
|
|
22
42
|
manifest_columns = getattr(model, "columns", {}) or {}
|
|
23
43
|
catalog_columns = getattr(catalog_node, "columns", {}) or {} if catalog_node else {}
|
|
44
|
+
catalog_by_lower = {str(name).lower(): col for name, col in catalog_columns.items()}
|
|
45
|
+
|
|
24
46
|
columns = []
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
"tags": (getattr(manifest_column, "tags", []) or []) if manifest_column else [],
|
|
33
|
-
"description": description.replace("\n", "<br>"),
|
|
34
|
-
}
|
|
47
|
+
seen_lower = set()
|
|
48
|
+
for name, manifest_column in manifest_columns.items():
|
|
49
|
+
lower = str(name).lower()
|
|
50
|
+
seen_lower.add(lower)
|
|
51
|
+
catalog_column = catalog_by_lower.get(lower)
|
|
52
|
+
col_type = getattr(catalog_column, "type", None) or getattr(
|
|
53
|
+
manifest_column, "data_type", None
|
|
35
54
|
)
|
|
55
|
+
columns.append(_column_entry(name, manifest_column, col_type or ""))
|
|
56
|
+
|
|
57
|
+
for name, catalog_column in catalog_columns.items():
|
|
58
|
+
if str(name).lower() in seen_lower:
|
|
59
|
+
continue
|
|
60
|
+
columns.append(_column_entry(name, None, getattr(catalog_column, "type", "") or ""))
|
|
36
61
|
return columns
|
|
37
62
|
|
|
38
63
|
|
|
@@ -6,10 +6,11 @@ column-level lineage, ERDs, nav tree) → stage the bundled SPA → base64-injec
|
|
|
6
6
|
data → write ``index.html`` + a debug ``dbdocs-data.json``.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
+
import gzip
|
|
9
10
|
import json
|
|
10
11
|
from datetime import datetime
|
|
11
12
|
from pathlib import Path
|
|
12
|
-
from shutil import copytree, rmtree
|
|
13
|
+
from shutil import copyfile, copytree, rmtree
|
|
13
14
|
from typing import Any
|
|
14
15
|
|
|
15
16
|
from dbdocs.core.artifacts import adapter_type, load_artifacts
|
|
@@ -17,10 +18,10 @@ from dbdocs.core.config import DbDocsConfig, _resolve_within_cwd
|
|
|
17
18
|
from dbdocs.core.exceptions import DbDocsConfigError
|
|
18
19
|
from dbdocs.core.log import logger
|
|
19
20
|
from dbdocs.extract.column_lineage import ColumnLineageExtractor
|
|
20
|
-
from dbdocs.extract.erd import build_erd, build_erd_data
|
|
21
|
+
from dbdocs.extract.erd import build_erd, build_erd_data, erd_algo
|
|
21
22
|
from dbdocs.extract.graph import LineageGraph
|
|
22
23
|
from dbdocs.extract.nodes import build_nodes, build_tree
|
|
23
|
-
from dbdocs.site.inject import
|
|
24
|
+
from dbdocs.site.inject import strip_marker
|
|
24
25
|
|
|
25
26
|
#: The bundled SPA (shell + assets), shipped in the wheel.
|
|
26
27
|
BUNDLE_DIR = Path(__file__).resolve().parent / "bundle"
|
|
@@ -54,6 +55,7 @@ class ReportBuilder:
|
|
|
54
55
|
"generated_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
|
|
55
56
|
"adapter_type": adapter,
|
|
56
57
|
"dialect": dialect,
|
|
58
|
+
"erd_algo": erd_algo(self.config.dbterd),
|
|
57
59
|
"counts": self._counts(manifest),
|
|
58
60
|
},
|
|
59
61
|
"nodes": nodes,
|
|
@@ -82,6 +84,40 @@ class ReportBuilder:
|
|
|
82
84
|
except OSError:
|
|
83
85
|
return ""
|
|
84
86
|
|
|
87
|
+
def _stage_branding(self, out: Path) -> dict:
|
|
88
|
+
"""Copy any custom logo/favicon into the output and return their metadata.
|
|
89
|
+
|
|
90
|
+
Returns a dict with ``logo`` / ``favicon`` keys set to the deployed asset
|
|
91
|
+
URL (e.g. ``assets/logo.png``) for each override that resolved to a real
|
|
92
|
+
file. Absent/unresolvable overrides are simply omitted — the SPA then
|
|
93
|
+
keeps the bundled default. Fail-soft, mirroring ``_read_readme``.
|
|
94
|
+
"""
|
|
95
|
+
meta: dict = {}
|
|
96
|
+
for field_name in ("logo", "favicon"):
|
|
97
|
+
url = self._copy_branding_asset(getattr(self.config, field_name), field_name, out)
|
|
98
|
+
if url:
|
|
99
|
+
meta[field_name] = url
|
|
100
|
+
return meta
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def _copy_branding_asset(source: str, name: str, out: Path) -> str:
|
|
104
|
+
"""Copy *source* into ``out/assets`` as ``<name><ext>``; return its URL.
|
|
105
|
+
|
|
106
|
+
``""`` for an empty config value, a path escaping the cwd, or a missing
|
|
107
|
+
file (fail-soft — bad branding must never sink generate).
|
|
108
|
+
"""
|
|
109
|
+
if not source:
|
|
110
|
+
return ""
|
|
111
|
+
try:
|
|
112
|
+
path = _resolve_within_cwd(source, name)
|
|
113
|
+
except DbDocsConfigError:
|
|
114
|
+
return ""
|
|
115
|
+
if not path.is_file():
|
|
116
|
+
return ""
|
|
117
|
+
target_name = f"{name}{path.suffix}"
|
|
118
|
+
copyfile(path, out / "assets" / target_name)
|
|
119
|
+
return f"assets/{target_name}"
|
|
120
|
+
|
|
85
121
|
#: Manifest collections that aren't keyed by a ``<type>.`` unique_id prefix,
|
|
86
122
|
#: mapped to the resource-type label to report them under.
|
|
87
123
|
_MANIFEST_COLLECTIONS = {
|
|
@@ -112,7 +148,15 @@ class ReportBuilder:
|
|
|
112
148
|
return counts
|
|
113
149
|
|
|
114
150
|
def generate(self, output_dir: "str | None" = None) -> str:
|
|
115
|
-
"""Render the site into ``output_dir`` (or config's). Returns its path.
|
|
151
|
+
"""Render the site into ``output_dir`` (or config's). Returns its path.
|
|
152
|
+
|
|
153
|
+
The data dict is written as an external ``dbdocs-data.json.gz`` that the
|
|
154
|
+
SPA fetches at load — never inlined into ``index.html``. This keeps the
|
|
155
|
+
HTML tiny regardless of project size, where a multi-MB inlined base64
|
|
156
|
+
payload would otherwise make the page slow to parse. The site must be
|
|
157
|
+
served over HTTP (``dbdocs serve`` or any static host); a plain
|
|
158
|
+
``dbdocs-data.json`` is written alongside for debugging.
|
|
159
|
+
"""
|
|
116
160
|
out = Path(output_dir) if output_dir else Path(self.config.output_path)
|
|
117
161
|
if out.exists():
|
|
118
162
|
rmtree(out)
|
|
@@ -120,13 +164,18 @@ class ReportBuilder:
|
|
|
120
164
|
copytree(src=BUNDLE_DIR, dst=out, dirs_exist_ok=True)
|
|
121
165
|
|
|
122
166
|
data = self.build_data()
|
|
167
|
+
data["metadata"].update(self._stage_branding(out))
|
|
123
168
|
index = out / "index.html"
|
|
124
|
-
index.write_text(
|
|
125
|
-
#
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
169
|
+
index.write_text(strip_marker(index.read_text(encoding="utf-8")), encoding="utf-8")
|
|
170
|
+
# Serialize once (compact, not indented — keeps it cheap on large
|
|
171
|
+
# projects); write the plain debug dump and the gzipped payload the SPA
|
|
172
|
+
# fetches.
|
|
173
|
+
payload = json.dumps(
|
|
174
|
+
data, separators=(",", ":"), sort_keys=True, default=self._json_default
|
|
175
|
+
).encode("utf-8")
|
|
176
|
+
(out / "dbdocs-data.json").write_bytes(payload)
|
|
177
|
+
# mtime=0 keeps the gzip header byte-for-byte reproducible across runs.
|
|
178
|
+
(out / "dbdocs-data.json.gz").write_bytes(gzip.compress(payload, mtime=0))
|
|
130
179
|
|
|
131
180
|
logger.info(
|
|
132
181
|
"Generated site at %s (%s nodes, %s column-lineage edges).",
|
{dbdocs-1.0.1/dbdocs/site/bundle/assets → dbdocs-1.1.0/dbdocs/site/bundle/assets/css}/style.css
RENAMED
|
@@ -53,7 +53,11 @@ code, pre { font-family: "SF Mono", "Roboto Mono", Menlo, Consolas, monospace; }
|
|
|
53
53
|
position: sticky; top: 0; z-index: 30;
|
|
54
54
|
}
|
|
55
55
|
.brand { display: flex; align-items: center; gap: 8px; font-weight: 600; min-width: 160px; }
|
|
56
|
-
|
|
56
|
+
/* The logo + name link home (catalog overview). */
|
|
57
|
+
.brand-home { display: flex; align-items: center; gap: 8px; color: var(--text); }
|
|
58
|
+
.brand-home:hover { text-decoration: none; color: var(--accent); }
|
|
59
|
+
.brand-mark { display: block; width: 24px; height: 24px; border-radius: 6px; align-self: center; }
|
|
60
|
+
.brand-generated { color: var(--text-soft); font-size: 12px; font-weight: 400; white-space: nowrap; }
|
|
57
61
|
.search-wrap { position: relative; flex: 1; max-width: 520px; }
|
|
58
62
|
#search {
|
|
59
63
|
width: 100%; padding: 7px 12px; border-radius: var(--radius);
|
|
@@ -118,21 +122,16 @@ code, pre { font-family: "SF Mono", "Roboto Mono", Menlo, Consolas, monospace; }
|
|
|
118
122
|
}
|
|
119
123
|
.footer-copy { color: var(--text-soft); font-size: 12px; }
|
|
120
124
|
|
|
121
|
-
/* Build metadata in the header (generated / adapter / edges) */
|
|
122
|
-
.topbar-meta {
|
|
123
|
-
color: var(--text-soft); font-size: 11.5px; white-space: nowrap;
|
|
124
|
-
padding-right: 6px; flex-shrink: 0;
|
|
125
|
-
}
|
|
126
|
-
@media (max-width: 1100px) { .topbar-meta { display: none; } }
|
|
127
|
-
|
|
128
125
|
/* End-of-page content footer */
|
|
129
126
|
.content-foot {
|
|
130
127
|
display: flex; align-items: center; justify-content: space-between; gap: 12px;
|
|
131
128
|
margin-top: 40px; padding-top: 16px; border-top: 1px solid var(--border);
|
|
132
129
|
color: var(--text-soft); font-size: 12px;
|
|
133
130
|
}
|
|
134
|
-
.content-foot .cf-left { display: inline-flex; align-items: center; gap: 6px; }
|
|
131
|
+
.content-foot .cf-left { display: inline-flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
|
135
132
|
.content-foot .cf-left .ic { color: var(--text-soft); }
|
|
133
|
+
/* Build metadata (generated / adapter / edges) appended to the footer's left */
|
|
134
|
+
.content-foot .cf-meta { color: var(--text-soft); }
|
|
136
135
|
.content-foot .cf-top { color: var(--text-soft); }
|
|
137
136
|
.content-foot .cf-top:hover { color: var(--accent); }
|
|
138
137
|
.bmc {
|
|
@@ -212,6 +211,14 @@ th, td { text-align: left; padding: 7px 10px; border-bottom: 1px solid var(--bor
|
|
|
212
211
|
th { color: var(--text-soft); font-weight: 600; font-size: 12px; text-transform: uppercase; letter-spacing: .03em; }
|
|
213
212
|
td code { background: var(--code-bg); padding: 1px 5px; border-radius: 4px; }
|
|
214
213
|
|
|
214
|
+
/* Briefly highlight a column row deep-linked from an upstream-lineage chip. */
|
|
215
|
+
tr.col-flash > td { background: var(--accent-soft); }
|
|
216
|
+
tr.col-flash { animation: col-flash-fade 1.6s ease-out; }
|
|
217
|
+
@keyframes col-flash-fade {
|
|
218
|
+
0% { background: var(--accent-soft); }
|
|
219
|
+
100% { background: transparent; }
|
|
220
|
+
}
|
|
221
|
+
|
|
215
222
|
/* Tabs */
|
|
216
223
|
.tabs { display: flex; gap: 4px; border-bottom: 1px solid var(--border); margin-top: 8px; }
|
|
217
224
|
.tab { padding: 6px 14px; cursor: pointer; border: 1px solid transparent; border-bottom: none; border-radius: 6px 6px 0 0; color: var(--text-soft); }
|
|
@@ -287,3 +294,61 @@ details.macro pre.code { margin: 0; border: none; border-top: 1px solid var(--bo
|
|
|
287
294
|
.markdown-body th { background: var(--bg-soft); }
|
|
288
295
|
.markdown-body img { max-width: 100%; height: auto; border-radius: 6px; }
|
|
289
296
|
.markdown-body hr { border: none; border-top: 1px solid var(--border); margin: 20px 0; }
|
|
297
|
+
|
|
298
|
+
/* Hamburger + search toggle — hidden on desktop, revealed on phones below. */
|
|
299
|
+
.nav-toggle { display: none; font-size: 18px; }
|
|
300
|
+
.search-toggle { display: none; }
|
|
301
|
+
/* Backdrop behind the off-canvas sidebar drawer. */
|
|
302
|
+
.nav-overlay {
|
|
303
|
+
position: fixed; inset: 52px 0 0 0; z-index: 25;
|
|
304
|
+
background: rgba(0, 0, 0, 0.4);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/* Responsive: collapse the fixed sidebar into a drawer, reflow the topbar, and
|
|
308
|
+
let wide tables scroll instead of overflowing the viewport. */
|
|
309
|
+
@media (max-width: 768px) {
|
|
310
|
+
.nav-toggle { display: inline-flex; }
|
|
311
|
+
.brand { min-width: 0; }
|
|
312
|
+
.layout { height: calc(100vh - 52px); }
|
|
313
|
+
/* Sidebar becomes an off-canvas drawer, slid in when body.nav-open is set. */
|
|
314
|
+
.sidebar-col {
|
|
315
|
+
position: fixed; top: 52px; bottom: 0; left: 0; z-index: 26;
|
|
316
|
+
width: 84vw; max-width: 320px;
|
|
317
|
+
transform: translateX(-100%); transition: transform 0.2s ease;
|
|
318
|
+
box-shadow: var(--shadow);
|
|
319
|
+
}
|
|
320
|
+
body.nav-open .sidebar-col { transform: translateX(0); }
|
|
321
|
+
.content { padding: 20px 16px; }
|
|
322
|
+
/* Let the columns table scroll horizontally rather than blow out the page. */
|
|
323
|
+
.content table { display: block; overflow-x: auto; white-space: nowrap; }
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/* Phones: the topbar is a single row (hamburger + brand + actions, including a
|
|
327
|
+
search-toggle icon). The search hides until the icon reveals it as a
|
|
328
|
+
full-width row underneath, so it never fights the actions for space. */
|
|
329
|
+
@media (max-width: 480px) {
|
|
330
|
+
.topbar {
|
|
331
|
+
flex-wrap: wrap; gap: 8px 10px; padding: 8px 10px; height: auto;
|
|
332
|
+
}
|
|
333
|
+
.brand-generated { display: none; }
|
|
334
|
+
/* The long project name forces the actions onto a second row, so on phones
|
|
335
|
+
keep just the logo mark — the page heading already names the project. */
|
|
336
|
+
.brand-name { display: none; }
|
|
337
|
+
.search-toggle { display: inline-flex; }
|
|
338
|
+
/* Row 1 order: hamburger, brand mark, actions (all fit once the name is out). */
|
|
339
|
+
.nav-toggle { order: 0; }
|
|
340
|
+
.brand { order: 1; }
|
|
341
|
+
.topbar-actions { order: 2; margin-left: auto; }
|
|
342
|
+
/* The search is its own full-width row, collapsed until toggled open. */
|
|
343
|
+
.search-wrap {
|
|
344
|
+
order: 3; flex-basis: 100%; max-width: none;
|
|
345
|
+
display: none;
|
|
346
|
+
}
|
|
347
|
+
body.search-open .search-wrap { display: block; }
|
|
348
|
+
body.search-open .search-toggle { border-color: var(--accent); color: var(--accent); }
|
|
349
|
+
/* Drawer/overlay clear the single-row topbar (taller only while search open,
|
|
350
|
+
where the sticky topbar's higher z-index covers any slight overlap). */
|
|
351
|
+
.sidebar-col { top: 52px; }
|
|
352
|
+
.nav-overlay { top: 0; }
|
|
353
|
+
.content-foot { flex-direction: column; align-items: flex-start; gap: 8px; }
|
|
354
|
+
}
|