quickdbd-cli 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/PKG-INFO +22 -1
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/README.md +21 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/pyproject.toml +1 -1
- quickdbd_cli-0.1.3/src/quickdbd_cli/__init__.py +1 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/render.js +1 -1
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/cli.py +59 -0
- quickdbd_cli-0.1.3/src/quickdbd_cli/dbt.py +412 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/measure.py +1 -1
- quickdbd_cli-0.1.3/tests/fixtures/jaffle/LICENSE +201 -0
- quickdbd_cli-0.1.3/tests/fixtures/jaffle/README.md +2 -0
- quickdbd_cli-0.1.3/tests/fixtures/jaffle/catalog.json +227 -0
- quickdbd_cli-0.1.3/tests/fixtures/jaffle/manifest.json +880 -0
- quickdbd_cli-0.1.3/tests/test_dbt.py +231 -0
- quickdbd_cli-0.1.2/src/quickdbd_cli/__init__.py +0 -1
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/.gitignore +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/LICENSE +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/NOTICE +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/quickdbd-sample.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/finance.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/hr.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/inventory.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/marketing.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/qdbd.layout.json +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/sales.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/gen_font_metrics.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/gen_schema.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/release.sh +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/__main__.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Bold.ttf +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Bold.woff2 +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-LICENSE.txt +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Regular.ttf +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Regular.woff2 +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/editor.js +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/elk-LICENSE.md +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/elk.bundled.js +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/font-metrics.json +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/style.css +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/viewer.js +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/export.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/html.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/jsrt.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/layout.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/model.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/parser.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/project.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/serve.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_parser.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_pipeline.py +0 -0
- {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_serve.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quickdbd-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit.
|
|
5
5
|
Project-URL: Homepage, https://github.com/MarkPhamm/quickdbd-cli
|
|
6
6
|
Author: Minh (Mark) Pham
|
|
@@ -62,6 +62,26 @@ Not affiliated with QuickDBD. It reads the same text syntax, so schemas move fre
|
|
|
62
62
|
- **Pip-only install:** no Graphviz, Node or browser needed. Layout uses [ELK](https://eclipse.dev/elk/), running in an embedded V8.
|
|
63
63
|
- **CI-friendly:** builds are deterministic, and errors are `file:line: error: …` with exit code 1.
|
|
64
64
|
|
|
65
|
+
## From a dbt project
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
dbt docs generate # optional: adds real column types (catalog.json)
|
|
69
|
+
qdbd import dbt target/manifest.json -o erd/ # one .txt per folder (staging, marts, sources, ...)
|
|
70
|
+
qdbd build erd/ -o erd.html
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
- **Tables:** models, seeds and snapshots of your project, plus sources.
|
|
74
|
+
- **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
|
|
75
|
+
- **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
|
|
76
|
+
- **Docs:** model and column descriptions become doc comments, shown as tooltips.
|
|
77
|
+
- **Options:**
|
|
78
|
+
- `--select 'marts.*'` narrows by name, fqn or path (repeatable)
|
|
79
|
+
- `--package all` includes installed packages
|
|
80
|
+
- `--no-sources` drops sources
|
|
81
|
+
- `-o schema.txt` writes a single file
|
|
82
|
+
|
|
83
|
+
Re-running the import overwrites the generated files. Table positions survive because they live in the layout file, keyed by table name.
|
|
84
|
+
|
|
65
85
|
## Commands
|
|
66
86
|
|
|
67
87
|
```bash
|
|
@@ -70,6 +90,7 @@ qdbd build PATH... -o diagram.html # shareable interactive HTML (pos
|
|
|
70
90
|
qdbd serve PATH... [--port 8765] # live editor + drag-to-arrange
|
|
71
91
|
qdbd export PATH... -o out.png [--focus Customer --depth 1] # PNG or SVG image
|
|
72
92
|
qdbd layout PATH... [--reset] # (re)write the layout file
|
|
93
|
+
qdbd import dbt target/manifest.json -o erd/ # generate schema files from a dbt project
|
|
73
94
|
```
|
|
74
95
|
|
|
75
96
|
`PATH` is a `.txt` file or a folder. Every `*.txt` under a folder is read recursively.
|
|
@@ -37,6 +37,26 @@ Not affiliated with QuickDBD. It reads the same text syntax, so schemas move fre
|
|
|
37
37
|
- **Pip-only install:** no Graphviz, Node or browser needed. Layout uses [ELK](https://eclipse.dev/elk/), running in an embedded V8.
|
|
38
38
|
- **CI-friendly:** builds are deterministic, and errors are `file:line: error: …` with exit code 1.
|
|
39
39
|
|
|
40
|
+
## From a dbt project
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
dbt docs generate # optional: adds real column types (catalog.json)
|
|
44
|
+
qdbd import dbt target/manifest.json -o erd/ # one .txt per folder (staging, marts, sources, ...)
|
|
45
|
+
qdbd build erd/ -o erd.html
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
- **Tables:** models, seeds and snapshots of your project, plus sources.
|
|
49
|
+
- **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
|
|
50
|
+
- **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
|
|
51
|
+
- **Docs:** model and column descriptions become doc comments, shown as tooltips.
|
|
52
|
+
- **Options:**
|
|
53
|
+
- `--select 'marts.*'` narrows by name, fqn or path (repeatable)
|
|
54
|
+
- `--package all` includes installed packages
|
|
55
|
+
- `--no-sources` drops sources
|
|
56
|
+
- `-o schema.txt` writes a single file
|
|
57
|
+
|
|
58
|
+
Re-running the import overwrites the generated files. Table positions survive because they live in the layout file, keyed by table name.
|
|
59
|
+
|
|
40
60
|
## Commands
|
|
41
61
|
|
|
42
62
|
```bash
|
|
@@ -45,6 +65,7 @@ qdbd build PATH... -o diagram.html # shareable interactive HTML (pos
|
|
|
45
65
|
qdbd serve PATH... [--port 8765] # live editor + drag-to-arrange
|
|
46
66
|
qdbd export PATH... -o out.png [--focus Customer --depth 1] # PNG or SVG image
|
|
47
67
|
qdbd layout PATH... [--reset] # (re)write the layout file
|
|
68
|
+
qdbd import dbt target/manifest.json -o erd/ # generate schema files from a dbt project
|
|
48
69
|
```
|
|
49
70
|
|
|
50
71
|
`PATH` is a `.txt` file or a folder. Every `*.txt` under a folder is read recursively.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "quickdbd-cli"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
description = "Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.3"
|
|
@@ -99,7 +99,7 @@
|
|
|
99
99
|
'" font-family="' + FONT + '" font-size="' + st.badgeSize + '">' + esc(b.t) + "</text>");
|
|
100
100
|
bx += b.w + st.badgeGap;
|
|
101
101
|
});
|
|
102
|
-
if (c.pk) out.push(keyIcon(w - st.padX - c.typeW - st.keyW
|
|
102
|
+
if (c.pk) out.push(keyIcon(w - st.padX - c.typeW - st.keyW, cy));
|
|
103
103
|
out.push('<text x="' + (w - st.padX) + '" y="' + base + '" text-anchor="end" fill="' + C.type + '" font-family="' + FONT +
|
|
104
104
|
'" font-size="' + st.rowSize + '">' + esc(c.type) + "</text>");
|
|
105
105
|
out.push("</g>");
|
|
@@ -103,6 +103,43 @@ def cmd_layout(args) -> int:
|
|
|
103
103
|
return 0
|
|
104
104
|
|
|
105
105
|
|
|
106
|
+
def cmd_import_dbt(args) -> int:
|
|
107
|
+
from .dbt import DbtImportError, import_dbt, write_result
|
|
108
|
+
from .parser import compile_sources
|
|
109
|
+
|
|
110
|
+
out = Path(args.output)
|
|
111
|
+
split = out.suffix != ".txt"
|
|
112
|
+
try:
|
|
113
|
+
result = import_dbt(
|
|
114
|
+
Path(args.manifest),
|
|
115
|
+
Path(args.catalog) if args.catalog else None,
|
|
116
|
+
packages=args.package,
|
|
117
|
+
select=args.select,
|
|
118
|
+
sources=not args.no_sources,
|
|
119
|
+
split=split,
|
|
120
|
+
)
|
|
121
|
+
except DbtImportError as e:
|
|
122
|
+
print(f"error: {e}", file=sys.stderr)
|
|
123
|
+
return 1
|
|
124
|
+
# Self-check: the generated text must compile cleanly.
|
|
125
|
+
schema = compile_sources([(name, text) for name, text in result.files.items()])
|
|
126
|
+
if schema.errors:
|
|
127
|
+
for d in schema.errors:
|
|
128
|
+
print(d.format(), file=sys.stderr)
|
|
129
|
+
print("error: generated schema did not compile (please report this as a bug)", file=sys.stderr)
|
|
130
|
+
return 1
|
|
131
|
+
written = write_result(result, out)
|
|
132
|
+
for s in result.skipped:
|
|
133
|
+
print(f"skipped: {s}", file=sys.stderr)
|
|
134
|
+
where = str(out) if not split else f"{out}/ ({len(written)} files)"
|
|
135
|
+
print(f"Wrote {where}: {result.tables} tables, {result.relationships} relationships")
|
|
136
|
+
if result.tables and not result.relationships:
|
|
137
|
+
print(
|
|
138
|
+
"hint: no relationships found; add `relationships` tests or foreign_key constraints in dbt", file=sys.stderr
|
|
139
|
+
)
|
|
140
|
+
return 0
|
|
141
|
+
|
|
142
|
+
|
|
106
143
|
def cmd_serve(args) -> int:
|
|
107
144
|
from .serve import serve
|
|
108
145
|
|
|
@@ -147,6 +184,28 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
147
184
|
sp.add_argument("--layout")
|
|
148
185
|
sp.set_defaults(func=cmd_layout)
|
|
149
186
|
|
|
187
|
+
sp = sub.add_parser("import", help="generate QuickDBD schema files from another source")
|
|
188
|
+
isub = sp.add_subparsers(dest="source", required=True)
|
|
189
|
+
dp = isub.add_parser(
|
|
190
|
+
"dbt",
|
|
191
|
+
help="from a dbt project's target/manifest.json",
|
|
192
|
+
description="Tables from models/seeds/snapshots/sources, relationships from `relationships` tests and "
|
|
193
|
+
"foreign_key constraints, types from catalog.json (run `dbt docs generate` first for types).",
|
|
194
|
+
)
|
|
195
|
+
dp.add_argument("manifest", help="path to manifest.json (or the target/ directory)")
|
|
196
|
+
dp.add_argument("-o", "--output", default="schema.txt", help="a .txt file, or a directory for one file per folder")
|
|
197
|
+
dp.add_argument("--catalog", help="catalog.json for column types (default: next to the manifest, if present)")
|
|
198
|
+
dp.add_argument(
|
|
199
|
+
"--package",
|
|
200
|
+
action="append",
|
|
201
|
+
help="dbt package(s) to include (default: the root project; 'all' for every package)",
|
|
202
|
+
)
|
|
203
|
+
dp.add_argument(
|
|
204
|
+
"--select", action="append", help="glob on model name, fqn (a.b.c) or file path, e.g. 'marts.*' (repeatable)"
|
|
205
|
+
)
|
|
206
|
+
dp.add_argument("--no-sources", action="store_true", help="leave out dbt sources")
|
|
207
|
+
dp.set_defaults(func=cmd_import_dbt)
|
|
208
|
+
|
|
150
209
|
sp = sub.add_parser("serve", help="live editor: reload on save, drag tables to arrange")
|
|
151
210
|
paths(sp)
|
|
152
211
|
sp.add_argument("--port", type=int, default=8765)
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""`qdbd import dbt`: turn a dbt project's manifest.json (+ optional catalog.json) into QuickDBD text.
|
|
2
|
+
|
|
3
|
+
- Tables: models, seeds and snapshots of the selected packages, plus sources.
|
|
4
|
+
- Columns: documented columns, catalog columns (with real types) and columns used by tests.
|
|
5
|
+
- Primary keys: `primary_key` constraints, else the first column with both `unique` and
|
|
6
|
+
`not_null` tests. Other `unique` columns get UNIQUE.
|
|
7
|
+
- Relationships: `relationships` tests (incl. dbt_utils `relationships_where`) and
|
|
8
|
+
`foreign_key` constraints. A unique child column gives one-to-one (`-`), otherwise many-to-one (`>-`).
|
|
9
|
+
- Descriptions become QuickDBD doc comments.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import fnmatch
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
HEADER = "# Generated by qdbd import dbt"
|
|
21
|
+
TABLE_TYPES = ("model", "seed", "snapshot")
|
|
22
|
+
REL_TESTS = ("relationships", "relationships_where")
|
|
23
|
+
_IDENT = re.compile(r"^[A-Za-z0-9_]+$")
|
|
24
|
+
_REF = re.compile(
|
|
25
|
+
r"""^\s*ref\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]+)['"]\s*)?(?:,\s*v(?:ersion)?\s*=\s*\S+\s*)?\)\s*$"""
|
|
26
|
+
)
|
|
27
|
+
_SOURCE = re.compile(r"""^\s*source\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*$""")
|
|
28
|
+
_TYPE_ALIASES = {
|
|
29
|
+
"character varying": "varchar",
|
|
30
|
+
"timestamp without time zone": "timestamp",
|
|
31
|
+
"timestamp with time zone": "timestamptz",
|
|
32
|
+
"time without time zone": "time",
|
|
33
|
+
"time with time zone": "timetz",
|
|
34
|
+
"double precision": "double",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class DbtImportError(Exception):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class Col:
|
|
44
|
+
name: str
|
|
45
|
+
type: str | None = None
|
|
46
|
+
description: str = ""
|
|
47
|
+
pk: bool = False
|
|
48
|
+
unique: bool = False
|
|
49
|
+
not_null: bool = False
|
|
50
|
+
fk: tuple[str, str, str] | None = None # (op, parent table uid, parent column)
|
|
51
|
+
order: int = 10_000
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class Tbl:
|
|
56
|
+
uid: str
|
|
57
|
+
name: str # display name in the .txt
|
|
58
|
+
group: str
|
|
59
|
+
description: str = ""
|
|
60
|
+
cols: dict[str, Col] = field(default_factory=dict)
|
|
61
|
+
|
|
62
|
+
def col(self, name: str) -> Col:
|
|
63
|
+
key = name.lower()
|
|
64
|
+
for k, c in self.cols.items():
|
|
65
|
+
if k.lower() == key:
|
|
66
|
+
return c
|
|
67
|
+
c = self.cols[name] = Col(name)
|
|
68
|
+
return c
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class ImportResult:
|
|
73
|
+
files: dict[str, str] # file name -> text
|
|
74
|
+
tables: int
|
|
75
|
+
relationships: int
|
|
76
|
+
skipped: list[str]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _strip_quotes(s: str) -> str:
|
|
80
|
+
s = s.strip()
|
|
81
|
+
if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`[":
|
|
82
|
+
return s[1:-1]
|
|
83
|
+
if s.startswith("[") and s.endswith("]"):
|
|
84
|
+
return s[1:-1]
|
|
85
|
+
return s
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def norm_type(t: str | None) -> str | None:
|
|
89
|
+
if not t:
|
|
90
|
+
return None
|
|
91
|
+
t = t.strip()
|
|
92
|
+
low = t.lower()
|
|
93
|
+
for long, short in _TYPE_ALIASES.items():
|
|
94
|
+
if low.startswith(long):
|
|
95
|
+
t = short + t[len(long) :]
|
|
96
|
+
break
|
|
97
|
+
t = re.sub(r"\s*,\s*", ",", t)
|
|
98
|
+
t = re.sub(r"\s*\(\s*", "(", t)
|
|
99
|
+
t = re.sub(r"\s*\)", ")", t)
|
|
100
|
+
t = re.sub(r"\s+", "_", t)
|
|
101
|
+
return t.replace("=", ":").replace("#", "").replace('"', "")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def quote(name: str) -> str:
|
|
105
|
+
name = name.replace('"', "'")
|
|
106
|
+
return name if _IDENT.match(name) else f'"{name}"'
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _comment_lines(text: str) -> list[str]:
|
|
110
|
+
out = []
|
|
111
|
+
for line in (text or "").replace("\r", "").split("\n"):
|
|
112
|
+
line = " ".join(line.split()).replace("#", "#")
|
|
113
|
+
if line:
|
|
114
|
+
out.append(f"# {line}")
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class _Importer:
|
|
119
|
+
def __init__(
|
|
120
|
+
self, manifest: dict, catalog: dict | None, packages: list[str] | None, select: list[str], sources: bool
|
|
121
|
+
):
|
|
122
|
+
self.m = manifest
|
|
123
|
+
self.catalog = catalog or {}
|
|
124
|
+
self.select = select
|
|
125
|
+
self.sources = sources
|
|
126
|
+
root = (manifest.get("metadata") or {}).get("project_name")
|
|
127
|
+
if packages and "all" in packages:
|
|
128
|
+
self.packages = None
|
|
129
|
+
elif packages:
|
|
130
|
+
self.packages = set(packages)
|
|
131
|
+
else:
|
|
132
|
+
self.packages = {root} if root else None
|
|
133
|
+
self.tables: dict[str, Tbl] = {}
|
|
134
|
+
self.skipped: list[str] = []
|
|
135
|
+
self.by_name: dict[str, list[str]] = {}
|
|
136
|
+
|
|
137
|
+
# -- selection ----------------------------------------------------------
|
|
138
|
+
def _wanted(self, node: dict) -> bool:
|
|
139
|
+
if self.packages is not None and node.get("package_name") not in self.packages:
|
|
140
|
+
return False
|
|
141
|
+
if not self.select:
|
|
142
|
+
return True
|
|
143
|
+
cands = [node.get("name", ""), ".".join(node.get("fqn", [])), node.get("original_file_path", "")]
|
|
144
|
+
return any(fnmatch.fnmatch(c, pat) for pat in self.select for c in cands)
|
|
145
|
+
|
|
146
|
+
@staticmethod
|
|
147
|
+
def _group(node: dict) -> str:
|
|
148
|
+
rt = node["resource_type"]
|
|
149
|
+
if rt == "source":
|
|
150
|
+
return "sources"
|
|
151
|
+
if rt != "model":
|
|
152
|
+
return rt + "s"
|
|
153
|
+
fqn = node.get("fqn", [])
|
|
154
|
+
return fqn[1] if len(fqn) > 2 else "models"
|
|
155
|
+
|
|
156
|
+
def collect(self) -> None:
|
|
157
|
+
nodes = [n for n in self.m.get("nodes", {}).values() if n.get("resource_type") in TABLE_TYPES]
|
|
158
|
+
if self.sources:
|
|
159
|
+
nodes += list(self.m.get("sources", {}).values())
|
|
160
|
+
nodes = [n for n in nodes if self._wanted(n)]
|
|
161
|
+
names: dict[str, int] = {}
|
|
162
|
+
for n in nodes:
|
|
163
|
+
names[self._base_name(n)] = names.get(self._base_name(n), 0) + 1
|
|
164
|
+
for n in sorted(nodes, key=lambda n: n["unique_id"]):
|
|
165
|
+
name = self._base_name(n)
|
|
166
|
+
if names[name] > 1: # same name in two packages
|
|
167
|
+
name = f"{n.get('package_name')}.{name}"
|
|
168
|
+
t = Tbl(n["unique_id"], name, self._group(n), n.get("description") or "")
|
|
169
|
+
self._columns(t, n)
|
|
170
|
+
self.tables[t.uid] = t
|
|
171
|
+
self.by_name.setdefault(n.get("name", "").lower(), []).append(t.uid)
|
|
172
|
+
if n.get("resource_type") == "source":
|
|
173
|
+
self.by_name.setdefault(f"source:{n['source_name']}.{n['name']}".lower(), []).append(t.uid)
|
|
174
|
+
|
|
175
|
+
@staticmethod
|
|
176
|
+
def _base_name(n: dict) -> str:
|
|
177
|
+
if n.get("resource_type") == "source":
|
|
178
|
+
return f"{n['source_name']}.{n['name']}"
|
|
179
|
+
return n.get("name", "")
|
|
180
|
+
|
|
181
|
+
def _columns(self, t: Tbl, n: dict) -> None:
|
|
182
|
+
cat = (self.catalog.get("sources" if n["resource_type"] == "source" else "nodes") or {}).get(n["unique_id"])
|
|
183
|
+
if cat:
|
|
184
|
+
for cname, c in cat.get("columns", {}).items():
|
|
185
|
+
col = t.col(c.get("name") or cname)
|
|
186
|
+
col.type = c.get("type")
|
|
187
|
+
col.order = int(c.get("index") or col.order)
|
|
188
|
+
for i, (cname, c) in enumerate(n.get("columns", {}).items()):
|
|
189
|
+
col = t.col(c.get("name") or cname)
|
|
190
|
+
col.description = c.get("description") or ""
|
|
191
|
+
col.type = col.type or c.get("data_type")
|
|
192
|
+
if not cat:
|
|
193
|
+
col.order = i
|
|
194
|
+
for con in c.get("constraints") or []:
|
|
195
|
+
self._constraint(t, col, con)
|
|
196
|
+
for con in n.get("constraints") or []:
|
|
197
|
+
cols = con.get("columns") or []
|
|
198
|
+
if con.get("type") == "primary_key":
|
|
199
|
+
for cn in cols:
|
|
200
|
+
t.col(cn).pk = True
|
|
201
|
+
elif con.get("type") == "unique" and len(cols) == 1:
|
|
202
|
+
t.col(cols[0]).unique = True
|
|
203
|
+
elif con.get("type") == "foreign_key" and len(cols) == 1:
|
|
204
|
+
self._constraint(t, t.col(cols[0]), con)
|
|
205
|
+
|
|
206
|
+
def _constraint(self, t: Tbl, col: Col, con: dict) -> None:
|
|
207
|
+
kind = con.get("type")
|
|
208
|
+
if kind == "primary_key":
|
|
209
|
+
col.pk = True
|
|
210
|
+
elif kind == "unique":
|
|
211
|
+
col.unique = True
|
|
212
|
+
elif kind == "not_null":
|
|
213
|
+
col.not_null = True
|
|
214
|
+
elif kind == "foreign_key":
|
|
215
|
+
target, tcol = con.get("to"), (con.get("to_columns") or [None])[0]
|
|
216
|
+
if not target and con.get("expression"): # "other_table (id)"
|
|
217
|
+
m = re.match(r"\s*([\w.\"`]+)\s*\(\s*([\w\"`]+)\s*\)", con["expression"])
|
|
218
|
+
if m:
|
|
219
|
+
target, tcol = m.group(1).split(".")[-1].strip('"`'), m.group(2).strip('"`')
|
|
220
|
+
# Resolved in link(): the target table may not have been collected yet.
|
|
221
|
+
self._constraint_fks.append((t.uid, col.name, target or "", tcol or ""))
|
|
222
|
+
|
|
223
|
+
_pending: list[tuple[str, str, str, str]]
|
|
224
|
+
_constraint_fks: list[tuple[str, str, str, str]]
|
|
225
|
+
|
|
226
|
+
def _resolve(self, text: str) -> str | None:
|
|
227
|
+
m = _REF.match(text)
|
|
228
|
+
if m:
|
|
229
|
+
name = (m.group(2) or m.group(1)).lower()
|
|
230
|
+
else:
|
|
231
|
+
s = _SOURCE.match(text)
|
|
232
|
+
name = f"source:{s.group(1)}.{s.group(2)}".lower() if s else text.strip().lower()
|
|
233
|
+
uids = self.by_name.get(name, [])
|
|
234
|
+
return uids[0] if len(uids) == 1 else None
|
|
235
|
+
|
|
236
|
+
# -- tests --------------------------------------------------------------
|
|
237
|
+
def tests(self) -> None:
|
|
238
|
+
uniq: dict[tuple[str, str], bool] = {}
|
|
239
|
+
notnull: dict[tuple[str, str], bool] = {}
|
|
240
|
+
for n in self.m.get("nodes", {}).values():
|
|
241
|
+
if n.get("resource_type") != "test":
|
|
242
|
+
continue
|
|
243
|
+
meta = n.get("test_metadata") or {}
|
|
244
|
+
kw = meta.get("kwargs") or {}
|
|
245
|
+
name = meta.get("name")
|
|
246
|
+
child = n.get("attached_node") or self._child_from_kwargs(kw)
|
|
247
|
+
colname = _strip_quotes(kw.get("column_name") or n.get("column_name") or "")
|
|
248
|
+
if not child or child not in self.tables or not colname:
|
|
249
|
+
continue
|
|
250
|
+
if name == "unique":
|
|
251
|
+
uniq[(child, colname.lower())] = True
|
|
252
|
+
elif name == "not_null":
|
|
253
|
+
notnull[(child, colname.lower())] = True
|
|
254
|
+
elif name in REL_TESTS:
|
|
255
|
+
deps = [d for d in (n.get("depends_on") or {}).get("nodes", []) if d != child]
|
|
256
|
+
parent = deps[0] if deps else (self._resolve(kw.get("to", "")) or child)
|
|
257
|
+
field_name = _strip_quotes(kw.get("field") or "")
|
|
258
|
+
if parent not in self.tables:
|
|
259
|
+
self.skipped.append(f"{self.tables[child].name}.{colname}: parent {parent} not in selection")
|
|
260
|
+
continue
|
|
261
|
+
if field_name:
|
|
262
|
+
self._pending.append((child, colname, parent, field_name))
|
|
263
|
+
for (uid, cname), _ in uniq.items():
|
|
264
|
+
self.tables[uid].col(cname).unique = True
|
|
265
|
+
for (uid, cname), _ in notnull.items():
|
|
266
|
+
self.tables[uid].col(cname).not_null = True
|
|
267
|
+
for t in self.tables.values(): # implied primary key: unique + not_null
|
|
268
|
+
if any(c.pk for c in t.cols.values()):
|
|
269
|
+
continue
|
|
270
|
+
for c in sorted(t.cols.values(), key=lambda c: c.order):
|
|
271
|
+
if c.unique and c.not_null:
|
|
272
|
+
c.pk = True
|
|
273
|
+
break
|
|
274
|
+
for t in self.tables.values():
|
|
275
|
+
for c in t.cols.values():
|
|
276
|
+
if c.pk:
|
|
277
|
+
c.unique = False
|
|
278
|
+
|
|
279
|
+
def _child_from_kwargs(self, kw: dict) -> str | None:
|
|
280
|
+
m = re.search(r"(ref\([^)]*\)|source\([^)]*\))", kw.get("model") or "")
|
|
281
|
+
return self._resolve(m.group(1)) if m else None
|
|
282
|
+
|
|
283
|
+
def link(self) -> int:
|
|
284
|
+
for child, ccol, target, tcol in self._constraint_fks:
|
|
285
|
+
parent = self._resolve(target) if target else None
|
|
286
|
+
if parent and tcol:
|
|
287
|
+
self._pending.append((child, ccol, parent, tcol))
|
|
288
|
+
else:
|
|
289
|
+
self.skipped.append(f"{self.tables[child].name}.{ccol}: foreign_key target '{target}' not found")
|
|
290
|
+
count = 0
|
|
291
|
+
seen = set()
|
|
292
|
+
for child, ccol, parent, pcol in self._pending:
|
|
293
|
+
key = (child, ccol.lower(), parent, pcol.lower())
|
|
294
|
+
if key in seen:
|
|
295
|
+
continue
|
|
296
|
+
seen.add(key)
|
|
297
|
+
c = self.tables[child].col(ccol)
|
|
298
|
+
p = self.tables[parent].col(pcol)
|
|
299
|
+
op = "-" if (c.unique or c.pk) else ">-"
|
|
300
|
+
if c.fk is None:
|
|
301
|
+
c.fk = (op, parent, p.name)
|
|
302
|
+
elif p.fk is None: # one FK per column line: put the second one on the parent side
|
|
303
|
+
p.fk = ({"-": "-", ">-": "-<"}[op], child, c.name)
|
|
304
|
+
else:
|
|
305
|
+
self.skipped.append(
|
|
306
|
+
f"{self.tables[child].name}.{ccol} -> {self.tables[parent].name}.{pcol}: "
|
|
307
|
+
"column already has a relationship"
|
|
308
|
+
)
|
|
309
|
+
continue
|
|
310
|
+
count += 1
|
|
311
|
+
return count
|
|
312
|
+
|
|
313
|
+
# -- output -------------------------------------------------------------
|
|
314
|
+
def render_table(self, t: Tbl) -> list[str]:
|
|
315
|
+
lines = _comment_lines(t.description)
|
|
316
|
+
lines += [quote(t.name), "-"]
|
|
317
|
+
for c in sorted(t.cols.values(), key=lambda c: (c.order, c.name.lower())):
|
|
318
|
+
lines += _comment_lines(c.description)
|
|
319
|
+
parts = [quote(c.name)]
|
|
320
|
+
if c.pk:
|
|
321
|
+
parts.append("PK")
|
|
322
|
+
ty = norm_type(c.type)
|
|
323
|
+
if ty:
|
|
324
|
+
parts.append(ty)
|
|
325
|
+
if c.unique:
|
|
326
|
+
parts.append("UNIQUE")
|
|
327
|
+
if c.fk:
|
|
328
|
+
op, uid, pcol = c.fk
|
|
329
|
+
parts += ["FK", op, f"{quote(self.tables[uid].name)}.{quote(pcol)}"]
|
|
330
|
+
lines.append(" ".join(parts))
|
|
331
|
+
return lines
|
|
332
|
+
|
|
333
|
+
def run(self) -> tuple[dict[str, list[Tbl]], int]:
|
|
334
|
+
self._pending = []
|
|
335
|
+
self._constraint_fks = []
|
|
336
|
+
self.collect()
|
|
337
|
+
self.tests()
|
|
338
|
+
rels = self.link()
|
|
339
|
+
groups: dict[str, list[Tbl]] = {}
|
|
340
|
+
for t in sorted(self.tables.values(), key=lambda t: (t.group, t.name.lower())):
|
|
341
|
+
groups.setdefault(t.group, []).append(t)
|
|
342
|
+
return groups, rels
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def load_json(path: Path) -> dict:
|
|
346
|
+
try:
|
|
347
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
348
|
+
except FileNotFoundError:
|
|
349
|
+
raise DbtImportError(f"no such file: {path}") from None
|
|
350
|
+
except json.JSONDecodeError as e:
|
|
351
|
+
raise DbtImportError(f"{path}: not valid JSON ({e})") from None
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def import_dbt(
|
|
355
|
+
manifest_path: Path,
|
|
356
|
+
catalog_path: Path | None = None,
|
|
357
|
+
packages: list[str] | None = None,
|
|
358
|
+
select: list[str] | None = None,
|
|
359
|
+
sources: bool = True,
|
|
360
|
+
split: bool = False,
|
|
361
|
+
) -> ImportResult:
|
|
362
|
+
if manifest_path.is_dir():
|
|
363
|
+
manifest_path = manifest_path / "manifest.json"
|
|
364
|
+
manifest = load_json(manifest_path)
|
|
365
|
+
if "nodes" not in manifest:
|
|
366
|
+
raise DbtImportError(f"{manifest_path}: not a dbt manifest (no 'nodes')")
|
|
367
|
+
if catalog_path is None and (manifest_path.parent / "catalog.json").exists():
|
|
368
|
+
catalog_path = manifest_path.parent / "catalog.json"
|
|
369
|
+
catalog = load_json(catalog_path) if catalog_path else None
|
|
370
|
+
imp = _Importer(manifest, catalog, packages, select or [], sources)
|
|
371
|
+
groups, rels = imp.run()
|
|
372
|
+
project = (manifest.get("metadata") or {}).get("project_name") or "dbt"
|
|
373
|
+
head = [
|
|
374
|
+
f"{HEADER} from {manifest_path.name} (project {project}).",
|
|
375
|
+
"# Re-run the import to refresh; manual edits will be overwritten.",
|
|
376
|
+
"",
|
|
377
|
+
]
|
|
378
|
+
files: dict[str, str] = {}
|
|
379
|
+
if split:
|
|
380
|
+
for g, tables in groups.items():
|
|
381
|
+
body: list[str] = list(head)
|
|
382
|
+
for t in tables:
|
|
383
|
+
body += imp.render_table(t) + [""]
|
|
384
|
+
files[f"{g}.txt"] = "\n".join(body).rstrip("\n") + "\n"
|
|
385
|
+
else:
|
|
386
|
+
body = list(head)
|
|
387
|
+
for tables in groups.values():
|
|
388
|
+
for t in tables:
|
|
389
|
+
body += imp.render_table(t) + [""]
|
|
390
|
+
files["schema.txt"] = "\n".join(body).rstrip("\n") + "\n"
|
|
391
|
+
return ImportResult(files, len(imp.tables), rels, imp.skipped)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def write_result(result: ImportResult, out: Path) -> list[Path]:
|
|
395
|
+
"""Write to a .txt file, or into a directory (one file per group). Stale generated files are removed."""
|
|
396
|
+
written: list[Path] = []
|
|
397
|
+
if out.suffix == ".txt":
|
|
398
|
+
if len(result.files) != 1:
|
|
399
|
+
raise DbtImportError("--split needs a directory for -o")
|
|
400
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
401
|
+
out.write_text(next(iter(result.files.values())), encoding="utf-8")
|
|
402
|
+
return [out]
|
|
403
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
404
|
+
for name, text in result.files.items():
|
|
405
|
+
p = out / name
|
|
406
|
+
p.write_text(text, encoding="utf-8")
|
|
407
|
+
written.append(p)
|
|
408
|
+
keep = {p.name for p in written}
|
|
409
|
+
for old in out.glob("*.txt"):
|
|
410
|
+
if old.name not in keep and old.read_text(encoding="utf-8").startswith(HEADER):
|
|
411
|
+
old.unlink()
|
|
412
|
+
return written
|