quickdbd-cli 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/PKG-INFO +22 -1
  2. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/README.md +21 -0
  3. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/pyproject.toml +1 -1
  4. quickdbd_cli-0.1.3/src/quickdbd_cli/__init__.py +1 -0
  5. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/render.js +1 -1
  6. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/cli.py +59 -0
  7. quickdbd_cli-0.1.3/src/quickdbd_cli/dbt.py +412 -0
  8. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/measure.py +1 -1
  9. quickdbd_cli-0.1.3/tests/fixtures/jaffle/LICENSE +201 -0
  10. quickdbd_cli-0.1.3/tests/fixtures/jaffle/README.md +2 -0
  11. quickdbd_cli-0.1.3/tests/fixtures/jaffle/catalog.json +227 -0
  12. quickdbd_cli-0.1.3/tests/fixtures/jaffle/manifest.json +880 -0
  13. quickdbd_cli-0.1.3/tests/test_dbt.py +231 -0
  14. quickdbd_cli-0.1.2/src/quickdbd_cli/__init__.py +0 -1
  15. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/.gitignore +0 -0
  16. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/LICENSE +0 -0
  17. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/NOTICE +0 -0
  18. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/quickdbd-sample.txt +0 -0
  19. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/finance.txt +0 -0
  20. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/hr.txt +0 -0
  21. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/inventory.txt +0 -0
  22. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/marketing.txt +0 -0
  23. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/qdbd.layout.json +0 -0
  24. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/examples/synthetic/sales.txt +0 -0
  25. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/gen_font_metrics.py +0 -0
  26. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/gen_schema.py +0 -0
  27. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/scripts/release.sh +0 -0
  28. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/__main__.py +0 -0
  29. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Bold.ttf +0 -0
  30. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Bold.woff2 +0 -0
  31. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-LICENSE.txt +0 -0
  32. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Regular.ttf +0 -0
  33. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/Inter-Regular.woff2 +0 -0
  34. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/editor.js +0 -0
  35. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/elk-LICENSE.md +0 -0
  36. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/elk.bundled.js +0 -0
  37. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/font-metrics.json +0 -0
  38. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/style.css +0 -0
  39. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/assets/viewer.js +0 -0
  40. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/export.py +0 -0
  41. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/html.py +0 -0
  42. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/jsrt.py +0 -0
  43. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/layout.py +0 -0
  44. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/model.py +0 -0
  45. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/parser.py +0 -0
  46. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/project.py +0 -0
  47. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/src/quickdbd_cli/serve.py +0 -0
  48. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_parser.py +0 -0
  49. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_pipeline.py +0 -0
  50. {quickdbd_cli-0.1.2 → quickdbd_cli-0.1.3}/tests/test_serve.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quickdbd-cli
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit.
5
5
  Project-URL: Homepage, https://github.com/MarkPhamm/quickdbd-cli
6
6
  Author: Minh (Mark) Pham
@@ -62,6 +62,26 @@ Not affiliated with QuickDBD. It reads the same text syntax, so schemas move fre
62
62
  - **Pip-only install:** no Graphviz, Node or browser needed. Layout uses [ELK](https://eclipse.dev/elk/), running in an embedded V8.
63
63
  - **CI-friendly:** builds are deterministic, and errors are `file:line: error: …` with exit code 1.
64
64
 
65
+ ## From a dbt project
66
+
67
+ ```bash
68
+ dbt docs generate # optional: adds real column types (catalog.json)
69
+ qdbd import dbt target/manifest.json -o erd/ # one .txt per folder (staging, marts, sources, ...)
70
+ qdbd build erd/ -o erd.html
71
+ ```
72
+
73
+ - **Tables:** models, seeds and snapshots of your project, plus sources.
74
+ - **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
75
+ - **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
76
+ - **Docs:** model and column descriptions become doc comments, shown as tooltips.
77
+ - **Options:**
78
+ - `--select 'marts.*'` narrows by name, fqn or path (repeatable)
79
+ - `--package all` includes installed packages
80
+ - `--no-sources` drops sources
81
+ - `-o schema.txt` writes a single file
82
+
83
+ Re-running the import overwrites the generated files. Table positions survive because they live in the layout file, keyed by table name.
84
+
65
85
  ## Commands
66
86
 
67
87
  ```bash
@@ -70,6 +90,7 @@ qdbd build PATH... -o diagram.html # shareable interactive HTML (pos
70
90
  qdbd serve PATH... [--port 8765] # live editor + drag-to-arrange
71
91
  qdbd export PATH... -o out.png [--focus Customer --depth 1] # PNG or SVG image
72
92
  qdbd layout PATH... [--reset] # (re)write the layout file
93
+ qdbd import dbt target/manifest.json -o erd/ # generate schema files from a dbt project
73
94
  ```
74
95
 
75
96
  `PATH` is a `.txt` file or a folder. Every `*.txt` under a folder is read recursively.
@@ -37,6 +37,26 @@ Not affiliated with QuickDBD. It reads the same text syntax, so schemas move fre
37
37
  - **Pip-only install:** no Graphviz, Node or browser needed. Layout uses [ELK](https://eclipse.dev/elk/), running in an embedded V8.
38
38
  - **CI-friendly:** builds are deterministic, and errors are `file:line: error: …` with exit code 1.
39
39
 
40
+ ## From a dbt project
41
+
42
+ ```bash
43
+ dbt docs generate # optional: adds real column types (catalog.json)
44
+ qdbd import dbt target/manifest.json -o erd/ # one .txt per folder (staging, marts, sources, ...)
45
+ qdbd build erd/ -o erd.html
46
+ ```
47
+
48
+ - **Tables:** models, seeds and snapshots of your project, plus sources.
49
+ - **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
50
+ - **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
51
+ - **Docs:** model and column descriptions become doc comments, shown as tooltips.
52
+ - **Options:**
53
+ - `--select 'marts.*'` narrows by name, fqn or path (repeatable)
54
+ - `--package all` includes installed packages
55
+ - `--no-sources` drops sources
56
+ - `-o schema.txt` writes a single file
57
+
58
+ Re-running the import overwrites the generated files. Table positions survive because they live in the layout file, keyed by table name.
59
+
40
60
  ## Commands
41
61
 
42
62
  ```bash
@@ -45,6 +65,7 @@ qdbd build PATH... -o diagram.html # shareable interactive HTML (pos
45
65
  qdbd serve PATH... [--port 8765] # live editor + drag-to-arrange
46
66
  qdbd export PATH... -o out.png [--focus Customer --depth 1] # PNG or SVG image
47
67
  qdbd layout PATH... [--reset] # (re)write the layout file
68
+ qdbd import dbt target/manifest.json -o erd/ # generate schema files from a dbt project
48
69
  ```
49
70
 
50
71
  `PATH` is a `.txt` file or a folder. Every `*.txt` under a folder is read recursively.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "quickdbd-cli"
7
- version = "0.1.2"
7
+ version = "0.1.3"
8
8
  description = "Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit."
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -0,0 +1 @@
1
+ __version__ = "0.1.3"
@@ -99,7 +99,7 @@
99
99
  '" font-family="' + FONT + '" font-size="' + st.badgeSize + '">' + esc(b.t) + "</text>");
100
100
  bx += b.w + st.badgeGap;
101
101
  });
102
- if (c.pk) out.push(keyIcon(w - st.padX - c.typeW - st.keyW + 1, cy));
102
+ if (c.pk) out.push(keyIcon(w - st.padX - c.typeW - st.keyW, cy));
103
103
  out.push('<text x="' + (w - st.padX) + '" y="' + base + '" text-anchor="end" fill="' + C.type + '" font-family="' + FONT +
104
104
  '" font-size="' + st.rowSize + '">' + esc(c.type) + "</text>");
105
105
  out.push("</g>");
@@ -103,6 +103,43 @@ def cmd_layout(args) -> int:
103
103
  return 0
104
104
 
105
105
 
106
+ def cmd_import_dbt(args) -> int:
107
+ from .dbt import DbtImportError, import_dbt, write_result
108
+ from .parser import compile_sources
109
+
110
+ out = Path(args.output)
111
+ split = out.suffix != ".txt"
112
+ try:
113
+ result = import_dbt(
114
+ Path(args.manifest),
115
+ Path(args.catalog) if args.catalog else None,
116
+ packages=args.package,
117
+ select=args.select,
118
+ sources=not args.no_sources,
119
+ split=split,
120
+ )
121
+ except DbtImportError as e:
122
+ print(f"error: {e}", file=sys.stderr)
123
+ return 1
124
+ # Self-check: the generated text must compile cleanly.
125
+ schema = compile_sources([(name, text) for name, text in result.files.items()])
126
+ if schema.errors:
127
+ for d in schema.errors:
128
+ print(d.format(), file=sys.stderr)
129
+ print("error: generated schema did not compile (please report this as a bug)", file=sys.stderr)
130
+ return 1
131
+ written = write_result(result, out)
132
+ for s in result.skipped:
133
+ print(f"skipped: {s}", file=sys.stderr)
134
+ where = str(out) if not split else f"{out}/ ({len(written)} files)"
135
+ print(f"Wrote {where}: {result.tables} tables, {result.relationships} relationships")
136
+ if result.tables and not result.relationships:
137
+ print(
138
+ "hint: no relationships found; add `relationships` tests or foreign_key constraints in dbt", file=sys.stderr
139
+ )
140
+ return 0
141
+
142
+
106
143
  def cmd_serve(args) -> int:
107
144
  from .serve import serve
108
145
 
@@ -147,6 +184,28 @@ def build_parser() -> argparse.ArgumentParser:
147
184
  sp.add_argument("--layout")
148
185
  sp.set_defaults(func=cmd_layout)
149
186
 
187
+ sp = sub.add_parser("import", help="generate QuickDBD schema files from another source")
188
+ isub = sp.add_subparsers(dest="source", required=True)
189
+ dp = isub.add_parser(
190
+ "dbt",
191
+ help="from a dbt project's target/manifest.json",
192
+ description="Tables from models/seeds/snapshots/sources, relationships from `relationships` tests and "
193
+ "foreign_key constraints, types from catalog.json (run `dbt docs generate` first for types).",
194
+ )
195
+ dp.add_argument("manifest", help="path to manifest.json (or the target/ directory)")
196
+ dp.add_argument("-o", "--output", default="schema.txt", help="a .txt file, or a directory for one file per folder")
197
+ dp.add_argument("--catalog", help="catalog.json for column types (default: next to the manifest, if present)")
198
+ dp.add_argument(
199
+ "--package",
200
+ action="append",
201
+ help="dbt package(s) to include (default: the root project; 'all' for every package)",
202
+ )
203
+ dp.add_argument(
204
+ "--select", action="append", help="glob on model name, fqn (a.b.c) or file path, e.g. 'marts.*' (repeatable)"
205
+ )
206
+ dp.add_argument("--no-sources", action="store_true", help="leave out dbt sources")
207
+ dp.set_defaults(func=cmd_import_dbt)
208
+
150
209
  sp = sub.add_parser("serve", help="live editor: reload on save, drag tables to arrange")
151
210
  paths(sp)
152
211
  sp.add_argument("--port", type=int, default=8765)
@@ -0,0 +1,412 @@
1
+ """`qdbd import dbt`: turn a dbt project's manifest.json (+ optional catalog.json) into QuickDBD text.
2
+
3
+ - Tables: models, seeds and snapshots of the selected packages, plus sources.
4
+ - Columns: documented columns, catalog columns (with real types) and columns used by tests.
5
+ - Primary keys: `primary_key` constraints, else the first column with both `unique` and
6
+ `not_null` tests. Other `unique` columns get UNIQUE.
7
+ - Relationships: `relationships` tests (incl. dbt_utils `relationships_where`) and
8
+ `foreign_key` constraints. A unique child column gives one-to-one (`-`), otherwise many-to-one (`>-`).
9
+ - Descriptions become QuickDBD doc comments.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import fnmatch
15
+ import json
16
+ import re
17
+ from dataclasses import dataclass, field
18
+ from pathlib import Path
19
+
20
+ HEADER = "# Generated by qdbd import dbt"
21
+ TABLE_TYPES = ("model", "seed", "snapshot")
22
+ REL_TESTS = ("relationships", "relationships_where")
23
+ _IDENT = re.compile(r"^[A-Za-z0-9_]+$")
24
+ _REF = re.compile(
25
+ r"""^\s*ref\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]+)['"]\s*)?(?:,\s*v(?:ersion)?\s*=\s*\S+\s*)?\)\s*$"""
26
+ )
27
+ _SOURCE = re.compile(r"""^\s*source\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)\s*$""")
28
+ _TYPE_ALIASES = {
29
+ "character varying": "varchar",
30
+ "timestamp without time zone": "timestamp",
31
+ "timestamp with time zone": "timestamptz",
32
+ "time without time zone": "time",
33
+ "time with time zone": "timetz",
34
+ "double precision": "double",
35
+ }
36
+
37
+
38
+ class DbtImportError(Exception):
39
+ pass
40
+
41
+
42
+ @dataclass
43
+ class Col:
44
+ name: str
45
+ type: str | None = None
46
+ description: str = ""
47
+ pk: bool = False
48
+ unique: bool = False
49
+ not_null: bool = False
50
+ fk: tuple[str, str, str] | None = None # (op, parent table uid, parent column)
51
+ order: int = 10_000
52
+
53
+
54
+ @dataclass
55
+ class Tbl:
56
+ uid: str
57
+ name: str # display name in the .txt
58
+ group: str
59
+ description: str = ""
60
+ cols: dict[str, Col] = field(default_factory=dict)
61
+
62
+ def col(self, name: str) -> Col:
63
+ key = name.lower()
64
+ for k, c in self.cols.items():
65
+ if k.lower() == key:
66
+ return c
67
+ c = self.cols[name] = Col(name)
68
+ return c
69
+
70
+
71
+ @dataclass
72
+ class ImportResult:
73
+ files: dict[str, str] # file name -> text
74
+ tables: int
75
+ relationships: int
76
+ skipped: list[str]
77
+
78
+
79
+ def _strip_quotes(s: str) -> str:
80
+ s = s.strip()
81
+ if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`[":
82
+ return s[1:-1]
83
+ if s.startswith("[") and s.endswith("]"):
84
+ return s[1:-1]
85
+ return s
86
+
87
+
88
+ def norm_type(t: str | None) -> str | None:
89
+ if not t:
90
+ return None
91
+ t = t.strip()
92
+ low = t.lower()
93
+ for long, short in _TYPE_ALIASES.items():
94
+ if low.startswith(long):
95
+ t = short + t[len(long) :]
96
+ break
97
+ t = re.sub(r"\s*,\s*", ",", t)
98
+ t = re.sub(r"\s*\(\s*", "(", t)
99
+ t = re.sub(r"\s*\)", ")", t)
100
+ t = re.sub(r"\s+", "_", t)
101
+ return t.replace("=", ":").replace("#", "").replace('"', "")
102
+
103
+
104
+ def quote(name: str) -> str:
105
+ name = name.replace('"', "'")
106
+ return name if _IDENT.match(name) else f'"{name}"'
107
+
108
+
109
+ def _comment_lines(text: str) -> list[str]:
110
+ out = []
111
+ for line in (text or "").replace("\r", "").split("\n"):
112
+ line = " ".join(line.split()).replace("#", "#")
113
+ if line:
114
+ out.append(f"# {line}")
115
+ return out
116
+
117
+
118
+ class _Importer:
119
+ def __init__(
120
+ self, manifest: dict, catalog: dict | None, packages: list[str] | None, select: list[str], sources: bool
121
+ ):
122
+ self.m = manifest
123
+ self.catalog = catalog or {}
124
+ self.select = select
125
+ self.sources = sources
126
+ root = (manifest.get("metadata") or {}).get("project_name")
127
+ if packages and "all" in packages:
128
+ self.packages = None
129
+ elif packages:
130
+ self.packages = set(packages)
131
+ else:
132
+ self.packages = {root} if root else None
133
+ self.tables: dict[str, Tbl] = {}
134
+ self.skipped: list[str] = []
135
+ self.by_name: dict[str, list[str]] = {}
136
+
137
+ # -- selection ----------------------------------------------------------
138
+ def _wanted(self, node: dict) -> bool:
139
+ if self.packages is not None and node.get("package_name") not in self.packages:
140
+ return False
141
+ if not self.select:
142
+ return True
143
+ cands = [node.get("name", ""), ".".join(node.get("fqn", [])), node.get("original_file_path", "")]
144
+ return any(fnmatch.fnmatch(c, pat) for pat in self.select for c in cands)
145
+
146
+ @staticmethod
147
+ def _group(node: dict) -> str:
148
+ rt = node["resource_type"]
149
+ if rt == "source":
150
+ return "sources"
151
+ if rt != "model":
152
+ return rt + "s"
153
+ fqn = node.get("fqn", [])
154
+ return fqn[1] if len(fqn) > 2 else "models"
155
+
156
+ def collect(self) -> None:
157
+ nodes = [n for n in self.m.get("nodes", {}).values() if n.get("resource_type") in TABLE_TYPES]
158
+ if self.sources:
159
+ nodes += list(self.m.get("sources", {}).values())
160
+ nodes = [n for n in nodes if self._wanted(n)]
161
+ names: dict[str, int] = {}
162
+ for n in nodes:
163
+ names[self._base_name(n)] = names.get(self._base_name(n), 0) + 1
164
+ for n in sorted(nodes, key=lambda n: n["unique_id"]):
165
+ name = self._base_name(n)
166
+ if names[name] > 1: # same name in two packages
167
+ name = f"{n.get('package_name')}.{name}"
168
+ t = Tbl(n["unique_id"], name, self._group(n), n.get("description") or "")
169
+ self._columns(t, n)
170
+ self.tables[t.uid] = t
171
+ self.by_name.setdefault(n.get("name", "").lower(), []).append(t.uid)
172
+ if n.get("resource_type") == "source":
173
+ self.by_name.setdefault(f"source:{n['source_name']}.{n['name']}".lower(), []).append(t.uid)
174
+
175
+ @staticmethod
176
+ def _base_name(n: dict) -> str:
177
+ if n.get("resource_type") == "source":
178
+ return f"{n['source_name']}.{n['name']}"
179
+ return n.get("name", "")
180
+
181
+ def _columns(self, t: Tbl, n: dict) -> None:
182
+ cat = (self.catalog.get("sources" if n["resource_type"] == "source" else "nodes") or {}).get(n["unique_id"])
183
+ if cat:
184
+ for cname, c in cat.get("columns", {}).items():
185
+ col = t.col(c.get("name") or cname)
186
+ col.type = c.get("type")
187
+ col.order = int(c.get("index") or col.order)
188
+ for i, (cname, c) in enumerate(n.get("columns", {}).items()):
189
+ col = t.col(c.get("name") or cname)
190
+ col.description = c.get("description") or ""
191
+ col.type = col.type or c.get("data_type")
192
+ if not cat:
193
+ col.order = i
194
+ for con in c.get("constraints") or []:
195
+ self._constraint(t, col, con)
196
+ for con in n.get("constraints") or []:
197
+ cols = con.get("columns") or []
198
+ if con.get("type") == "primary_key":
199
+ for cn in cols:
200
+ t.col(cn).pk = True
201
+ elif con.get("type") == "unique" and len(cols) == 1:
202
+ t.col(cols[0]).unique = True
203
+ elif con.get("type") == "foreign_key" and len(cols) == 1:
204
+ self._constraint(t, t.col(cols[0]), con)
205
+
206
+ def _constraint(self, t: Tbl, col: Col, con: dict) -> None:
207
+ kind = con.get("type")
208
+ if kind == "primary_key":
209
+ col.pk = True
210
+ elif kind == "unique":
211
+ col.unique = True
212
+ elif kind == "not_null":
213
+ col.not_null = True
214
+ elif kind == "foreign_key":
215
+ target, tcol = con.get("to"), (con.get("to_columns") or [None])[0]
216
+ if not target and con.get("expression"): # "other_table (id)"
217
+ m = re.match(r"\s*([\w.\"`]+)\s*\(\s*([\w\"`]+)\s*\)", con["expression"])
218
+ if m:
219
+ target, tcol = m.group(1).split(".")[-1].strip('"`'), m.group(2).strip('"`')
220
+ # Resolved in link(): the target table may not have been collected yet.
221
+ self._constraint_fks.append((t.uid, col.name, target or "", tcol or ""))
222
+
223
+ _pending: list[tuple[str, str, str, str]]
224
+ _constraint_fks: list[tuple[str, str, str, str]]
225
+
226
+ def _resolve(self, text: str) -> str | None:
227
+ m = _REF.match(text)
228
+ if m:
229
+ name = (m.group(2) or m.group(1)).lower()
230
+ else:
231
+ s = _SOURCE.match(text)
232
+ name = f"source:{s.group(1)}.{s.group(2)}".lower() if s else text.strip().lower()
233
+ uids = self.by_name.get(name, [])
234
+ return uids[0] if len(uids) == 1 else None
235
+
236
+ # -- tests --------------------------------------------------------------
237
+ def tests(self) -> None:
238
+ uniq: dict[tuple[str, str], bool] = {}
239
+ notnull: dict[tuple[str, str], bool] = {}
240
+ for n in self.m.get("nodes", {}).values():
241
+ if n.get("resource_type") != "test":
242
+ continue
243
+ meta = n.get("test_metadata") or {}
244
+ kw = meta.get("kwargs") or {}
245
+ name = meta.get("name")
246
+ child = n.get("attached_node") or self._child_from_kwargs(kw)
247
+ colname = _strip_quotes(kw.get("column_name") or n.get("column_name") or "")
248
+ if not child or child not in self.tables or not colname:
249
+ continue
250
+ if name == "unique":
251
+ uniq[(child, colname.lower())] = True
252
+ elif name == "not_null":
253
+ notnull[(child, colname.lower())] = True
254
+ elif name in REL_TESTS:
255
+ deps = [d for d in (n.get("depends_on") or {}).get("nodes", []) if d != child]
256
+ parent = deps[0] if deps else (self._resolve(kw.get("to", "")) or child)
257
+ field_name = _strip_quotes(kw.get("field") or "")
258
+ if parent not in self.tables:
259
+ self.skipped.append(f"{self.tables[child].name}.{colname}: parent {parent} not in selection")
260
+ continue
261
+ if field_name:
262
+ self._pending.append((child, colname, parent, field_name))
263
+ for (uid, cname), _ in uniq.items():
264
+ self.tables[uid].col(cname).unique = True
265
+ for (uid, cname), _ in notnull.items():
266
+ self.tables[uid].col(cname).not_null = True
267
+ for t in self.tables.values(): # implied primary key: unique + not_null
268
+ if any(c.pk for c in t.cols.values()):
269
+ continue
270
+ for c in sorted(t.cols.values(), key=lambda c: c.order):
271
+ if c.unique and c.not_null:
272
+ c.pk = True
273
+ break
274
+ for t in self.tables.values():
275
+ for c in t.cols.values():
276
+ if c.pk:
277
+ c.unique = False
278
+
279
+ def _child_from_kwargs(self, kw: dict) -> str | None:
280
+ m = re.search(r"(ref\([^)]*\)|source\([^)]*\))", kw.get("model") or "")
281
+ return self._resolve(m.group(1)) if m else None
282
+
283
+ def link(self) -> int:
284
+ for child, ccol, target, tcol in self._constraint_fks:
285
+ parent = self._resolve(target) if target else None
286
+ if parent and tcol:
287
+ self._pending.append((child, ccol, parent, tcol))
288
+ else:
289
+ self.skipped.append(f"{self.tables[child].name}.{ccol}: foreign_key target '{target}' not found")
290
+ count = 0
291
+ seen = set()
292
+ for child, ccol, parent, pcol in self._pending:
293
+ key = (child, ccol.lower(), parent, pcol.lower())
294
+ if key in seen:
295
+ continue
296
+ seen.add(key)
297
+ c = self.tables[child].col(ccol)
298
+ p = self.tables[parent].col(pcol)
299
+ op = "-" if (c.unique or c.pk) else ">-"
300
+ if c.fk is None:
301
+ c.fk = (op, parent, p.name)
302
+ elif p.fk is None: # one FK per column line: put the second one on the parent side
303
+ p.fk = ({"-": "-", ">-": "-<"}[op], child, c.name)
304
+ else:
305
+ self.skipped.append(
306
+ f"{self.tables[child].name}.{ccol} -> {self.tables[parent].name}.{pcol}: "
307
+ "column already has a relationship"
308
+ )
309
+ continue
310
+ count += 1
311
+ return count
312
+
313
+ # -- output -------------------------------------------------------------
314
+ def render_table(self, t: Tbl) -> list[str]:
315
+ lines = _comment_lines(t.description)
316
+ lines += [quote(t.name), "-"]
317
+ for c in sorted(t.cols.values(), key=lambda c: (c.order, c.name.lower())):
318
+ lines += _comment_lines(c.description)
319
+ parts = [quote(c.name)]
320
+ if c.pk:
321
+ parts.append("PK")
322
+ ty = norm_type(c.type)
323
+ if ty:
324
+ parts.append(ty)
325
+ if c.unique:
326
+ parts.append("UNIQUE")
327
+ if c.fk:
328
+ op, uid, pcol = c.fk
329
+ parts += ["FK", op, f"{quote(self.tables[uid].name)}.{quote(pcol)}"]
330
+ lines.append(" ".join(parts))
331
+ return lines
332
+
333
+ def run(self) -> tuple[dict[str, list[Tbl]], int]:
334
+ self._pending = []
335
+ self._constraint_fks = []
336
+ self.collect()
337
+ self.tests()
338
+ rels = self.link()
339
+ groups: dict[str, list[Tbl]] = {}
340
+ for t in sorted(self.tables.values(), key=lambda t: (t.group, t.name.lower())):
341
+ groups.setdefault(t.group, []).append(t)
342
+ return groups, rels
343
+
344
+
345
+ def load_json(path: Path) -> dict:
346
+ try:
347
+ return json.loads(path.read_text(encoding="utf-8"))
348
+ except FileNotFoundError:
349
+ raise DbtImportError(f"no such file: {path}") from None
350
+ except json.JSONDecodeError as e:
351
+ raise DbtImportError(f"{path}: not valid JSON ({e})") from None
352
+
353
+
354
+ def import_dbt(
355
+ manifest_path: Path,
356
+ catalog_path: Path | None = None,
357
+ packages: list[str] | None = None,
358
+ select: list[str] | None = None,
359
+ sources: bool = True,
360
+ split: bool = False,
361
+ ) -> ImportResult:
362
+ if manifest_path.is_dir():
363
+ manifest_path = manifest_path / "manifest.json"
364
+ manifest = load_json(manifest_path)
365
+ if "nodes" not in manifest:
366
+ raise DbtImportError(f"{manifest_path}: not a dbt manifest (no 'nodes')")
367
+ if catalog_path is None and (manifest_path.parent / "catalog.json").exists():
368
+ catalog_path = manifest_path.parent / "catalog.json"
369
+ catalog = load_json(catalog_path) if catalog_path else None
370
+ imp = _Importer(manifest, catalog, packages, select or [], sources)
371
+ groups, rels = imp.run()
372
+ project = (manifest.get("metadata") or {}).get("project_name") or "dbt"
373
+ head = [
374
+ f"{HEADER} from {manifest_path.name} (project {project}).",
375
+ "# Re-run the import to refresh; manual edits will be overwritten.",
376
+ "",
377
+ ]
378
+ files: dict[str, str] = {}
379
+ if split:
380
+ for g, tables in groups.items():
381
+ body: list[str] = list(head)
382
+ for t in tables:
383
+ body += imp.render_table(t) + [""]
384
+ files[f"{g}.txt"] = "\n".join(body).rstrip("\n") + "\n"
385
+ else:
386
+ body = list(head)
387
+ for tables in groups.values():
388
+ for t in tables:
389
+ body += imp.render_table(t) + [""]
390
+ files["schema.txt"] = "\n".join(body).rstrip("\n") + "\n"
391
+ return ImportResult(files, len(imp.tables), rels, imp.skipped)
392
+
393
+
394
+ def write_result(result: ImportResult, out: Path) -> list[Path]:
395
+ """Write to a .txt file, or into a directory (one file per group). Stale generated files are removed."""
396
+ written: list[Path] = []
397
+ if out.suffix == ".txt":
398
+ if len(result.files) != 1:
399
+ raise DbtImportError("--split needs a directory for -o")
400
+ out.parent.mkdir(parents=True, exist_ok=True)
401
+ out.write_text(next(iter(result.files.values())), encoding="utf-8")
402
+ return [out]
403
+ out.mkdir(parents=True, exist_ok=True)
404
+ for name, text in result.files.items():
405
+ p = out / name
406
+ p.write_text(text, encoding="utf-8")
407
+ written.append(p)
408
+ keep = {p.name for p in written}
409
+ for old in out.glob("*.txt"):
410
+ if old.name not in keep and old.read_text(encoding="utf-8").startswith(HEADER):
411
+ old.unlink()
412
+ return written
@@ -20,7 +20,7 @@ STYLE = {
20
20
  "rowSize": 12,
21
21
  "padX": 12,
22
22
  "gap": 20,
23
- "keyW": 16,
23
+ "keyW": 19,
24
24
  "badgeSize": 9,
25
25
  "badgePad": 4,
26
26
  "badgeGap": 4,