quickdbd-cli 0.1.5__tar.gz → 0.1.6__tar.gz

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/.gitignore

@@ -221,3 +221,4 @@ __marimo__/
 .venv/
 .superpowers/
 site/
+jaffle-erd/

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quickdbd-cli
-Version: 0.1.5
+Version: 0.1.6
 Summary: Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit.
 Project-URL: Homepage, https://github.com/MarkPhamm/quickdbd-cli
 Author: Minh (Mark) Pham
@@ -83,6 +83,8 @@ qdbd build erd/ -o erd.html
 - **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
 - **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
 - **Docs:** model and column descriptions become doc comments, shown as tooltips.
+- **`--infer-fks`:** also links `customer_id`-style columns to the table they name, when no relationship is declared. Ambiguous matches are skipped, never guessed.
+- **`--check`:** in CI, fails when the committed files are out of date.
 - **Options:**
   - `--select 'marts.*'` narrows by name, fqn or path (repeatable)
   - `--package all` includes installed packages
@@ -183,7 +185,7 @@ jobs:
     permissions: { contents: read, pull-requests: write }
     steps:
       - uses: actions/checkout@v4
-      - uses: MarkPhamm/quickdbd-cli@v0.1.5
+      - uses: MarkPhamm/quickdbd-cli@v0.1.6
         with:
           path: erd/                 # schema file(s) or folder(s)
           output: site/index.html
@@ -207,7 +209,7 @@ Outputs are `html`, `changed` and `diff-file`.
 ```yaml
 repos:
   - repo: https://github.com/MarkPhamm/quickdbd-cli
-    rev: v0.1.5
+    rev: v0.1.6
     hooks:
       - id: qdbd-check
         files: ^erd/.*\.txt$   # point at your schema files

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/README.md

@@ -55,6 +55,8 @@ qdbd build erd/ -o erd.html
 - **Relationships:** `relationships` tests (including dbt_utils `relationships_where`) and `foreign_key` constraints.
 - **Primary keys:** `primary_key` constraints, or else a column with both `unique` and `not_null` tests.
 - **Docs:** model and column descriptions become doc comments, shown as tooltips.
+- **`--infer-fks`:** also links `customer_id`-style columns to the table they name, when no relationship is declared. Ambiguous matches are skipped, never guessed.
+- **`--check`:** in CI, fails when the committed files are out of date.
 - **Options:**
   - `--select 'marts.*'` narrows by name, fqn or path (repeatable)
   - `--package all` includes installed packages
@@ -155,7 +157,7 @@ jobs:
     permissions: { contents: read, pull-requests: write }
     steps:
       - uses: actions/checkout@v4
-      - uses: MarkPhamm/quickdbd-cli@v0.1.5
+      - uses: MarkPhamm/quickdbd-cli@v0.1.6
         with:
           path: erd/                 # schema file(s) or folder(s)
           output: site/index.html
@@ -179,7 +181,7 @@ Outputs are `html`, `changed` and `diff-file`.
 ```yaml
 repos:
   - repo: https://github.com/MarkPhamm/quickdbd-cli
-    rev: v0.1.5
+    rev: v0.1.6
     hooks:
       - id: qdbd-check
         files: ^erd/.*\.txt$   # point at your schema files

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quickdbd-cli"
-version = "0.1.5"
+version = "0.1.6"
 description = "Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit."
 readme = "README.md"
 license = "Apache-2.0"

quickdbd_cli-0.1.6/src/quickdbd_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.6"

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/src/quickdbd_cli/cli.py

@@ -145,7 +145,7 @@ def cmd_layout(args) -> int:
 
 
 def cmd_import_dbt(args) -> int:
-    from .dbt import DbtImportError, import_dbt, write_result
+    from .dbt import DbtImportError, import_dbt, stale_files, write_result
     from .parser import compile_sources
 
     out = Path(args.output)
@@ -158,6 +158,7 @@ def cmd_import_dbt(args) -> int:
             select=args.select,
             sources=not args.no_sources,
             split=split,
+            infer_fks=args.infer_fks,
         )
     except DbtImportError as e:
         print(f"error: {e}", file=sys.stderr)
@@ -169,14 +170,32 @@ def cmd_import_dbt(args) -> int:
             print(d.format(), file=sys.stderr)
         print("error: generated schema did not compile (please report this as a bug)", file=sys.stderr)
         return 1
-    written = write_result(result, out)
     for s in result.skipped:
         print(f"skipped: {s}", file=sys.stderr)
+    for s in result.inferred:
+        print(f"inferred: {s}", file=sys.stderr)
+    inferred = f" ({len(result.inferred)} inferred)" if args.infer_fks else ""
+    if args.check:
+        stale = stale_files(result, out)
+        if stale:
+            for s in stale:
+                print(f"stale: {s}", file=sys.stderr)
+            print(
+                f"error: schema files are out of date; run `qdbd import dbt {args.manifest} -o {args.output}` "
+                "and commit the result",
+                file=sys.stderr,
+            )
+            return 1
+        print(f"Up to date: {out}: {result.tables} tables, {result.relationships} relationships{inferred}")
+        return 0
+    written = write_result(result, out)
     where = str(out) if not split else f"{out}/ ({len(written)} files)"
-    print(f"Wrote {where}: {result.tables} tables, {result.relationships} relationships")
+    print(f"Wrote {where}: {result.tables} tables, {result.relationships} relationships{inferred}")
     if result.tables and not result.relationships:
         print(
-            "hint: no relationships found; add `relationships` tests or foreign_key constraints in dbt", file=sys.stderr
+            "hint: no relationships found; add `relationships` tests or foreign_key constraints in dbt, "
+            "or try --infer-fks",
+            file=sys.stderr,
         )
     return 0
 
@@ -357,6 +376,14 @@ def build_parser() -> argparse.ArgumentParser:
         "--select", action="append", help="glob on model name, fqn (a.b.c) or file path, e.g. 'marts.*' (repeatable)"
     )
     dp.add_argument("--no-sources", action="store_true", help="leave out dbt sources")
+    dp.add_argument(
+        "--infer-fks",
+        action="store_true",
+        help="also link <entity>_id columns to the table named after the entity when no relationship is declared",
+    )
+    dp.add_argument(
+        "--check", action="store_true", help="don't write; exit 1 if the files at -o are out of date (for CI)"
+    )
     dp.set_defaults(func=cmd_import_dbt)
 
     sp = sub.add_parser("serve", help="live editor: reload on save, drag tables to arrange")

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/src/quickdbd_cli/dbt.py

@@ -35,6 +35,47 @@ _TYPE_ALIASES = {
 }
 
 
+_ENTITY = re.compile(r"^(.+?)(_id|_key|_sk|Id|ID|Key)$")
+_LAYERS = ("staging_", "stg_", "int_", "base_", "raw_", "src_", "dim_", "fct_", "fact_", "mart_", "d_", "f_")
+
+
+def _layer(name: str) -> tuple[str, str]:
+    """Split a dbt-style layer prefix off a table name: 'stg_orders' -> ('stg_', 'orders')."""
+    low = name.lower()
+    for p in _LAYERS:
+        if low.startswith(p) and len(low) > len(p):
+            return p, low[len(p) :]
+    return "", low
+
+
+def _prefix_rank(child: str, parent: str) -> int:
+    """Lower is better: facts point at dimensions; otherwise stay within the same layer (stg_ -> stg_)."""
+    if child in ("fct_", "fact_", "f_") and parent in ("dim_", "d_"):
+        return 0
+    return 1 if parent == child else 2
+
+
+def _same_entity(table_base: str, entity: str) -> bool:
+    forms = {entity, entity + "s", entity + "es"}
+    if entity.endswith("y"):
+        forms.add(entity[:-1] + "ies")
+    return table_base in forms
+
+
+def _key_column(t: Tbl, name: str) -> Col | None:
+    """The parent column an `<entity>_id` points at: same name, else `id`; primary keys first."""
+
+    def norm(n: str) -> str:  # customer_id == customerId == CUSTOMER_ID
+        return n.lower().replace("_", "")
+
+    cols = {norm(c.name): c for c in t.cols.values()}
+    same, plain = cols.get(norm(name)), cols.get("id")
+    for c in (same, plain):
+        if c is not None and c.pk:
+            return c
+    return same or plain
+
+
 class DbtImportError(Exception):
     pass
 
@@ -74,6 +115,7 @@ class ImportResult:
     tables: int
     relationships: int
     skipped: list[str]
+    inferred: list[str] = field(default_factory=list)
 
 
 def _strip_quotes(s: str) -> str:
@@ -117,9 +159,17 @@ def _comment_lines(text: str) -> list[str]:
 
 class _Importer:
     def __init__(
-        self, manifest: dict, catalog: dict | None, packages: list[str] | None, select: list[str], sources: bool
+        self,
+        manifest: dict,
+        catalog: dict | None,
+        packages: list[str] | None,
+        select: list[str],
+        sources: bool,
+        infer: bool = False,
     ):
         self.m = manifest
+        self.infer = infer
+        self.inferred: list[str] = []
         self.catalog = catalog or {}
         self.select = select
         self.sources = sources
@@ -310,6 +360,62 @@ class _Importer:
             count += 1
         return count
 
+    # -- inference from column names (--infer-fks) ---------------------------
+    def infer_fks(self) -> int:
+        """Link `<entity>_id` columns to the one table named after the entity, when no relationship is declared.
+
+        Candidates are tables whose name, without a dbt layer prefix (stg_, dim_, fct_, ...) and in singular
+        or plural, equals the entity, and that have a matching key column. Among several, prefer a dim_ parent
+        for a fct_ child, else the same prefix as the child, then the same folder, then a declared primary key;
+        a tie is skipped rather than guessed.
+        """
+        # columns already on the parent side of a declared relationship
+        referenced = {(c.fk[1], c.fk[2].lower()) for t in self.tables.values() for c in t.cols.values() if c.fk}
+        count = 0
+        for t in sorted(self.tables.values(), key=lambda t: t.uid):
+            pks = [c for c in t.cols.values() if c.pk]
+            for c in sorted(t.cols.values(), key=lambda c: (c.order, c.name.lower())):
+                if c.fk is not None or (c.pk and len(pks) == 1) or (t.uid, c.name.lower()) in referenced:
+                    continue
+                m = _ENTITY.match(c.name)
+                if not m:
+                    continue
+                entity = m.group(1).rstrip("_").lower()
+                found = self._infer_target(t, c, entity)
+                if found is None:
+                    continue
+                parent, pcol = found
+                c.fk = (">-", parent.uid, pcol.name)
+                note = "Relationship inferred from the column name (qdbd import dbt --infer-fks)."
+                c.description = (c.description + "\n" if c.description else "") + note
+                self.inferred.append(f"{t.name}.{c.name} -> {parent.name}.{pcol.name}")
+                count += 1
+        return count
+
+    def _infer_target(self, child: Tbl, col: Col, entity: str) -> tuple[Tbl, Col] | None:
+        child_prefix = _layer(child.name.split(".")[-1])[0]
+        ranked = []
+        for t in self.tables.values():
+            if t.uid == child.uid:
+                continue
+            prefix, base = _layer(t.name.split(".")[-1])
+            if not _same_entity(base, entity):
+                continue
+            key = _key_column(t, col.name)
+            if key is None:
+                continue
+            score = (_prefix_rank(child_prefix, prefix), t.group != child.group, not key.pk)
+            ranked.append((score, t, key))
+        if not ranked:
+            return None
+        ranked.sort(key=lambda r: r[0])
+        best = [r for r in ranked if r[0] == ranked[0][0]]
+        if len(best) > 1:
+            names = ", ".join(sorted(r[1].name for r in best))
+            self.skipped.append(f"{child.name}.{col.name}: ambiguous ({names}), not inferred")
+            return None
+        return best[0][1], best[0][2]
+
     # -- output -------------------------------------------------------------
     def render_table(self, t: Tbl) -> list[str]:
         lines = _comment_lines(t.description)
@@ -336,6 +442,8 @@ class _Importer:
         self.collect()
         self.tests()
         rels = self.link()
+        if self.infer:
+            rels += self.infer_fks()
         groups: dict[str, list[Tbl]] = {}
         for t in sorted(self.tables.values(), key=lambda t: (t.group, t.name.lower())):
             groups.setdefault(t.group, []).append(t)
@@ -358,6 +466,7 @@ def import_dbt(
     select: list[str] | None = None,
     sources: bool = True,
     split: bool = False,
+    infer_fks: bool = False,
 ) -> ImportResult:
     if manifest_path.is_dir():
         manifest_path = manifest_path / "manifest.json"
@@ -367,7 +476,7 @@ def import_dbt(
     if catalog_path is None and (manifest_path.parent / "catalog.json").exists():
         catalog_path = manifest_path.parent / "catalog.json"
     catalog = load_json(catalog_path) if catalog_path else None
-    imp = _Importer(manifest, catalog, packages, select or [], sources)
+    imp = _Importer(manifest, catalog, packages, select or [], sources, infer=infer_fks)
     groups, rels = imp.run()
     project = (manifest.get("metadata") or {}).get("project_name") or "dbt"
     head = [
@@ -388,7 +497,7 @@ def import_dbt(
             for t in tables:
                 body += imp.render_table(t) + [""]
         files["schema.txt"] = "\n".join(body).rstrip("\n") + "\n"
-    return ImportResult(files, len(imp.tables), rels, imp.skipped)
+    return ImportResult(files, len(imp.tables), rels, imp.skipped, imp.inferred)
 
 
 def write_result(result: ImportResult, out: Path) -> list[Path]:
@@ -410,3 +519,24 @@ def write_result(result: ImportResult, out: Path) -> list[Path]:
         if old.name not in keep and old.read_text(encoding="utf-8").startswith(HEADER):
             old.unlink()
     return written
+
+
+def stale_files(result: ImportResult, out: Path) -> list[str]:
+    """What `write_result` would change: files that differ, are missing, or would be removed. Empty = up to date."""
+    if out.suffix == ".txt":
+        text = next(iter(result.files.values()))
+        if not out.exists():
+            return [f"{out} (missing)"]
+        return [] if out.read_text(encoding="utf-8") == text else [f"{out} (out of date)"]
+    problems = []
+    for name, text in sorted(result.files.items()):
+        p = out / name
+        if not p.exists():
+            problems.append(f"{p} (missing)")
+        elif p.read_text(encoding="utf-8") != text:
+            problems.append(f"{p} (out of date)")
+    if out.is_dir():
+        for old in sorted(out.glob("*.txt")):
+            if old.name not in result.files and old.read_text(encoding="utf-8").startswith(HEADER):
+                problems.append(f"{old} (no longer generated)")
+    return problems

{quickdbd_cli-0.1.5 → quickdbd_cli-0.1.6}/tests/test_dbt.py

@@ -229,3 +229,96 @@ def test_helpers():
     assert norm_type("NUMBER(38, 0)") == "NUMBER(38,0)"
     assert norm_type("struct<a int>") == "struct<a_int>"
     assert norm_type(None) is None
+
+
+# ------------------------------------------------------------------ --infer-fks / --check
+
+
+def test_infer_fks_on_jaffle():
+    res = import_dbt(JAFFLE / "manifest.json", infer_fks=True)
+    s = compiled(res)
+    assert res.relationships == 4
+    assert sorted(res.inferred) == [
+        "raw_payments.order_id -> raw_orders.id",  # same raw_ prefix; parent key is plain `id`
+        "stg_orders.customer_id -> stg_customers.customer_id",  # same stg_ prefix wins over marts customers
+        "stg_payments.order_id -> stg_orders.order_id",
+    ]
+    assert "raw_orders.user_id" not in " ".join(res.inferred)  # no `users` table: nothing to link
+    col = s.tables["stg_orders"].columns["customer_id"]
+    assert col.fk and "inferred from the column name" in col.comments[-1]
+    # declared relationships are untouched and not duplicated
+    assert ("orders", "customer_id", "many", "customers", "customer_id", "one") in rels(s)
+
+
+def infer_manifest():
+    def m(uid, name, cols, fqn=None, pk=None):
+        n = node(uid, name, fqn=fqn, cols=[(c, "int", [{"type": "primary_key"}] if c == pk else None) for c in cols])
+        return n
+
+    nodes = {
+        "model.shop.dim_category": m("model.shop.dim_category", "dim_category", ["category_id"], pk="category_id"),
+        "model.shop.categories": m(
+            "model.shop.categories",
+            "categories",
+            ["category_id"],
+            pk="category_id",
+            fqn=["shop", "other", "categories"],
+        ),
+        "model.shop.fct_sales": m(
+            "model.shop.fct_sales",
+            "fct_sales",
+            ["sale_id", "categoryId", "store_key", "owner_id", "region_id"],
+            pk="sale_id",
+        ),
+        "model.shop.dim_stores": m("model.shop.dim_stores", "dim_stores", ["store_key"], pk="store_key"),
+        "model.shop.people": m("model.shop.people", "people", ["id"]),
+        "model.shop.owners_a": m("model.shop.owners_a", "a_owner", ["owner_id"], pk="owner_id"),
+        "model.shop.dim_owner": m(
+            "model.shop.dim_owner", "dim_owner", ["owner_id"], pk="owner_id", fqn=["shop", "x", "dim_owner"]
+        ),
+        "model.shop.dim_region": m("model.shop.dim_region", "dim_region", ["region_id"], pk="region_id"),
+        "model.shop.dim_regions": m("model.shop.dim_regions", "dim_regions", ["region_id"], pk="region_id"),
+        "model.shop.fct_owner": m(
+            "model.shop.fct_owner", "fct_owner", ["owner_id"], pk="owner_id", fqn=["shop", "x", "fct_owner"]
+        ),
+    }
+    return {"metadata": {"project_name": "shop"}, "nodes": nodes, "sources": {}}
+
+
+def test_infer_rules(tmp_path):
+    p = tmp_path / "manifest.json"
+    p.write_text(json.dumps(infer_manifest()))
+    res = import_dbt(p, infer_fks=True)
+    compiled(res)
+    inferred = sorted(res.inferred)
+    # camelCase `categoryId` -> category; both dim_category and categories match, the fct_ child's folder decides
+    assert "fct_sales.categoryId -> dim_category.category_id" in inferred
+    assert "fct_sales.store_key -> dim_stores.store_key" in inferred  # `_key` suffix, plural table
+    # facts point at dimensions: dim_owner beats the same-layer fct_owner
+    assert "fct_sales.owner_id -> dim_owner.owner_id" in inferred
+    # region_id: dim_region and dim_regions tie on every rule -> skipped as ambiguous, never guessed
+    assert not any(x.startswith("fct_sales.region_id") for x in inferred)
+    assert any("fct_sales.region_id: ambiguous (dim_region, dim_regions)" in x for x in res.skipped)
+    # single primary keys are never turned into FKs
+    assert not any(x.startswith(("dim_category.", "dim_stores.")) for x in inferred)
+
+
+def test_infer_off_by_default():
+    res = import_dbt(JAFFLE / "manifest.json")
+    assert res.inferred == [] and res.relationships == 1
+
+
+def test_check_mode(tmp_path, capsys):
+    out = tmp_path / "erd"
+    assert main(["import", "dbt", str(JAFFLE), "-o", str(out), "--check"]) == 1  # nothing written yet
+    assert "(missing)" in capsys.readouterr().err
+    assert not out.exists()
+    assert main(["import", "dbt", str(JAFFLE), "-o", str(out)]) == 0
+    assert main(["import", "dbt", str(JAFFLE), "-o", str(out), "--check"]) == 0
+    assert "Up to date" in capsys.readouterr().out
+    assert main(["import", "dbt", str(JAFFLE), "-o", str(out), "--check", "--infer-fks"]) == 1
+    err = capsys.readouterr().err
+    assert "(out of date)" in err and "commit the result" in err
+    (out / "old.txt").write_text(HEADER + " from x\n")
+    assert main(["import", "dbt", str(JAFFLE), "-o", str(out), "--check"]) == 1
+    assert "no longer generated" in capsys.readouterr().err

quickdbd_cli-0.1.5/src/quickdbd_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.5"