quickdbd-cli 0.4.3__tar.gz → 0.6.0__tar.gz

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: quickdbd-cli
-Version: 0.4.3
+Version: 0.6.0
 Summary: Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit.
 Project-URL: Homepage, https://qdbd.dev/
 Project-URL: Documentation, https://qdbd.dev/guide/
@@ -25,6 +25,7 @@ Classifier: Topic :: Database
 Classifier: Topic :: Software Development :: Documentation
 Requires-Python: >=3.11
 Requires-Dist: mini-racer>=0.14
+Requires-Dist: pyyaml>=6
 Requires-Dist: resvg-py>=0.5
 Provides-Extra: dev
 Requires-Dist: fonttools>=4.50; extra == 'dev'
@@ -193,7 +194,7 @@ jobs:
     permissions: { contents: read, pull-requests: write }
     steps:
       - uses: actions/checkout@v4
-      - uses: MarkPhamm/quickdbd-cli@v0.4.2
+      - uses: MarkPhamm/quickdbd-cli@v0.6.0
         with:
           path: erd/                  # schema file(s) or folder(s)
           output: site/index.html

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/README.md

@@ -155,7 +155,7 @@ jobs:
     permissions: { contents: read, pull-requests: write }
     steps:
       - uses: actions/checkout@v4
-      - uses: MarkPhamm/quickdbd-cli@v0.4.2
+      - uses: MarkPhamm/quickdbd-cli@v0.6.0
         with:
           path: erd/                  # schema file(s) or folder(s)
           output: site/index.html

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quickdbd-cli"
-version = "0.4.3"
+version = "0.6.0"
 description = "Compile QuickDBD-syntax schema files into interactive HTML ER diagrams, with no table limit."
 readme = "README.md"
 license = "Apache-2.0"
@@ -25,6 +25,7 @@ classifiers = [
 ]
 dependencies = [
   "mini-racer>=0.14",
+  "pyyaml>=6",
   "resvg-py>=0.5",
 ]
 

quickdbd_cli-0.6.0/src/quickdbd_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/assets/syntax-reference.md

@@ -79,6 +79,7 @@ Declare a relationship once, in either direction.
 - Tags give tables a role and a header colour: `# @tag: gold, pii` above a table. Names like `dim_`, `fct_`, `stg_`, `int_`, `raw_` are tagged automatically.
 - `# @color gold #b8860b` sets a tag's colour (anywhere in any file).
 - Columns take tags too, on or above the column line: `discount decimal # @tag: added`. The row is tinted in the tag's colour; `added` is green, `removed` is red and struck through. Use them to mark proposed column changes for review.
+- `# @dbt: ../my_dbt_project` (anywhere in one file; the path is relative to that file) links a dbt project. Tables matching a model, seed, snapshot or source (`"source_name.table"`) by name get its yml description (shown before the comment), tags and `data_type` (for columns with no type). Where the schema and the yml disagree, `qdbd check` warns (`--strict` fails). `# @dbt: none` above a table skips it.
 
 ## Settings and layout sections
 
@@ -128,7 +129,7 @@ Ref: order_items.(order_id, region) > orders.(id, region)
 - Drawn: tables (`schema.table`, `public.users` = `users`), columns and types, `pk`, `null`, `unique`, `increment`, `default:`, indexes, `note`/`Note`, every `Ref` form (short, long, inline, composite, with `?` for an optional side), `TableGroup`, `headercolor`, `TablePartial`.
 - Accepted but not drawn: `Project`, `Enum` (the column shows the enum name as its type), `checks`, `records`, sticky notes, ref `delete:`/`update:`, `TableGroup` colours and notes.
 - `use` imports aren't followed: every file you give qdbd is read into one namespace.
-- `// @tag:` and `// @color` comments work; plain `//` comments are not documentation (use `note`).
+- `// @tag:`, `// @color` and `// @dbt:` comments work; plain `//` comments are not documentation (use `note`).
 
 # Notes
 

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/cli.py

@@ -32,11 +32,12 @@ def _open(args) -> Project:
     return Project.open([Path(p) for p in args.paths], Path(args.layout) if getattr(args, "layout", None) else None)
 
 
+def _name(p: Path) -> str:
+    return p.stem if p.is_file() else (p.resolve().name or "diagram")
+
+
 def _title(project: Project) -> str:
-    if len(project.inputs) == 1:
-        p = project.inputs[0]
-        return p.stem if p.is_file() else (p.resolve().name or "diagram")
-    return "diagram"
+    return _name(project.inputs[0]) if len(project.inputs) == 1 else "diagram"
 
 
 def _github_annotation(d: Diagnostic, level: str) -> str:
@@ -57,14 +58,14 @@ def _symbols(schema: Schema) -> dict:
     tables = []
     for t in schema.tables.values():
         cols = [
-            {"name": c.name, "line": c.line, "type": c.type, "pk": c.pk, "fk": c.fk}
+            {"name": c.name, "line": c.line, "type": c.shown_type(), "pk": c.pk, "fk": c.fk}
             for c in t.columns.values()
             if not c.virtual
         ]
         end = t.end_line or max([t.line + 1] + [c["line"] for c in cols])
         table = {"name": t.name, "aliases": t.aliases, "file": t.path, "line": t.line, "endLine": end, "columns": cols}
-        if t.comments:
-            table["doc"] = "\n".join(t.comments)
+        if t.doc():
+            table["doc"] = t.doc()
         tables.append(table)
 
     def end(e) -> dict:
@@ -76,7 +77,30 @@ def _symbols(schema: Schema) -> dict:
     return {"tables": tables, "rels": rels}
 
 
+def _check_each(args) -> int:
+    """Check each PATH on its own, as ``build --site`` builds them; folders with no schema files are skipped."""
+    from .project import discover
+
+    if args.format == "json" or args.stdin_file:
+        print("error: --each can't be used with --format json or --stdin-file", file=sys.stderr)
+        return 2
+    code, checked = 0, 0
+    for p in args.paths:
+        if Path(p).is_dir() and not discover([Path(p)]):
+            print(f"Skipped {p}: no .txt or .dbml schema files", file=sys.stderr)
+            continue
+        print(f"{p}:")
+        code = max(code, cmd_check(argparse.Namespace(**{**vars(args), "paths": [p], "each": False})))
+        checked += 1
+    if not checked:
+        print("error: no schema files found in: " + ", ".join(args.paths), file=sys.stderr)
+        return 2
+    return code
+
+
 def cmd_check(args) -> int:
+    if args.each:
+        return _check_each(args)
     project = _open(args)
     overrides = None
     if args.stdin_file:
@@ -143,7 +167,52 @@ def _built_page(args) -> tuple[str, dict] | int:
     return render_page(model, pos, _title(project)), model
 
 
+def _build_site(args) -> int:
+    """One page per PATH plus an index linking them, in the folder ``args.site``."""
+    from urllib.parse import quote
+
+    from .html import render_index
+    from .project import discover
+
+    if args.layout:
+        print("error: --layout can't be used with --site: each PATH uses its own layout file", file=sys.stderr)
+        return 2
+    pages, seen, failed = [], {}, 0
+    for p in map(Path, args.paths):
+        if p.is_dir() and not discover([p]):
+            print(f"Skipped {p}: no .txt or .dbml schema files", file=sys.stderr)
+            continue
+        name = _name(p)
+        if name.lower() in seen or name.lower() == "index":
+            other = seen.get(name.lower(), "the index page")
+            print(f"error: {p} and {other} would both be written to {name}.html", file=sys.stderr)
+            return 2
+        seen[name.lower()] = p
+        res = _built_page(argparse.Namespace(**{**vars(args), "paths": [str(p)]}))
+        if isinstance(res, int):
+            failed = max(failed, res)
+            continue
+        pages.append((name, *res))
+    if failed:  # nothing is written, so a failed build never leaves a half-updated site
+        return failed
+    if not pages:
+        print("error: no schema files found in: " + ", ".join(args.paths), file=sys.stderr)
+        return 2
+    out = Path(args.site)
+    out.mkdir(parents=True, exist_ok=True)
+    index = []
+    for name, page, model in pages:
+        (out / f"{name}.html").write_text(page, encoding="utf-8")
+        index.append((name, quote(f"{name}.html"), len(model["tables"]), len(model["rels"])))
+        print(f"Wrote {out / f'{name}.html'} ({len(model['tables'])} tables, {len(model['rels'])} relationships)")
+    (out / "index.html").write_text(render_index(index), encoding="utf-8")
+    print(f"Wrote {out / 'index.html'} ({len(pages)} diagrams)")
+    return 0
+
+
 def cmd_build(args) -> int:
+    if args.site:
+        return _build_site(args)
     res = _built_page(args)
     if isinstance(res, int):
         return res
@@ -337,13 +406,13 @@ def cmd_convert(args) -> int:
         root = next((p for p in inputs if p.is_dir() and f.resolve().is_relative_to(p.resolve())), None)
         rel = f.resolve().relative_to(root.resolve()).as_posix() if root else f.name
         sources.append((str(f), rel, f.read_text(encoding="utf-8")))
-    schema = compile_sources([(d, t) for d, _, t in sources])
+    schema = compile_sources([(d, t) for d, _, t in sources], dbt=False)
     if not _print_diagnostics(schema, False):
         return 1
     res = convert.convert(sources, args.to, split_groups=to_dir)
     for w in res.warnings:
         print(w.format(), file=sys.stderr)
-    again = compile_sources(list(res.files.items()))
+    again = compile_sources(list(res.files.items()), dbt=False)
     if again.errors or convert.canonical(again, res.lost) != convert.canonical(schema, res.lost):
         for d in again.errors:
             print(d.format(), file=sys.stderr)
@@ -575,11 +644,20 @@ def build_parser() -> argparse.ArgumentParser:
         metavar="PATH",
         help="read this schema file's text from stdin instead of disk (an editor's unsaved buffer)",
     )
+    sp.add_argument(
+        "--each",
+        action="store_true",
+        help="check each PATH as its own schema, as build --site does (folders with no schema files are skipped)",
+    )
     sp.set_defaults(func=cmd_check)
 
     sp = sub.add_parser("build", help="write an interactive HTML diagram")
     paths(sp)
-    sp.add_argument("-o", "--output", default="diagram.html")
+    out = sp.add_mutually_exclusive_group()
+    out.add_argument("-o", "--output", default="diagram.html")
+    out.add_argument(
+        "--site", metavar="DIR", help="one page per PATH (named after it) plus an index.html linking them, in DIR"
+    )
     selectors(sp)
     sp.add_argument(
         "--diff-base", metavar="GIT_REF", help="colour tables/columns changed since this git ref (e.g. origin/main)"

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/convert.py

@@ -38,7 +38,7 @@ class ConvertResult:
 
 def convert(sources: list[tuple[str, str, str]], to: str, split_groups: bool) -> ConvertResult:
     """``sources``: ``(display path, output-relative path, text)``. Files already in ``to`` are copied as is."""
-    schema, parses = compile_files([(d, t) for d, _, t in sources])
+    schema, parses = compile_files([(d, t) for d, _, t in sources], dbt=False)
     res = ConvertResult()
     for t in schema.tables.values():
         if t.id in schema.layout:
@@ -88,7 +88,10 @@ def canonical(schema: Schema, lost: set[tuple]) -> dict:
                 if ("col", t.name, c.name, k) in lost:
                     del col[k]
             cols[c.name] = col
-        table = {"cols": cols, "doc": _doc(t.comments), "aliases": t.aliases, "tags": t.tags, "dbt_tags": t.dbt_tags}
+        table = {
+            "cols": cols, "doc": _doc(t.comments), "aliases": t.aliases, "tags": t.tags, "dbt_tags": t.dbt_tags,
+            "dbt_skip": t.dbt_skip,
+        }  # fmt: skip
         for k in list(table):
             if ("table", t.name, k) in lost:
                 del table[k]
@@ -153,7 +156,9 @@ class _DbmlWriter:
                 self.warn(line, f"settings: [settings] {k.capitalize()} has no DBML equivalent; dropped")
         for tag, (color, _) in sorted(fp.colors.items()):
             lines.append(f"// @color {tag} {color}")
-        if fp.colors:
+        if fp.dbt:  # shortcut: kept as written, so it breaks if -o puts the file in another folder
+            lines.append(f"// @dbt: {fp.dbt[0]}")
+        if fp.colors or fp.dbt:
             lines.append("")
         for t in (t for t in self.schema.tables.values() if t.path == fp.path):
             lines += self.table(t)
@@ -170,6 +175,8 @@ class _DbmlWriter:
             out.append(f"// @tag: {', '.join(t.tags)}")
         if t.dbt_tags:
             out.append(f"// @dbt-tags: {', '.join(t.dbt_tags)}")
+        if t.dbt_skip:
+            out.append("// @dbt: none")
         head = f"Table {_dbml_name(t.name)}"
         if t.aliases:
             head += f" as {_dbml_name(t.aliases[0])}"
@@ -313,6 +320,8 @@ class _QuickdbdWriter:
             header.append(f"# enum {name}: {', '.join(values)}")
         for tag, (color, _) in sorted(fp.colors.items()):
             header.append(f"# @color {tag} {color}")
+        if fp.dbt:  # shortcut: kept as written, so it breaks if -o puts the file in another folder
+            header.append(f"# @dbt: {fp.dbt[0]}")
         if not self.split:
             for group, line in fp.groups.items():
                 self.warn(line, f"TableGroup {group}: QuickDBD groups tables by file (use -o DIR to split); dropped")
@@ -347,6 +356,8 @@ class _QuickdbdWriter:
             out.append(f"# @tag: {', '.join(t.tags)}")
         if t.dbt_tags:
             out.append(f"# @dbt-tags: {', '.join(t.dbt_tags)}")
+        if t.dbt_skip:
+            out.append("# @dbt: none")
         out.append(_qdbd_name(t.name) + (f" as {', '.join(t.aliases)}" if t.aliases else ""))
         out.append("-")
         for c in t.columns.values():

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/dbml.py

@@ -16,7 +16,18 @@ import unicodedata
 from dataclasses import dataclass, field
 
 from .model import Column, Diagnostic, Table
-from .parser import COLOR_RE, HEX_RE, TAG_NAME_RE, TAG_RE, FileParse, PendingRel, encode, split_tags
+from .parser import (
+    COLOR_RE,
+    DBT_RE,
+    HEX_RE,
+    TAG_NAME_RE,
+    TAG_RE,
+    FileParse,
+    PendingRel,
+    encode,
+    is_dbt_none,
+    split_tags,
+)
 
 # Relationship operator -> (cardinality of the left end, of the right end). A `?` makes its own side optional.
 _BASE = {"-": ("one", "one"), "<>": ("many", "many"), ">": ("many", "one"), "<": ("one", "many")}
@@ -1287,7 +1298,9 @@ class _Interpreter:
         line = t.line - 1
         while line in self.comments and line not in self.code_lines:
             m = TAG_RE.match(self.comments[line])
-            if m:
+            if is_dbt_none(self.comments[line]):
+                t.dbt_skip = True
+            elif m:
                 target = t.dbt_tags if m.group(1).lower().startswith("dbt") else t.tags
                 target[:0] = [x for x in split_tags(m.group(2)) if x not in target]
             line -= 1
@@ -1306,10 +1319,17 @@ class _Interpreter:
 
     def colors(self) -> None:
         first_code = min(self.code_lines, default=0)
+
+        def directive(text: str) -> bool:
+            return bool(COLOR_RE.match(text) or (DBT_RE.match(text) and not is_dbt_none(text)))
+
         self.fp.general_comments = [
-            text for line, text in sorted(self.comments.items()) if line < first_code and not COLOR_RE.match(text)
+            text for line, text in sorted(self.comments.items()) if line < first_code and not directive(text)
         ]
         for line, text in sorted(self.comments.items()):
+            d = DBT_RE.match(text)
+            if d and not is_dbt_none(text):
+                self.fp.set_dbt(d.group(1), line)
             m = COLOR_RE.match(text)
             if not m:
                 continue

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/dbt.py

@@ -174,10 +174,12 @@ def _comment_lines(text: str) -> list[str]:
     return out
 
 
-def _node_tags(n: dict) -> list[str]:
+def node_tags(n: dict) -> list[str]:
     """dbt tags (node-level and config), lowercased, in order, usable as qdbd tag names."""
     out: list[str] = []
-    raw = list(n.get("tags") or []) + list((n.get("config") or {}).get("tags") or [])
+    raw: list = []
+    for v in (n.get("tags"), (n.get("config") or {}).get("tags")):
+        raw += [v] if isinstance(v, str) else list(v or [])  # yml allows `tags: pii`
     for tag in raw:
         tag = re.sub(r"[^a-z0-9_.-]+", "-", str(tag).lower()).strip("-.")
         if tag and tag not in out:
@@ -243,7 +245,7 @@ class _Importer:
             name = self._base_name(n)
             if names[name] > 1:  # same name in two packages
                 name = f"{n.get('package_name')}.{name}"
-            t = Tbl(n["unique_id"], name, self._group(n), n.get("description") or "", tags=_node_tags(n))
+            t = Tbl(n["unique_id"], name, self._group(n), n.get("description") or "", tags=node_tags(n))
             self._columns(t, n)
             self.tables[t.uid] = t
             self.by_name.setdefault(n.get("name", "").lower(), []).append(t.uid)

quickdbd_cli-0.6.0/src/quickdbd_cli/dbtyml.py

@@ -0,0 +1,275 @@
+"""`# @dbt: PATH`: descriptions, tags and types from a dbt project's yml, for hand-written schema files.
+
+No manifest.json is needed (dbt Cloud projects have none locally). qdbd reads ``dbt_project.yml`` and the yml, md
+and sql files under its resource paths (the root project only; ``dbt_packages/`` and ``target/`` are skipped).
+
+- Tables match models, seeds, snapshots and sources (``source_name.table_name``) by name, ignoring case;
+  columns match by name inside the matched table. ``# @dbt: none`` above a table opts it out.
+- The dbt description is shown before the schema file's own comment; ``{{ doc('x') }}`` is filled from
+  ``{% docs x %}`` blocks. Model and column tags rank below ``# @tag:`` and name prefixes. ``data_type`` fills
+  columns that have no type.
+- Drift is reported as warnings (``--strict`` fails on them): a table dbt doesn't have, a column only one side has,
+  a type that differs (by base type: ``varchar(50)`` and ``string`` agree).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+from .dbt import node_tags, norm_type
+from .model import DbtInfo, Schema
+
+SKIP_DIRS = {"dbt_packages", "dbt_modules", "target", "logs", "node_modules"}
+WATCHED = (".yml", ".yaml", ".md", ".sql", ".py", ".csv")
+_DOC_REF = re.compile(r"""\{\{\s*doc\(\s*(?:['"][^'"]*['"]\s*,\s*)?['"]([^'"]+)['"]\s*\)\s*\}\}""")
+_DOCS_BLOCK = re.compile(r"\{%-?\s*docs\s+([\w.-]+)\s*-?%\}(.*?)\{%-?\s*enddocs\s*-?%\}", re.S)
+_SNAPSHOT = re.compile(r"\{%-?\s*snapshot\s+(\w+)\s*-?%\}")
+
+# Base types that mean the same thing across warehouses; sizes and precision are ignored.
+_FAMILIES = {
+    "int": "int integer int2 int4 int8 int16 int32 int64 smallint bigint tinyint byteint mediumint hugeint long short",
+    "decimal": "decimal numeric number dec bignumeric",
+    "float": "float float4 float8 float32 float64 double double_precision real",
+    "string": "string text varchar char character nvarchar nchar bpchar varchar2 nvarchar2 citext clob",
+    "bool": "bool boolean",
+    "timestamp": "timestamp timestamptz datetime datetime2 smalldatetime datetimeoffset timestamp_ntz timestamp_ltz "
+    "timestamp_tz",
+    "time": "time timetz",
+    "json": "json jsonb variant object super",
+    "binary": "binary varbinary bytes bytea blob",
+    "uuid": "uuid uniqueidentifier",
+    "array": "array list",
+}
+_BASE = {name: family for family, names in _FAMILIES.items() for name in names.split()}
+
+
+class DbtError(Exception):
+    pass
+
+
+@dataclass
+class DbtColumn:
+    name: str
+    description: str
+    data_type: str | None
+    tags: list[str]
+    line: int
+
+
+@dataclass
+class DbtModel:
+    name: str  # model / seed / snapshot name, or `source_name.table_name`
+    description: str
+    tags: list[str]
+    columns: dict[str, DbtColumn]  # by lowercased name
+    path: str  # yml file, relative to the project root
+    line: int
+
+
+@dataclass
+class DbtProject:
+    root: Path
+    models: dict[str, DbtModel] = field(default_factory=dict)  # by lowercased name
+    known: set[str] = field(default_factory=set)  # lowercased names that exist as files, documented or not
+    ambiguous: set[str] = field(default_factory=set)
+    docs: dict[str, str] = field(default_factory=dict)  # `{% docs name %}` blocks
+
+
+class _Loader(getattr(yaml, "CSafeLoader", yaml.SafeLoader)):  # type: ignore[misc]
+    """Records each mapping's line as `__line__`, so drift messages can point into the yml."""
+
+    def construct_mapping(self, node, deep=False):
+        m = super().construct_mapping(node, deep=deep)
+        m["__line__"] = node.start_mark.line + 1
+        return m
+
+
+# (path, parser) -> (mtime_ns, parsed): keeps `qdbd serve` recompiles cheap
+_cache: dict[tuple[Path, Callable], tuple[int, object]] = {}
+
+
+def _cached(path: Path, parse: Callable[[str], object]) -> object:
+    mtime = path.stat().st_mtime_ns
+    hit = _cache.get((path, parse))
+    if hit is None or hit[0] != mtime:
+        hit = _cache[(path, parse)] = (mtime, parse(path.read_text(encoding="utf-8")))
+    return hit[1]
+
+
+def _load_yaml(text: str) -> object:
+    return yaml.load(text, Loader=_Loader)  # a SafeLoader: no arbitrary Python objects
+
+
+def _yaml(path: Path, root: Path) -> object:
+    try:
+        return _cached(path, _load_yaml)
+    except yaml.YAMLError as e:
+        mark = getattr(e, "problem_mark", None)
+        where = f"{path.relative_to(root).as_posix()}:{mark.line + 1}" if mark else path.relative_to(root).as_posix()
+        raise DbtError(f"{where}: invalid YAML ({getattr(e, 'problem', None) or e})") from None
+
+
+def find_root(path: Path) -> Path:
+    """The dbt project a `# @dbt:` path names: its folder, or its dbt_project.yml."""
+    root = path.parent if path.name == "dbt_project.yml" else path
+    if not (root / "dbt_project.yml").is_file():
+        raise DbtError(f"no dbt_project.yml in {root}")
+    return root
+
+
+def _paths(root: Path) -> tuple[dict[str, list[Path]], list[Path]]:
+    """(resource kind -> folders, docs folders), from dbt_project.yml with dbt's defaults."""
+    cfg = _yaml(root / "dbt_project.yml", root)
+    cfg = cfg if isinstance(cfg, dict) else {}
+
+    def get(key: str, old: str | None = None, default: str | None = None) -> list[Path]:
+        v = cfg.get(key) or (cfg.get(old) if old else None) or ([default] if default else [])
+        out = []
+        for p in [v] if isinstance(v, str) else v:
+            # dbt resolves these inside the project; anything else (`/`, `../..`) would walk the whole disk
+            if not (root / str(p)).resolve().is_relative_to(root.resolve()):
+                raise DbtError(f"dbt_project.yml: {key} '{p}' is outside the project")
+            out.append(root / str(p))
+        return out
+
+    kinds = {
+        "model": get("model-paths", "source-paths", "models"),
+        "seed": get("seed-paths", "data-paths", "seeds"),
+        "snapshot": get("snapshot-paths", None, "snapshots"),
+    }
+    doc_dirs = get("docs-paths")
+    if not doc_dirs:  # dbt's default: every resource path
+        doc_dirs = [d for ds in kinds.values() for d in ds]
+        doc_dirs += get("analysis-paths", None, "analyses") + get("macro-paths", None, "macros")
+    return kinds, doc_dirs
+
+
+def _walk(dirs: list[Path], suffixes: tuple[str, ...]) -> list[Path]:
+    out: dict[Path, None] = {}
+    for d in dirs:
+        for dirpath, dirnames, filenames in os.walk(d):
+            dirnames[:] = sorted(n for n in dirnames if not n.startswith(".") and n not in SKIP_DIRS)
+            for n in sorted(filenames):
+                if n.lower().endswith(suffixes):
+                    out[Path(dirpath, n)] = None
+    return list(out)
+
+
+def watched_files(root: Path) -> list[Path]:
+    """Every file `load_project` reads (or may read once it exists), for `qdbd serve`'s change polling."""
+    kinds, doc_dirs = _paths(root)
+    return [root / "dbt_project.yml", *_walk([d for ds in kinds.values() for d in ds] + doc_dirs, WATCHED)]
+
+
+def _text(v: object) -> str:
+    return str(v).strip() if v is not None else ""
+
+
+def _column_name(v: object) -> str:
+    return _text(v).strip('"`')
+
+
+def load_project(root: Path) -> DbtProject:
+    kinds, doc_dirs = _paths(root)
+    project = DbtProject(root)
+    for p in _walk(kinds["model"], (".sql", ".py")):
+        project.known.add(p.stem.lower())
+    for p in _walk(kinds["seed"], (".csv",)):
+        project.known.add(p.stem.lower())
+    for p in _walk(kinds["snapshot"], (".sql",)):
+        project.known.update(m.lower() for m in _cached(p, _SNAPSHOT.findall))  # type: ignore[union-attr]
+    for p in _walk(doc_dirs, (".md",)):
+        for name, body in _cached(p, _DOCS_BLOCK.findall):  # type: ignore[union-attr]
+            project.docs[name] = body.strip()
+    for p in _walk([d for ds in kinds.values() for d in ds], (".yml", ".yaml")):
+        data = _yaml(p, root)
+        if not isinstance(data, dict):
+            continue
+        rel = p.relative_to(root).as_posix()
+        for key in ("models", "seeds", "snapshots"):
+            for n in data.get(key) or []:
+                if isinstance(n, dict) and n.get("name"):
+                    _add(project, _text(n["name"]), n, rel)
+        for src in data.get("sources") or []:
+            if not isinstance(src, dict) or not src.get("name"):
+                continue
+            for n in src.get("tables") or []:
+                if isinstance(n, dict) and n.get("name"):
+                    _add(project, f"{_text(src['name'])}.{_text(n['name'])}", n, rel)
+    return project
+
+
+def _add(project: DbtProject, name: str, n: dict, rel: str) -> None:
+    key = name.lower()
+    if key in project.models:
+        project.ambiguous.add(key)
+        return
+    cols: dict[str, DbtColumn] = {}
+    for c in n.get("columns") or []:
+        if isinstance(c, dict) and _column_name(c.get("name")):
+            cname = _column_name(c["name"])
+            dtype = _text(c.get("data_type")) or None
+            cols[cname.lower()] = DbtColumn(cname, _text(c.get("description")), dtype, node_tags(c), c["__line__"])
+    project.models[key] = DbtModel(name, _text(n.get("description")), node_tags(n), cols, rel, n["__line__"])
+    project.known.add(key)
+
+
+def base_type(t: str) -> str:
+    """`varchar(255)` -> `string`, `INTEGER` -> `int`: what two types must share to count as the same."""
+    s = re.split(r"[(<\[]", (norm_type(t) or "").lower())[0].strip("_ ")
+    return _BASE.get(s, s)
+
+
+Report = Callable[[str, int, str, str], None]  # (path, line, message, severity)
+
+
+def apply(schema: Schema, project: DbtProject, report: Report) -> None:
+    """Attach dbt descriptions, tags and types to the matching tables and columns, and report drift."""
+
+    def doc(text: str, path: str, line: int) -> str:
+        def sub(m: re.Match) -> str:
+            if m.group(1) in project.docs:
+                return project.docs[m.group(1)]
+            report(path, line, f"dbt: no docs block named '{m.group(1)}'", "warning")
+            return m.group(0)
+
+        return _DOC_REF.sub(sub, text)
+
+    for t in schema.tables.values():
+        key = t.name.lower()
+        if t.dbt_skip or key in project.ambiguous:
+            continue
+        m = project.models.get(key)
+        if m is None:
+            if key not in project.known:
+                report(
+                    t.path,
+                    t.line,
+                    f"dbt: no model, seed, snapshot or source named '{t.name}' "
+                    "(put '# @dbt: none' above the table if it isn't one)",
+                    "warning",
+                )
+            continue
+        t.dbt = DbtInfo(doc(m.description, t.path, t.line), list(m.tags))
+        cols = {c.name.lower(): c for c in t.columns.values() if not c.virtual}
+        for ckey, dc in m.columns.items():
+            c = cols.get(ckey)
+            if c is None:
+                msg = f"dbt: column '{dc.name}' ({m.path}:{dc.line}) is missing from '{t.name}'"
+                report(t.path, t.line, msg, "warning")
+                continue
+            c.dbt = DbtInfo(doc(dc.description, t.path, c.line), list(dc.tags), norm_type(dc.data_type))
+            if dc.data_type and c.type and not c.type_default and base_type(c.type) != base_type(dc.data_type):
+                msg = f"dbt: type '{c.type}' differs from data_type '{dc.data_type}' ({m.path}:{dc.line})"
+                report(t.path, c.line, msg, "warning")
+        if m.columns:
+            for ckey, c in cols.items():
+                if ckey not in m.columns:
+                    msg = f"dbt: column '{c.name}' is not in the yml for '{m.name}' ({m.path}:{m.line})"
+                    report(t.path, c.line, msg, "warning")

{quickdbd_cli-0.4.3 → quickdbd_cli-0.6.0}/src/quickdbd_cli/diff.py

@@ -175,7 +175,7 @@ def sources_at_ref(ref: str, inputs: list[Path]) -> list[tuple[str, str]]:
 
 
 def schema_at_ref(ref: str, inputs: list[Path]) -> Schema:
-    return compile_sources(sources_at_ref(ref, inputs))
+    return compile_sources(sources_at_ref(ref, inputs), dbt=False)
 
 
 # --------------------------------------------------------------------------- output