dbdocs 1.7.0__tar.gz → 1.8.0__tar.gz

{dbdocs-1.7.0 → dbdocs-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: dbdocs
-Version: 1.7.0
+Version: 1.8.0
 Summary: Alternative dbt docs site: Catalog + ERD + column-level lineage
 Project-URL: Homepage, https://github.com/datnguye/dbt-docs
 Project-URL: Repository, https://github.com/datnguye/dbt-docs
@@ -14,12 +14,14 @@ Classifier: Topic :: Software Development :: Documentation
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Python: >=3.10
-Requires-Dist: artifact-parser[dbt]>=1.0.0
+Requires-Dist: artifact-parser[dbt]>=1.1.0
 Requires-Dist: click>=8.1
-Requires-Dist: dbterd>=1.28
+Requires-Dist: dbterd>=1.35.1
 Requires-Dist: pydantic>=2.13.4
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: sqlglot>=25
+Provides-Extra: info-schema
+Requires-Dist: duckdb>=1.1; extra == 'info-schema'
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -71,12 +73,20 @@ Requires Python 3.10+.
 
 ```bash
 dbt docs generate     # writes target/manifest.json + target/catalog.json
-dbdocs generate       # builds ./site/ with index.html + dbdocs-data.json.gz
+dbdocs generate       # builds target/site/ with index.html + dbdocs-data.json.gz
 dbdocs serve          # static http server on http://127.0.0.1:8000
 ```
 
 The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
 
+### On dbt 2.0
+
+dbt 2.0 stops writing `catalog.json` and can write a typed parquet **information schema** instead — on dbt Fusion with `--static-analysis strict`, complete with column types and dbt's own column-level lineage. dbdocs builds from either; to read the info schema, `pip install "dbdocs[info-schema]"` and set `info_schema: true` in `dbdocs.yml`:
+
+![dbt Fusion writes the info schema, dbdocs builds and serves the site from it](docs/assets/img/dbt-2-info-schema.gif)
+
+The [dbt 2.0 & Info Schema](https://dbdocs.datnguye.me/latest/nav/guide/dbt-2-info-schema.html) guide walks through it.
+
 Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
 
 ## Contributing

{dbdocs-1.7.0 → dbdocs-1.8.0}/README.md

@@ -47,12 +47,20 @@ Requires Python 3.10+.
 
 ```bash
 dbt docs generate     # writes target/manifest.json + target/catalog.json
-dbdocs generate       # builds ./site/ with index.html + dbdocs-data.json.gz
+dbdocs generate       # builds target/site/ with index.html + dbdocs-data.json.gz
 dbdocs serve          # static http server on http://127.0.0.1:8000
 ```
 
 The site must be served over HTTP (not opened as a local file) because it fetches the data payload at load time. `dbdocs serve` handles that locally; any static host works for deployment.
 
+### On dbt 2.0
+
+dbt 2.0 stops writing `catalog.json` and can write a typed parquet **information schema** instead — on dbt Fusion with `--static-analysis strict`, complete with column types and dbt's own column-level lineage. dbdocs builds from either; to read the info schema, `pip install "dbdocs[info-schema]"` and set `info_schema: true` in `dbdocs.yml`:
+
+![dbt Fusion writes the info schema, dbdocs builds and serves the site from it](docs/assets/img/dbt-2-info-schema.gif)
+
+The [dbt 2.0 & Info Schema](https://dbdocs.datnguye.me/latest/nav/guide/dbt-2-info-schema.html) guide walks through it.
+
 Full walkthrough, configuration, and architecture live in the **[documentation](https://dbdocs.datnguye.me/)**.
 
 ## Contributing

{dbdocs-1.7.0 → dbdocs-1.8.0}/dbdocs/cli/main.py

@@ -9,11 +9,31 @@ from dbdocs.core.config import DbDocsConfig
 from dbdocs.core.exceptions import DbDocsError
 from dbdocs.core.log import logger
 from dbdocs.site import deploy as deploy_module
-from dbdocs.site.builder import ReportBuilder
+from dbdocs.sources.factory import builder_for
 
 __version__ = importlib.metadata.version("dbdocs")
 
 
+def info_schema_options(command):
+    command = click.option(
+        "--info-schema-dir",
+        default=None,
+        help="Directory of the info-schema parquet files (default: <target_dir>/info_schema/v1).",
+    )(command)
+    return click.option(
+        "--info-schema/--no-info-schema",
+        default=None,
+        help="Read dbt 2.0's parquet information schema as the base artifact source.",
+    )(command)
+
+
+def _apply_info_schema(config: DbDocsConfig, info_schema, info_schema_dir) -> None:
+    if info_schema is not None:
+        config.info_schema = info_schema
+    if info_schema_dir is not None:
+        config.info_schema_dir = info_schema_dir
+
+
 # dbdocs
 @click.group(
     context_settings={"help_option_names": ["-h", "--help"]},
@@ -37,6 +57,7 @@ def dbdocs(ctx, config_path):
 
 
 @dbdocs.command(name="generate")
+@info_schema_options
 @click.option("-o", "--output-dir", default=None, help="Where to write the site (default: config).")
 @click.option(
     "--dialect", default=None, help="SQL dialect for column lineage (default: adapter_type)."
@@ -49,14 +70,15 @@ def dbdocs(ctx, config_path):
     ),
 )
 @click.pass_obj
-def generate(config: DbDocsConfig, output_dir, dialect, run_results):
+def generate(config: DbDocsConfig, output_dir, dialect, run_results, info_schema, info_schema_dir):
     """Build the site from dbt artifacts (served over HTTP; data loaded externally)."""
     if dialect is not None:
         config.dialect = dialect
     if run_results is not None:
         config.run_results = run_results
+    _apply_info_schema(config, info_schema, info_schema_dir)
     try:
-        ReportBuilder(config).generate(output_dir=output_dir)
+        builder_for(config).generate(output_dir=output_dir)
     except DbDocsError as exc:
         raise click.ClickException(str(exc)) from exc
     # The builder already logs "Generated site at <path> (N nodes, M edges)".
@@ -84,15 +106,24 @@ def serve(config: DbDocsConfig, port):
     "--delete", "delete", is_flag=True, default=False, help="Delete this version instead."
 )
 @click.option("--push/--no-push", default=False, help="Publish to the gh-pages branch.")
+@info_schema_options
 @click.pass_obj
-def deploy(config: DbDocsConfig, version, alias, title, delete, push):
+def deploy(config: DbDocsConfig, version, alias, title, delete, push, info_schema, info_schema_dir):
     """Generate a versioned build and update the version index (or --delete one)."""
+    _apply_info_schema(config, info_schema, info_schema_dir)
     try:
         if delete:
             deploy_module.delete(config, version=version, push=push)
             click.echo(f"Deleted version {version}")
             return
-        out = deploy_module.deploy(config, version=version, alias=alias, push=push, title=title)
+        out = deploy_module.deploy(
+            config,
+            version=version,
+            alias=alias,
+            push=push,
+            title=title,
+            builder=builder_for(config),
+        )
     except DbDocsError as exc:
         raise click.ClickException(str(exc)) from exc
     click.echo(f"Deployed version {version} into {out}")

{dbdocs-1.7.0 → dbdocs-1.8.0}/dbdocs/core/artifacts.py

@@ -19,6 +19,8 @@ from typing import Any
 
 from dbterd.helpers import file
 
+from dbdocs.core.log import logger
+
 #: Bucket label used when a node/source has no database or schema set.
 UNKNOWN = "_unknown"
 
@@ -54,10 +56,23 @@ def artifact_version(target_path: str, artifact: str) -> "int | None":
 
 
 def load_artifacts(target_path: str) -> "tuple[Any, Any]":
-    """Return the dbterd-parsed ``(manifest, catalog)`` for a dbt target dir."""
+    """Return the dbterd-parsed ``(manifest, catalog)`` for a dbt target dir.
+
+    ``catalog.json`` is optional (dbt 2.0's ``docs generate`` does not write it):
+    when absent, ``catalog`` is ``None`` and every consumer falls back to the
+    manifest — columns lose their warehouse type and column-level lineage can't
+    resolve ``SELECT *`` / unqualified columns.
+    """
     manifest = file.read_manifest(
         path=target_path, version=artifact_version(target_path, "manifest")
     )
+    if not (Path(target_path) / "catalog.json").is_file():
+        logger.warning(
+            "No catalog.json in %s — building from the manifest only "
+            "(no warehouse column types; reduced column-level lineage).",
+            target_path,
+        )
+        return manifest, None
     catalog = file.read_catalog(path=target_path, version=artifact_version(target_path, "catalog"))
     return manifest, catalog
 

{dbdocs-1.7.0 → dbdocs-1.8.0}/dbdocs/core/config.py

@@ -81,6 +81,13 @@ class DbDocsConfig:
     #: dotted import path whose ``register_rule`` calls add custom rules). Entry
     #: points under ``dbdocs.health_rules`` are also discovered automatically.
     health: dict = field(default_factory=dict)
+    #: Opt in to reading dbt 2.0's parquet information schema as the base artifact
+    #: source (``manifest.json`` / ``catalog.json`` / ``run_results.json`` in
+    #: ``target_dir`` then only enrich it). Needs ``dbdocs[info-schema]``.
+    info_schema: bool = False
+    #: Directory holding the information-schema parquet files; ``None`` ⇒
+    #: ``<target_dir>/info_schema/v1``.
+    info_schema_dir: "str | None" = None
     #: The pinned About link shows by default; set false to hide it.
     show_about: bool = True
     #: Links rendered on the About page as external CTA buttons.  Each entry is a
@@ -129,6 +136,8 @@ class DbDocsConfig:
         "favicon",
         "run_results",
         "health",
+        "info_schema",
+        "info_schema_dir",
     )
 
     def render_context(self) -> dict:
@@ -164,3 +173,10 @@ class DbDocsConfig:
         A relative path that would escape the cwd raises ``DbDocsConfigError``.
         """
         return str(_resolve_within_cwd(self.output_dir, "output_dir"))
+
+    @property
+    def info_schema_path(self) -> str:
+        """Directory of the information-schema parquet files (see ``info_schema_dir``)."""
+        if self.info_schema_dir:
+            return str(_resolve_within_cwd(self.info_schema_dir, "info_schema_dir"))
+        return str(Path(self.target_path) / "info_schema" / "v1")

{dbdocs-1.7.0 → dbdocs-1.8.0}/dbdocs/extract/column_lineage.py

@@ -61,10 +61,11 @@ def _leaf_columns(root: Node, relation_to_node: dict) -> list:
     """Collect distinct upstream ``{node, column}`` leaves of a lineage tree.
 
     A leaf is a node whose source is a real ``Table`` (not a CTE/subquery scope)
-    that maps back to a dbt node. The root itself is skipped.
+    that maps back to a dbt node. The root itself is skipped. Leaves are sorted
+    by ``(node, column)``: the tree's walk order follows sqlglot's internal set
+    iteration, which varies with ``PYTHONHASHSEED``.
     """
-    seen = set()
-    upstream = []
+    leaves = set()
     for node in root.walk():
         if node is root:
             continue
@@ -72,15 +73,9 @@ def _leaf_columns(root: Node, relation_to_node: dict) -> list:
         if not isinstance(source, exp.Table):
             continue
         mapped = _map_table(source, relation_to_node)
-        if mapped is None:
-            continue
-        column = node_name(node.name)
-        key = (mapped, column)
-        if key in seen:
-            continue
-        seen.add(key)
-        upstream.append({"node": mapped, "column": column})
-    return upstream
+        if mapped is not None:
+            leaves.add((mapped, node_name(node.name)))
+    return [{"node": mapped, "column": column} for mapped, column in sorted(leaves)]
 
 
 def _extract_model_columns(

{dbdocs-1.7.0 → dbdocs-1.8.0}/dbdocs/site/deploy.py

@@ -67,8 +67,14 @@ def deploy(
     alias: "str | None" = None,
     push: bool = False,
     title: "str | None" = None,
+    builder: "ReportBuilder | None" = None,
 ) -> str:
-    """Generate ``version`` into the output root and update the version index."""
+    """Generate ``version`` into the output root and update the version index.
+
+    ``builder`` renders the version; the CLI passes ``sources.factory.builder_for``'s
+    pick so an opt-in artifact source applies, keeping ``site/`` free of ``sources/``.
+    It defaults to a plain ``ReportBuilder``.
+    """
     _validate_segment(version, "version")
     if alias is not None:
         _validate_segment(alias, "alias")
@@ -77,7 +83,7 @@ def deploy(
     if version_dir.exists():
         rmtree(version_dir)
 
-    ReportBuilder(config).generate(output_dir=str(version_dir), versioned=True)
+    (builder or ReportBuilder(config)).generate(output_dir=str(version_dir), versioned=True)
 
     versions = _upsert_version(_read_versions(output_root), version, title or version, alias)
     (output_root / VERSIONS_FILE).write_text(json.dumps(versions, indent=2), encoding="utf-8")

dbdocs-1.8.0/dbdocs/sources/factory.py

@@ -0,0 +1,24 @@
+"""Pick the report builder for the configured artifact source."""
+
+import importlib
+
+from dbdocs.core.config import DbDocsConfig
+from dbdocs.core.exceptions import DbDocsConfigError
+from dbdocs.site.builder import ReportBuilder
+
+INFO_SCHEMA_BUILDER_MODULE = "dbdocs.sources.info_schema.builder"
+
+
+def builder_for(config: DbDocsConfig) -> ReportBuilder:
+    """``ReportBuilder`` for the JSON artifacts; the info-schema builder when opted in."""
+    if not config.info_schema:
+        return ReportBuilder(config)
+    try:
+        module = importlib.import_module(INFO_SCHEMA_BUILDER_MODULE)
+    except ModuleNotFoundError as exc:
+        if exc.name != "duckdb":
+            raise
+        raise DbDocsConfigError(
+            "info_schema needs DuckDB: install it with `pip install 'dbdocs[info-schema]'`."
+        ) from exc
+    return module.InfoSchemaReportBuilder(config)

dbdocs-1.8.0/dbdocs/sources/info_schema/builder.py

@@ -0,0 +1,65 @@
+"""Report builder that reads dbt 2.0's parquet information schema."""
+
+import json
+import tempfile
+from dataclasses import replace
+from pathlib import Path
+
+from dbdocs.core.config import DbDocsConfig
+from dbdocs.core.log import logger
+from dbdocs.site.builder import ReportBuilder
+from dbdocs.sources.info_schema import synth
+from dbdocs.sources.info_schema.reader import InfoSchemaReader
+
+ARTIFACT_SOURCE = "info_schema"
+
+
+def _read_json(path: Path) -> "dict | None":
+    """An optional JSON enrichment artifact; ``None`` when absent or unusable (logged)."""
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return None
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.warning("Ignoring %s as info_schema enrichment: %s", path, exc)
+        return None
+
+
+class InfoSchemaReportBuilder(ReportBuilder):
+    """Stages synthesized artifacts, then runs the unchanged ``build_data`` on them.
+
+    Parquet is the base; ``manifest.json`` / ``catalog.json`` / ``run_results.json``
+    in ``target_dir`` enrich it when present. ``dbt.column_lineage`` rows replace
+    the sqlglot column lineage; an empty table keeps the sqlglot result.
+    """
+
+    def __init__(self, config: DbDocsConfig) -> None:
+        super().__init__(config)
+        self.reader = InfoSchemaReader(config.info_schema_path)
+
+    def build_data(self) -> dict:
+        target = Path(self.config.target_path)
+        run_results = _read_json(Path(self._resolve_run_results_path()))
+        with tempfile.TemporaryDirectory() as staging:
+            self._stage(Path(staging), target, run_results)
+            staged = ReportBuilder(replace(self.config, target_dir=staging, run_results=None))
+            data = staged.build_data()
+        data["metadata"]["artifact_source"] = ARTIFACT_SOURCE
+        lineage = synth.column_lineage(self.reader)
+        if lineage:
+            data["columnLineage"] = lineage
+        else:
+            logger.info("dbt.column_lineage is empty — keeping sqlglot column lineage.")
+        return data
+
+    def _stage(self, staging: Path, target: Path, run_results: "dict | None") -> None:
+        artifacts = {
+            "manifest": synth.build_manifest(self.reader, _read_json(target / "manifest.json")),
+            "catalog": synth.build_catalog(self.reader, _read_json(target / "catalog.json")),
+            "run_results": synth.build_run_results(self.reader, run_results),
+        }
+        for name, artifact in artifacts.items():
+            if artifact is not None:
+                (staging / f"{name}.json").write_text(
+                    json.dumps(artifact, default=str), encoding="utf-8"
+                )

dbdocs-1.8.0/dbdocs/sources/info_schema/reader.py

@@ -0,0 +1,45 @@
+"""Read dbt 2.0's parquet information schema (``dbt.*`` / ``dbt_rt.*`` tables)."""
+
+from pathlib import Path
+
+import duckdb
+
+from dbdocs.core.exceptions import DbDocsConfigError
+
+SUPPORTED_SCHEMA_VERSION = 1
+
+
+class InfoSchemaReader:
+    """Rows of the information-schema parquet files under ``directory``.
+
+    A table with no parquet file reads as ``[]``. Construction fails unless the
+    directory exists and ``dbt.project.schema_version`` is the supported one.
+    """
+
+    def __init__(self, directory: "str | Path") -> None:
+        self.directory = Path(directory)
+        self._tables: dict[str, list[dict]] = {}
+        if not self.directory.is_dir():
+            raise DbDocsConfigError(f"info_schema directory not found: {self.directory}")
+        project = self.table("dbt.project")
+        version = project[0].get("schema_version") if project else None
+        if version != SUPPORTED_SCHEMA_VERSION:
+            raise DbDocsConfigError(
+                f"Unsupported info_schema schema_version {version!r} in {self.directory} "
+                f"(supported: {SUPPORTED_SCHEMA_VERSION})."
+            )
+
+    def table(self, name: str) -> "list[dict]":
+        """The table's rows, read once per reader and served from memory after."""
+        if name not in self._tables:
+            self._tables[name] = self._read(self.directory / f"{name}.parquet")
+        return self._tables[name]
+
+    @staticmethod
+    def _read(path: Path) -> "list[dict]":
+        if not path.is_file():
+            return []
+        with duckdb.connect() as connection:
+            cursor = connection.execute("SELECT * FROM read_parquet(?)", [str(path)])
+            columns = [col[0] for col in cursor.description]
+            return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()]

dbdocs-1.8.0/dbdocs/sources/info_schema/synth.py

@@ -0,0 +1,604 @@
+"""Synthesize standard dbt artifacts (manifest v12 / catalog v1 / run_results v6)
+from the parquet information schema.
+
+Merge rule, applied per record and field: the parquet set decides which records
+exist, a non-empty parquet value wins, and the matching JSON artifact record (when
+one was supplied) fills whatever parquet left empty. Column types are the one
+exception: warehouse ``catalog.json`` type, then ``data_type_actual``, then the
+declared type, then ``data_type_inferred``.
+"""
+
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Any
+
+from artifact_parser.dbt_info_schema.base import decode_json_column
+
+from dbdocs.sources.info_schema.reader import InfoSchemaReader
+
+MANIFEST_SCHEMA = "https://schemas.getdbt.com/dbt/manifest/v12.json"
+CATALOG_SCHEMA = "https://schemas.getdbt.com/dbt/catalog/v1.json"
+RUN_RESULTS_SCHEMA = "https://schemas.getdbt.com/dbt/run-results/v6.json"
+
+_NODE_TABLES = (
+    ("dbt.models", "model"),
+    ("dbt.seeds", "seed"),
+    ("dbt.snapshots", "snapshot"),
+    ("dbt.analyses", "analysis"),
+    ("dbt.data_tests", "test"),
+)
+_DEPENDENT_SECTIONS = (
+    "nodes",
+    "sources",
+    "unit_tests",
+    "metrics",
+    "semantic_models",
+    "saved_queries",
+    "exposures",
+)
+_ALL_SECTIONS = _DEPENDENT_SECTIONS + ("macros", "docs", "groups")
+_COLUMN_DEFAULTS = {"description": "", "meta": {}, "tags": [], "constraints": []}
+_READ_ONLY_EVOLUTIONS = frozenset({"scan"})
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _empty(value: Any) -> bool:
+    return value is None or (isinstance(value, (str, list, dict)) and not value)
+
+
+def _compact(record: dict) -> dict:
+    return {k: v for k, v in record.items() if not _empty(v)}
+
+
+def _fill(base: Any, extra: Any) -> Any:
+    """``base`` where it has a value, else ``extra``; dicts merge key by key."""
+    if _empty(base):
+        return extra
+    if isinstance(base, dict) and isinstance(extra, dict):
+        merged = dict(base)
+        for key, value in extra.items():
+            merged[key] = _fill(base.get(key), value)
+        return merged
+    return base
+
+
+def _path(original_file_path: "str | None") -> "str | None":
+    return original_file_path.split("/", 1)[-1] if original_file_path else None
+
+
+def _common(row: dict, kind: str) -> dict:
+    properties = row.get("properties_yml_file_path")
+    package = row.get("package_name") or row["unique_id"].split(".")[1]
+    return {
+        "unique_id": row["unique_id"],
+        "name": row.get("name"),
+        "resource_type": row.get("resource_type") or kind,
+        "package_name": package,
+        "original_file_path": row.get("original_file_path"),
+        "path": _path(row.get("original_file_path")),
+        "fqn": row.get("fqn"),
+        "description": row.get("description"),
+        "tags": row.get("tags"),
+        "meta": decode_json_column(row.get("meta")),
+        "patch_path": f"{package}://{properties}" if properties else None,
+    }
+
+
+def _config(row: dict) -> dict:
+    config = decode_json_column(row.get("config"))
+    config = dict(config) if isinstance(config, dict) else {}
+    config["materialized"] = row.get("materialized")
+    return _compact(config)
+
+
+def _physical(row: dict, kind: str) -> dict:
+    compiled_code = row.get("compiled_code")
+    record = _common(row, kind)
+    record.update(
+        database=row.get("database_name"),
+        schema=row.get("schema_name"),
+        alias=row.get("alias") or row.get("name"),
+        relation_name=row.get("relation_name"),
+        raw_code=row.get("raw_code"),
+        compiled_code=compiled_code,
+        compiled=True if compiled_code else None,
+        compiled_path=row.get("compiled_path"),
+        language=row.get("node_language"),
+        config=_config(row),
+        access=row.get("access"),
+        group=row.get("group"),
+        contract={"enforced": bool(row.get("contract_enforced"))},
+        version=decode_json_column(row.get("version")),
+        latest_version=decode_json_column(row.get("latest_version")),
+        docs={"show": row["docs_show"]} if row.get("docs_show") is not None else None,
+    )
+    if kind == "test":
+        record.update(
+            test_metadata=_compact(
+                {
+                    "name": row.get("test_name"),
+                    "namespace": row.get("test_definition_package"),
+                    "kwargs": decode_json_column(row.get("arguments")),
+                }
+            ),
+            column_name=row.get("column_name"),
+            attached_node=row.get("node_unique_id"),
+        )
+    return record
+
+
+def _source(row: dict, kind: str) -> dict:
+    record = _physical(row, kind)
+    record.update(
+        source_name=row.get("source_name"),
+        source_description=row.get("source_description"),
+        loader=row.get("loader"),
+        identifier=row.get("identifier"),
+        loaded_at_field=row.get("loaded_at_field"),
+        freshness=decode_json_column(row.get("freshness")),
+    )
+    return record
+
+
+def _unit_test(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record.update(
+        model=row.get("model"),
+        given=decode_json_column(row.get("given")),
+        expect=decode_json_column(row.get("expect")),
+        overrides=decode_json_column(row.get("overrides")),
+        config=decode_json_column(row.get("config")),
+    )
+    return record
+
+
+def _metric(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record.update(
+        label=row.get("label"),
+        type=row.get("metric_type"),
+        type_params=decode_json_column(row.get("type_params")),
+        filter=decode_json_column(row.get("metric_filter")),
+        time_granularity=row.get("time_granularity"),
+        group=row.get("group"),
+        config=decode_json_column(row.get("config")),
+        created_at=row.get("created_at"),
+    )
+    return record
+
+
+def _semantic_model(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record.update(
+        label=row.get("label"),
+        model=row.get("model"),
+        node_relation=decode_json_column(row.get("node_relation")),
+        primary_entity=row.get("primary_entity"),
+        defaults=decode_json_column(row.get("defaults")),
+        group=row.get("group"),
+        config=decode_json_column(row.get("config")),
+        created_at=row.get("created_at"),
+    )
+    return record
+
+
+def _saved_query(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record.update(
+        label=row.get("label"),
+        query_params=decode_json_column(row.get("query_params")),
+        exports=decode_json_column(row.get("exports")),
+        group=row.get("group"),
+        config=decode_json_column(row.get("config")),
+        created_at=row.get("created_at"),
+    )
+    return record
+
+
+def _exposure(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record.update(
+        label=row.get("label"),
+        type=row.get("exposure_type"),
+        url=row.get("url"),
+        maturity=row.get("maturity"),
+        owner=_compact({"name": row.get("owner_name"), "email": row.get("owner_email")}),
+        config=decode_json_column(row.get("config")),
+        created_at=row.get("created_at"),
+    )
+    return record
+
+
+def _group(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record["owner"] = _compact({"name": row.get("owner_name"), "email": row.get("owner_email")})
+    return record
+
+
+def _macro(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    arguments = decode_json_column(row.get("arguments"))
+    record.update(
+        macro_sql=row.get("macro_sql"),
+        depends_on={"macros": row.get("depends_on_macros")},
+        arguments=arguments if isinstance(arguments, list) else None,
+    )
+    return record
+
+
+def _doc(row: dict, kind: str) -> dict:
+    record = _common(row, kind)
+    record["block_contents"] = row.get("content")
+    return record
+
+
+_COLLECTIONS = (
+    ("dbt.sources", "sources", "source", _source),
+    ("dbt.unit_tests", "unit_tests", "unit_test", _unit_test),
+    ("dbt.metrics", "metrics", "metric", _metric),
+    ("dbt.semantic_models", "semantic_models", "semantic_model", _semantic_model),
+    ("dbt.saved_queries", "saved_queries", "saved_query", _saved_query),
+    ("dbt.exposures", "exposures", "exposure", _exposure),
+    ("dbt.groups", "groups", "group", _group),
+    ("dbt.macros", "macros", "macro", _macro),
+    ("dbt.docs_blocks", "docs", "doc", _doc),
+)
+
+_DEFAULTS = {
+    "nodes": {
+        "description": "",
+        "tags": [],
+        "meta": {},
+        "columns": {},
+        "config": {},
+        "checksum": {"name": "none", "checksum": ""},
+    },
+    "sources": {
+        "description": "",
+        "tags": [],
+        "meta": {},
+        "columns": {},
+        "source_description": "",
+        "loader": "",
+    },
+    "unit_tests": {"description": "", "tags": [], "meta": {}, "checksum": ""},
+    "metrics": {"description": "", "label": "", "type_params": {}},
+    "semantic_models": {"description": "", "node_relation": {}},
+    "saved_queries": {"description": "", "exports": [], "query_params": {}},
+    "exposures": {"description": "", "owner": {}},
+    "groups": {"owner": {}},
+    "macros": {"description": "", "macro_sql": ""},
+    "docs": {"block_contents": ""},
+}
+
+
+def _group_by(rows: "list[dict]", key: str) -> "dict[str, list[dict]]":
+    grouped: dict[str, list[dict]] = defaultdict(list)
+    for row in rows:
+        grouped[row[key]].append(row)
+    return grouped
+
+
+def _columns(reader: InfoSchemaReader) -> "dict[str, dict]":
+    columns: dict[str, dict] = defaultdict(dict)
+    for row in sorted(reader.table("dbt.node_columns"), key=lambda r: r.get("column_index") or 0):
+        columns[row["node_unique_id"]][row["column_name"]] = _compact(
+            {
+                "name": row["column_name"],
+                "description": row.get("description"),
+                "data_type": row.get("data_type_declared"),
+                "tags": row.get("tags"),
+                "meta": decode_json_column(row.get("meta")),
+                "granularity": row.get("granularity"),
+            }
+        )
+    return columns
+
+
+def _semantic_parts(reader: InfoSchemaReader) -> "dict[str, dict]":
+    parts: dict[str, dict] = defaultdict(lambda: {"entities": [], "measures": [], "dimensions": []})
+    for row in reader.table("dbt.semantic_entities"):
+        parts[row["unique_id"]]["entities"].append(
+            {
+                "name": row["name"],
+                "type": row.get("entity_type"),
+                "description": row.get("description"),
+                "label": row.get("label"),
+                "role": row.get("entity_role"),
+                "expr": row.get("expr"),
+            }
+        )
+    for row in reader.table("dbt.semantic_measures"):
+        parts[row["unique_id"]]["measures"].append(
+            {
+                "name": row["name"],
+                "agg": row.get("agg"),
+                "description": row.get("description"),
+                "label": row.get("label"),
+                "expr": row.get("expr"),
+                "create_metric": row.get("create_metric"),
+                "agg_time_dimension": row.get("agg_time_dimension"),
+                "agg_params": decode_json_column(row.get("agg_params")),
+                "non_additive_dimension": decode_json_column(row.get("non_additive_dimension")),
+            }
+        )
+    for row in reader.table("dbt.semantic_dimensions"):
+        granularity = row.get("time_granularity")
+        parts[row["unique_id"]]["dimensions"].append(
+            _compact(
+                {
+                    "name": row["name"],
+                    "type": row.get("dimension_type"),
+                    "description": row.get("description"),
+                    "label": row.get("label"),
+                    "expr": row.get("expr"),
+                    "is_partition": row.get("is_partition"),
+                    "type_params": {
+                        "time_granularity": granularity,
+                        "validity_params": decode_json_column(row.get("validity_params")),
+                    }
+                    if granularity
+                    else None,
+                }
+            )
+        )
+    return parts
+
+
+def _edges(reader: InfoSchemaReader) -> "tuple[dict, dict]":
+    parents: dict[str, list[str]] = defaultdict(list)
+    children: dict[str, list[str]] = defaultdict(list)
+    for row in reader.table("dbt.edges"):
+        parent, child = row["parent_unique_id"], row["child_unique_id"]
+        if parent.startswith("macro.") or child.startswith("macro."):
+            continue
+        parents[child].append(parent)
+        children[parent].append(child)
+    return parents, children
+
+
+def _known_json_records(json_manifest: "dict | None") -> "dict[str, dict]":
+    known: dict[str, dict] = {}
+    for section in _ALL_SECTIONS:
+        known.update((json_manifest or {}).get(section) or {})
+    return known
+
+
+def _finish(record: dict, section: str, json_record: "dict | None") -> dict:
+    merged = _fill(_compact(record), json_record or {})
+    for key, value in _DEFAULTS[section].items():
+        merged.setdefault(key, value)
+    merged.setdefault("path", "")
+    merged.setdefault("original_file_path", "")
+    if section in _DEPENDENT_SECTIONS:
+        merged.setdefault("fqn", [merged.get("package_name"), merged.get("name")])
+    if section == "unit_tests" and isinstance(merged["checksum"], dict):
+        merged["checksum"] = merged["checksum"].get("checksum") or ""
+    for column in (merged.get("columns") or {}).values():
+        for key, value in _COLUMN_DEFAULTS.items():
+            column.setdefault(key, value)
+    return merged
+
+
+def build_manifest(reader: InfoSchemaReader, json_manifest: "dict | None" = None) -> dict:
+    known = _known_json_records(json_manifest)
+    parents, children = _edges(reader)
+    columns = _columns(reader)
+    semantic_parts = _semantic_parts(reader)
+    manifest: dict = {section: {} for section in _ALL_SECTIONS}
+
+    jobs = [
+        (section, kind, builder, row)
+        for table, section, kind, builder in _COLLECTIONS
+        for row in reader.table(table)
+    ]
+    jobs += [
+        ("nodes", kind, _physical, row)
+        for table, kind in _NODE_TABLES
+        for row in reader.table(table)
+    ]
+    for section, kind, builder, row in jobs:
+        record = builder(row, kind)
+        unique_id = record["unique_id"]
+        if section in _DEPENDENT_SECTIONS:
+            record["depends_on"] = {"nodes": parents.get(unique_id)}
+        if unique_id in columns:
+            record["columns"] = columns[unique_id]
+        record.update(semantic_parts.get(unique_id, {}) if section == "semantic_models" else {})
+        manifest[section][unique_id] = _finish(record, section, known.get(unique_id))
+
+    project = (reader.table("dbt.project") or [{}])[0]
+    metadata = _fill(
+        _compact(
+            {
+                "dbt_version": project.get("dbt_version"),
+                "project_name": project.get("project_name"),
+                "adapter_type": project.get("adapter_type"),
+                "quoting": decode_json_column(project.get("quoting")),
+            }
+        ),
+        (json_manifest or {}).get("metadata") or {},
+    )
+    metadata["dbt_schema_version"] = MANIFEST_SCHEMA
+    metadata.setdefault("generated_at", _now())
+    manifest["metadata"] = metadata
+
+    ids = [uid for section in _DEPENDENT_SECTIONS for uid in manifest[section]]
+    manifest["parent_map"] = _fill(
+        {uid: parents.get(uid, []) for uid in ids}, (json_manifest or {}).get("parent_map")
+    )
+    manifest["child_map"] = _fill(
+        {uid: children.get(uid, []) for uid in ids}, (json_manifest or {}).get("child_map")
+    )
+    group_map: dict[str, list[str]] = defaultdict(list)
+    for uid, node in manifest["nodes"].items():
+        if node.get("group"):
+            group_map[node["group"]].append(uid)
+    manifest["group_map"] = dict(group_map)
+    manifest["selectors"] = (json_manifest or {}).get("selectors") or {}
+    manifest["disabled"] = (json_manifest or {}).get("disabled") or {}
+    return manifest
+
+
+def _catalog_type(json_column: "dict | None", row: dict) -> "str | None":
+    return (
+        (json_column or {}).get("type")
+        or row.get("data_type_actual")
+        or row.get("data_type_declared")
+        or row.get("data_type_inferred")
+    )
+
+
+def build_catalog(reader: InfoSchemaReader, json_catalog: "dict | None" = None) -> dict:
+    json_catalog = json_catalog or {}
+    json_records = {**(json_catalog.get("sources") or {}), **(json_catalog.get("nodes") or {})}
+    manifest_rows = {
+        row["unique_id"]: row
+        for table in ("dbt.models", "dbt.seeds", "dbt.snapshots", "dbt.sources")
+        for row in reader.table(table)
+    }
+    by_node = _group_by(reader.table("dbt.node_columns"), "node_unique_id")
+    catalog: dict = {"nodes": {}, "sources": {}}
+    for unique_id, rows in by_node.items():
+        owner = manifest_rows.get(unique_id)
+        if owner is None:
+            continue
+        json_record = json_records.get(unique_id) or {}
+        json_columns = {str(k).lower(): v for k, v in (json_record.get("columns") or {}).items()}
+        columns = {}
+        for index, row in enumerate(sorted(rows, key=lambda r: r.get("column_index") or 0)):
+            name = row["column_name"]
+            columns[name] = _compact(
+                {
+                    "name": name,
+                    "index": index,
+                    "type": _catalog_type(json_columns.get(name.lower()), row),
+                    "comment": None,
+                }
+            )
+            columns[name].setdefault("type", "")
+        record = {
+            "unique_id": unique_id,
+            "metadata": {
+                "type": owner.get("materialized") or "table",
+                "schema": owner.get("schema_name"),
+                "name": owner.get("identifier") or owner.get("name"),
+                "database": owner.get("database_name"),
+            },
+            "columns": columns,
+            "stats": {},
+        }
+        section = "sources" if unique_id.startswith("source.") else "nodes"
+        catalog[section][unique_id] = _fill(record, json_record)
+    for section in ("nodes", "sources"):
+        for unique_id, json_record in (json_catalog.get(section) or {}).items():
+            catalog[section].setdefault(unique_id, json_record)
+    project = (reader.table("dbt.project") or [{}])[0]
+    catalog.update(
+        metadata={
+            "dbt_schema_version": CATALOG_SCHEMA,
+            "dbt_version": project.get("dbt_version") or "",
+            "generated_at": _now(),
+            "env": {},
+        },
+        errors=None,
+    )
+    return catalog
+
+
+def _run_result(row: dict, json_result: "dict | None") -> dict:
+    result = _fill(
+        _compact(
+            {
+                "unique_id": row["unique_id"],
+                "status": row.get("status"),
+                "timing": decode_json_column(row.get("timing")),
+                "thread_id": row.get("thread_id"),
+                "execution_time": row.get("execution_time"),
+                "adapter_response": decode_json_column(row.get("adapter_response")),
+                "message": row.get("message"),
+                "failures": row.get("failures"),
+                "compiled": row.get("compiled"),
+                "relation_name": row.get("relation_name"),
+            }
+        ),
+        json_result or {},
+    )
+    defaults = {
+        "timing": [],
+        "thread_id": "",
+        "execution_time": 0.0,
+        "adapter_response": {},
+        "message": "",
+        "failures": None,
+        "compiled": False,
+        "compiled_code": None,
+        "relation_name": None,
+    }
+    for key, value in defaults.items():
+        result.setdefault(key, value)
+    return result
+
+
+def _latest_invocation(reader: InfoSchemaReader, rows: "list[dict]") -> "dict | None":
+    with_tests = {
+        r["invocation_id"] for r in rows if r["unique_id"].split(".")[0] in ("test", "unit_test")
+    }
+    candidates = [i for i in reader.table("dbt_rt.invocations") if i["invocation_id"] in with_tests]
+    if not candidates:
+        return None
+    return max(
+        candidates, key=lambda i: i.get("generated_at") or datetime.min.replace(tzinfo=timezone.utc)
+    )
+
+
+def build_run_results(
+    reader: InfoSchemaReader, json_run_results: "dict | None" = None
+) -> "dict | None":
+    rows = reader.table("dbt_rt.run_results")
+    invocation = _latest_invocation(reader, rows)
+    if invocation is None:
+        return json_run_results
+    json_by_id = {r.get("unique_id"): r for r in (json_run_results or {}).get("results") or []}
+    results = [
+        _run_result(row, json_by_id.get(row["unique_id"]))
+        for row in rows
+        if row["invocation_id"] == invocation["invocation_id"]
+    ]
+    return {
+        "metadata": {
+            "dbt_schema_version": RUN_RESULTS_SCHEMA,
+            "dbt_version": invocation.get("dbt_version") or "",
+            "generated_at": invocation["generated_at"].isoformat()
+            if invocation.get("generated_at")
+            else _now(),
+            "invocation_id": invocation["invocation_id"],
+            "env": {},
+        },
+        "results": results,
+        "elapsed_time": invocation.get("elapsed_time") or 0.0,
+        "args": {},
+    }
+
+
+def column_lineage(reader: InfoSchemaReader) -> dict:
+    """``dbt.column_lineage`` value lineage as ``{"<node>.<column>": [{node, column}, ...]}``.
+
+    Rows whose ``evolution`` is in ``_READ_ONLY_EVOLUTIONS`` (``scan``: a join, filter
+    or grouping key that is read but does not feed the column's value) are dropped,
+    so the result means what the sqlglot lineage means. Refs are deduplicated and
+    sorted by ``(node, column)``.
+    """
+    refs: dict[str, set] = defaultdict(set)
+    for row in reader.table("dbt.column_lineage"):
+        if row.get("evolution") in _READ_ONLY_EVOLUTIONS:
+            continue
+        key = f"{row['child_node_unique_id']}.{row['child_column_name']}"
+        refs[key].add((row["parent_node_unique_id"], row["parent_column_name"]))
+    return {
+        key: [{"node": node, "column": column} for node, column in sorted(pairs)]
+        for key, pairs in refs.items()
+    }

{dbdocs-1.7.0 → dbdocs-1.8.0}/pyproject.toml

@@ -15,14 +15,17 @@ classifiers = [
   "Topic :: Software Development :: Quality Assurance",
 ]
 dependencies = [
-  "artifact-parser[dbt]>=1.0.0",
+  "artifact-parser[dbt]>=1.1.0",
   "click>=8.1",
-  "dbterd>=1.28",
+  "dbterd>=1.35.1",
   "pydantic>=2.13.4",
   "pyyaml>=6.0",
   "sqlglot>=25",
 ]
 
+[project.optional-dependencies]
+info-schema = ["duckdb>=1.1"]
+
 [project.urls]
 Homepage = "https://github.com/datnguye/dbt-docs"
 Repository = "https://github.com/datnguye/dbt-docs"
@@ -58,6 +61,7 @@ dev = [
   "dbt-postgres>=1.5",
   "pdoc>=14.4.0",
   "jsonschema>=4.20",
+  "duckdb>=1.1",
 ]
 docs = [
   "mkdocs>=1.6,<2.0",