dbdocs 0.0.0__py3-none-any.whl

dbdocs/__main__.py

@@ -0,0 +1,3 @@
+from dbdocs import main
+
+main.main()

dbdocs/cli/main.py

@@ -0,0 +1,86 @@
+import functools
+import importlib.metadata
+import socketserver
+from http.server import SimpleHTTPRequestHandler
+
+import click
+
+from dbdocs.core.config import DbDocsConfig
+from dbdocs.core.exceptions import DbDocsError
+from dbdocs.core.log import logger
+from dbdocs.site import deploy as deploy_module
+from dbdocs.site.builder import ReportBuilder
+
+__version__ = importlib.metadata.version("dbdocs")
+
+
+# dbdocs
+@click.group(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    invoke_without_command=True,
+    no_args_is_help=True,
+    epilog="Specify one of these sub-commands and you can find more help from there.",
+)
+@click.version_option(__version__)
+@click.option("-c", "--config", "config_path", default=None, help="Path to dbdocs.yml.")
+@click.pass_context
+def dbdocs(ctx, config_path):
+    """Alternative dbt docs site: dbt docs + ERD + column-level lineage."""
+    logger.info("Run with dbdocs==%s", __version__)
+    try:
+        ctx.obj = DbDocsConfig.load(config_path)
+    except DbDocsError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+
+@dbdocs.command(name="generate")
+@click.option("-o", "--output-dir", default=None, help="Where to write the site (default: config).")
+@click.option(
+    "--dialect", default=None, help="SQL dialect for column lineage (default: adapter_type)."
+)
+@click.pass_obj
+def generate(config: DbDocsConfig, output_dir, dialect):
+    """Build the self-contained site from dbt artifacts."""
+    if dialect is not None:
+        config.dialect = dialect
+    try:
+        out = ReportBuilder(config).generate(output_dir=output_dir)
+    except DbDocsError as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Generated site into {out}")
+
+
+@dbdocs.command(name="serve")
+@click.option("-p", "--port", default=8000, show_default=True, help="Port to serve on.")
+@click.pass_obj
+def serve(config: DbDocsConfig, port):
+    """Serve the generated site locally (static http server)."""
+    handler = functools.partial(SimpleHTTPRequestHandler, directory=config.output_path)
+    click.echo(f"Serving {config.output_path} at http://127.0.0.1:{port} (Ctrl-C to stop)")
+    socketserver.ThreadingTCPServer.allow_reuse_address = True
+    with socketserver.ThreadingTCPServer(("127.0.0.1", port), handler) as httpd:
+        httpd.serve_forever()
+
+
+@dbdocs.command(name="deploy")
+@click.option("--version", "version", required=True, help="Version label to deploy (e.g. 1.2).")
+@click.option("--alias", default=None, help="Moving alias for this version (e.g. latest).")
+@click.option(
+    "--title", default=None, help="Display title for this version (default: the version)."
+)
+@click.option(
+    "--delete", "delete", is_flag=True, default=False, help="Delete this version instead."
+)
+@click.option("--push/--no-push", default=False, help="Publish to the gh-pages branch.")
+@click.pass_obj
+def deploy(config: DbDocsConfig, version, alias, title, delete, push):
+    """Generate a versioned build and update the version index (or --delete one)."""
+    try:
+        if delete:
+            deploy_module.delete(config, version=version, push=push)
+            click.echo(f"Deleted version {version}")
+            return
+        out = deploy_module.deploy(config, version=version, alias=alias, push=push, title=title)
+    except DbDocsError as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Deployed version {version} into {out}")

dbdocs/core/artifacts.py

@@ -0,0 +1,82 @@
+"""Loading dbt artifacts (manifest/catalog) via the dbterd parser.
+
+dbterd parses ``manifest.json`` / ``catalog.json`` into ``dbt_artifacts_parser``
+Pydantic models. Two cross-cutting gotchas live here so the rest of dbdocs never
+has to think about them:
+
+* **Schema field aliasing.** ``dbt_artifacts_parser`` aliases the ``schema``
+  field to ``schema_`` to avoid clobbering Pydantic's ``BaseModel.schema()`` —
+  so ``node.schema`` is a *bound method*, not the value. Always read
+  ``node.schema_``; :func:`db_schema` centralizes that.
+* **Schema-version relaxation.** Passing the detected schema version to
+  ``read_manifest``/``read_catalog`` makes dbterd apply its relaxation policies,
+  keeping parsing robust across dbt versions (including dbt Core 2.0).
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from dbterd.helpers import file
+
+#: Bucket label used when a node/source has no database or schema set.
+UNKNOWN = "_unknown"
+
+#: unique_id prefixes surfaced as catalog nodes (tests/macros/etc. excluded).
+NODE_PREFIXES = ("model.", "seed.", "snapshot.")
+
+
+def artifact_version(target_path: str, artifact: str) -> "int | None":
+    """Resolve a dbt artifact's schema version int from its ``dbt_schema_version``.
+
+    Returns ``None`` (auto-detect, strict) if the version can't be determined —
+    e.g. the file is missing or not valid JSON.
+    """
+    artifact_path = Path(target_path) / f"{artifact}.json"
+    try:
+        metadata = json.loads(artifact_path.read_text(encoding="utf-8")).get("metadata", {})
+    except (OSError, json.JSONDecodeError):
+        return None
+    extracted = file.extract_artifact_version_from_file(metadata.get("dbt_schema_version", ""))
+    return int(extracted) if extracted else None
+
+
+def load_artifacts(target_path: str) -> "tuple[Any, Any]":
+    """Return the dbterd-parsed ``(manifest, catalog)`` for a dbt target dir."""
+    manifest = file.read_manifest(
+        path=target_path, version=artifact_version(target_path, "manifest")
+    )
+    catalog = file.read_catalog(path=target_path, version=artifact_version(target_path, "catalog"))
+    return manifest, catalog
+
+
+def adapter_type(target_path: str) -> "str | None":
+    """The warehouse adapter (``snowflake``/``bigquery``/…) from manifest metadata.
+
+    Read from the raw JSON rather than the parsed model so it works regardless of
+    how the parser exposes ``metadata``. Used as the default sqlglot dialect for
+    column-level lineage. ``None`` if unreadable.
+    """
+    manifest_path = Path(target_path) / "manifest.json"
+    try:
+        metadata = json.loads(manifest_path.read_text(encoding="utf-8")).get("metadata", {})
+    except (OSError, json.JSONDecodeError):
+        return None
+    return metadata.get("adapter_type")
+
+
+def db_schema(entity: Any) -> "tuple[str, str]":
+    """The ``(database, schema)`` an entity lands in, with safe fallbacks.
+
+    Reads ``schema_`` (the Pydantic alias — ``schema`` is a bound method) and
+    falls back to :data:`UNKNOWN` when either part is missing, so grouping never
+    produces a ``None`` bucket.
+    """
+    database = getattr(entity, "database", None) or UNKNOWN
+    schema = getattr(entity, "schema_", None) or UNKNOWN
+    return str(database), str(schema)
+
+
+def node_name(unique_id: str) -> str:
+    """The dbt node's short name — the last dotted segment of its unique_id."""
+    return unique_id.split(".")[-1]

dbdocs/core/config.py

@@ -0,0 +1,117 @@
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+import yaml
+
+from dbdocs.core.exceptions import DbDocsConfigError
+
+DEFAULT_CONFIG_FILENAME = "dbdocs.yml"
+
+
+@dataclass
+class DbDocsConfig:
+    """Site configuration for a dbdocs build.
+
+    Loaded from a ``dbdocs.yml`` in the working directory; every field has a
+    default so the file is optional. ``version`` is intentionally absent — it is
+    a ``deploy`` CLI argument, not site config.
+
+    ``target_dir`` is where the dbt artifacts are read from; ``output_dir`` is
+    where the generated self-contained site is written.
+    """
+
+    site_name: str = "dbt docs"
+    site_url: str = "https://github.com/datnguye/dbt-docs"
+    site_author: str = "Dat Nguyen"
+    site_description: str = "Alternative dbt documentation site"
+    repo_name: str = "datnguye/dbt-docs"
+    repo_url: str = "https://github.com/datnguye/dbt-docs"
+    project_name: str = "dbt docs"
+    #: The footer's Buy-me-a-coffee badge shows by default; set false to hide it.
+    show_buy_me_a_coffee: bool = True
+    #: Project README rendered on the overview (relative to the working dir). Set
+    #: empty to omit the README section. Missing file ⇒ section simply absent.
+    readme: str = "README.md"
+    target_dir: str = "target"
+    #: Where the generated site is written. Nested under the dbt ``target/`` by
+    #: default so docs sit alongside the artifacts they're built from.
+    output_dir: str = "target/site"
+    #: SQL dialect for column-lineage parsing; ``None`` ⇒ derive from the
+    #: artifact's ``adapter_type`` (e.g. snowflake, bigquery, postgres).
+    dialect: "str | None" = None
+    #: Alias the SPA's version switcher treats as the default landing version.
+    default_version: str = "latest"
+    #: dbterd ERD options (``algo``, ``entity_name_format``, ``select``,
+    #: ``resource_type``, …) passed straight to ``DbtErd``. Configured here so the
+    #: ERD shape lives in ``dbdocs.yml`` rather than a separate ``.dbterd.yml``.
+    dbterd: dict = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, path: "str | Path | None" = None) -> "DbDocsConfig":
+        """Load config from ``path`` (or ``./dbdocs.yml``); all-defaults if absent."""
+        config_path = Path(path) if path is not None else Path.cwd() / DEFAULT_CONFIG_FILENAME
+        if not config_path.is_file():
+            return cls()
+
+        try:
+            raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
+        except yaml.YAMLError as exc:
+            raise DbDocsConfigError(f"Could not parse {config_path}: {exc}") from exc
+
+        if raw is None:
+            return cls()
+        if not isinstance(raw, dict):
+            raise DbDocsConfigError(
+                f"{config_path} must contain a mapping, got {type(raw).__name__}"
+            )
+
+        known = {f.name for f in fields(cls)}
+        unknown = set(raw) - known
+        if unknown:
+            raise DbDocsConfigError(
+                f"Unknown keys in {config_path}: {', '.join(sorted(unknown))}. "
+                f"Allowed keys: {', '.join(sorted(known))}."
+            )
+        return cls(**raw)
+
+    #: Build-control fields that are not part of the site's display metadata.
+    _NON_METADATA_FIELDS = (
+        "target_dir",
+        "output_dir",
+        "dialect",
+        "default_version",
+        "dbterd",
+        "readme",
+    )
+
+    def render_context(self) -> dict:
+        """The site display metadata injected into the SPA's ``metadata`` block.
+
+        Excludes build-control fields (where artifacts are read, where the site
+        is written, the lineage dialect override) that aren't site metadata.
+        """
+        context = asdict(self)
+        for field_name in self._NON_METADATA_FIELDS:
+            context.pop(field_name, None)
+        return context
+
+    @property
+    def target_path(self) -> str:
+        """Absolute path to the dbt target/ dir where the artifacts live.
+
+        A relative ``target_dir`` is resolved against the current working
+        directory **at access time** — this is intentional and must stay aligned
+        with dbterd's ``DbtErd``, which also reads artifacts from ``./target``
+        relative to the cwd. An absolute ``target_dir`` is returned unchanged.
+        """
+        return str(Path.cwd() / self.target_dir)
+
+    @property
+    def output_path(self) -> str:
+        """Absolute path to the dir the generated site is written into.
+
+        Resolved against the cwd at access time, mirroring ``target_path`` — a
+        relative ``output_dir`` follows the working directory, an absolute one is
+        returned unchanged.
+        """
+        return str(Path.cwd() / self.output_dir)

dbdocs/core/exceptions.py

@@ -0,0 +1,24 @@
+"""dbdocs exception types.
+
+Multiple exception classes may share one file (per the project's Python style).
+"""
+
+
+class DbDocsError(Exception):
+    """Base class for all dbdocs errors."""
+
+
+class DbDocsConfigError(DbDocsError):
+    """Raised when dbdocs.yml is malformed or holds invalid values."""
+
+
+class LineageError(DbDocsError):
+    """Raised when column-level lineage can't be parsed for a model.
+
+    Always caught per-model by the extractor so one unparseable model never
+    fails the whole ``generate`` — the model is skipped and logged instead.
+    """
+
+
+class DeployError(DbDocsError):
+    """Raised when a versioned deploy step (e.g. the git push) fails."""

dbdocs/core/log.py

@@ -0,0 +1,58 @@
+import logging
+from pathlib import Path
+
+#: Where the DEBUG-level file log is streamed (relative to the working dir).
+LOG_FILE = Path("logs") / "dbdocs.log"
+#: Plain (non-ANSI) line format for the file — colour codes don't belong in a file.
+FILE_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+
+
+class LogFormatter(logging.Formatter):
+    grey = "\x1b[38;20m"
+    blue = "\x1b[34;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
+
+    FORMATS = {
+        logging.DEBUG: blue + format + reset,
+        logging.INFO: grey + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red + format + reset,
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
+# Named "dbdocs" (not "dbterd") so our handler/level config doesn't collide with
+# the dbterd library's own logger of the same name.
+logger = logging.getLogger("dbdocs")
+logger.setLevel(logging.DEBUG)
+# Emit only through our own handlers. Without this, records also propagate to the
+# root logger — which dbterd configures via basicConfig — producing duplicate,
+# differently-formatted "INFO:dbdocs:…" lines.
+logger.propagate = False
+
+if len(logger.handlers) == 0:  # pragma: no cover - import-time handler guard
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    ch.setFormatter(LogFormatter())
+    logger.addHandler(ch)
+
+    # Stream everything (DEBUG and up) to logs/dbdocs.log too. Best-effort: if the
+    # logs dir can't be created/written (read-only fs), the console handler still
+    # works and we don't crash on import.
+    try:
+        LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
+        fh = logging.FileHandler(LOG_FILE, encoding="utf-8")
+        fh.setLevel(logging.DEBUG)
+        fh.setFormatter(logging.Formatter(FILE_FORMAT))
+        logger.addHandler(fh)
+    except OSError:
+        pass

dbdocs/extract/_sqlglot_lineage.py

@@ -0,0 +1,267 @@
+"""Column-level lineage engine: trace a SELECT's output column to its sources.
+
+A self-contained lineage builder over sqlglot's optimizer, with case-insensitive
+column resolution and cycle-safe recursion so it copes with dbt-compiled SQL
+(uppercased warehouse identifiers, recursive CTEs) without relying on sqlglot's
+internal, version-unstable lineage API.
+"""
+
+from __future__ import annotations
+
+import logging
+import typing as t
+from dataclasses import dataclass, field
+
+from sqlglot import Schema, exp, maybe_parse
+from sqlglot.errors import SqlglotError
+from sqlglot.optimizer import (
+    Scope,
+    build_scope,
+    find_all_in_scope,
+    normalize_identifiers,
+    qualify,
+)
+from sqlglot.optimizer.scope import ScopeType
+
+if t.TYPE_CHECKING:
+    from sqlglot.dialects.dialect import DialectType
+
+logger = logging.getLogger("sqlglot")
+
+
+@dataclass
+class Node:
+    name: str
+    expression: exp.Expression
+    source: exp.Expression
+    downstream: list[Node] = field(default_factory=list)
+    source_name: str = ""
+    reference_node_name: str = ""
+
+    def walk(self) -> t.Iterator[Node]:
+        yield self
+        for d in self.downstream:
+            yield from d.walk()
+
+
+def lineage(
+    column: str | exp.Column,
+    sql: str | exp.Expression,
+    schema: dict | Schema | None = None,
+    sources: t.Mapping[str, str | exp.Query] | None = None,
+    dialect: DialectType = None,
+    scope: Scope | None = None,
+    trim_selects: bool = True,
+    **kwargs,
+) -> Node:
+    """Build the lineage graph for a column of a SQL query."""
+    expression = maybe_parse(sql, dialect=dialect)
+    column = normalize_identifiers.normalize_identifiers(column, dialect=dialect).name
+
+    if sources:
+        expression = exp.expand(
+            expression,
+            {k: t.cast(exp.Query, maybe_parse(v, dialect=dialect)) for k, v in sources.items()},
+            dialect=dialect,
+        )
+
+    if not scope:
+        expression = qualify.qualify(
+            expression,
+            dialect=dialect,
+            schema=schema,
+            **{
+                "validate_qualify_columns": False,
+                "identify": False,
+                "allow_partial_qualification": True,
+                **kwargs,
+            },
+        )
+        scope = build_scope(expression)
+
+    if not scope:
+        raise SqlglotError("Cannot build lineage, sql must be SELECT")
+
+    select_names_original = {select.alias_or_name for select in scope.expression.selects}
+    select_names_lower = {name.lower(): name for name in select_names_original}
+    # Case-insensitive resolution: dbt/warehouse casing rarely matches exactly.
+    if column not in select_names_original:
+        column_lower = column.lower()
+        if column_lower in select_names_lower:
+            column = select_names_lower[column_lower]
+        else:
+            raise SqlglotError(f"Cannot find column '{column}' in query.")
+
+    return to_node(column, scope, dialect, trim_selects=trim_selects)
+
+
+def to_node(
+    column: str | int,
+    scope: Scope,
+    dialect: DialectType,
+    scope_name: str | None = None,
+    upstream: Node | None = None,
+    source_name: str | None = None,
+    reference_node_name: str | None = None,
+    trim_selects: bool = True,
+    visited: set | None = None,
+) -> Node | None:
+    if visited is None:
+        visited = set()
+
+    key = (column, id(scope))
+    if key in visited:
+        # Already visited this column-scope: stop, or recursive CTEs loop forever.
+        return None
+    visited.add(key)
+
+    select = (
+        scope.expression.selects[column]
+        if isinstance(column, int)
+        else next(
+            (select for select in scope.expression.selects if select.alias_or_name == column),
+            exp.Star() if scope.expression.is_star else scope.expression,
+        )
+    )
+
+    if isinstance(scope.expression, exp.Subquery):
+        for source in scope.subquery_scopes:
+            return to_node(
+                column,
+                scope=source,
+                dialect=dialect,
+                upstream=upstream,
+                source_name=source_name,
+                reference_node_name=reference_node_name,
+                trim_selects=trim_selects,
+                visited=visited,
+            )
+    if isinstance(scope.expression, exp.SetOperation):
+        name = type(scope.expression).__name__.upper()
+        upstream = upstream or Node(name=name, source=scope.expression, expression=select)
+
+        index = (
+            column
+            if isinstance(column, int)
+            else next(
+                (
+                    i
+                    for i, select in enumerate(scope.expression.selects)
+                    if select.alias_or_name == column or select.is_star
+                ),
+                -1,
+            )
+        )
+
+        if index == -1:
+            raise ValueError(f"Could not find {column} in {scope.expression}")
+
+        for s in scope.union_scopes:
+            to_node(
+                index,
+                scope=s,
+                dialect=dialect,
+                upstream=upstream,
+                source_name=source_name,
+                reference_node_name=reference_node_name,
+                trim_selects=trim_selects,
+                visited=visited,
+            )
+        return upstream
+
+    if trim_selects and isinstance(scope.expression, exp.Select):
+        source = exp.Select()
+        source.set("expressions", [select])
+        source.set("from", scope.expression.args.get("from"))
+        source.set("where", scope.expression.args.get("where"))
+        source.set("group", scope.expression.args.get("group"))
+    else:
+        source = scope.expression
+
+    node = Node(
+        name=f"{scope_name}.{column}" if scope_name else str(column),
+        source=source,
+        expression=select,
+        source_name=source_name or "",
+        reference_node_name=reference_node_name or "",
+    )
+
+    if upstream:
+        upstream.downstream.append(node)
+
+    subquery_scopes = {
+        id(subquery_scope.expression): subquery_scope for subquery_scope in scope.subquery_scopes
+    }
+
+    for subquery in find_all_in_scope(select, exp.UNWRAPPED_QUERIES):
+        subquery_scope = subquery_scopes.get(id(subquery))
+        if not subquery_scope:
+            logger.warning("Unknown subquery scope: %s", subquery.sql(dialect=dialect))
+            continue
+
+        for name in subquery.named_selects:
+            to_node(
+                name,
+                scope=subquery_scope,
+                dialect=dialect,
+                upstream=node,
+                trim_selects=trim_selects,
+                visited=visited,
+            )
+
+    if select.is_star:
+        for source in scope.sources.values():
+            if isinstance(source, Scope):
+                source = source.expression
+            node.downstream.append(
+                Node(name=select.sql(comments=False), source=source, expression=source)
+            )
+
+    source_columns = set(find_all_in_scope(select, exp.Column))
+
+    if isinstance(source, exp.UDTF):
+        source_columns |= set(source.find_all(exp.Column))
+        derived_tables = [
+            source.expression.parent
+            for source in scope.sources.values()
+            if isinstance(source, Scope) and source.is_derived_table
+        ]
+    else:
+        derived_tables = scope.derived_tables
+
+    source_names = {
+        dt.alias: dt.comments[0].split()[1]
+        for dt in derived_tables
+        if dt.comments and dt.comments[0].startswith("source: ")
+    }
+
+    for c in source_columns:
+        table = c.table
+        source = scope.sources.get(table)
+
+        if isinstance(source, Scope):
+            reference_node_name = None
+            if source.scope_type == ScopeType.DERIVED_TABLE and table not in source_names:
+                reference_node_name = table
+            elif source.scope_type == ScopeType.CTE:
+                selected_node, _ = scope.selected_sources.get(table, (None, None))
+                reference_node_name = selected_node.name if selected_node else None
+
+            to_node(
+                c.name,
+                scope=source,
+                dialect=dialect,
+                scope_name=table,
+                upstream=node,
+                source_name=source_names.get(table) or source_name,
+                reference_node_name=reference_node_name,
+                trim_selects=trim_selects,
+                visited=visited,
+            )
+        else:
+            source = source or exp.Placeholder()
+            node.downstream.append(
+                Node(name=c.sql(comments=False), source=source, expression=source)
+            )
+
+    return node