dbt-swap 1.0__tar.gz

dbt_swap-1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.3
+Name: dbt-swap
+Version: 1.0
+Summary: A cli tool that adds a number of custom dbt commands.
+Author: Tobias Cadee
+Author-email: Tobias Cadee <tobias.cadee@ticketswap.com>
+Requires-Dist: sqlglot>=27
+Requires-Dist: typer>=0.19.2
+Requires-Python: >=3.12

dbt_swap-1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "dbt-swap"
+version = "1.0"
+description = "A cli tool that adds a number of custom dbt commands."
+authors = [
+    { name = "Tobias Cadee", email = "tobias.cadee@ticketswap.com" }
+]
+dependencies = [
+    "sqlglot>=27",
+    "typer>=0.19.2",
+]
+requires-python = ">=3.12"
+
+[project.scripts]
+dbt-swap = "dbt_swap.cli.main:app"
+
+[build-system]
+requires = ["uv_build>=0.8.6,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.2",
+    "dbt-core>=1.9.0,<1.10.0",
+    "dbt-postgres>=1.9.1",
+    "ruff>=0.14.0",
+]
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = 1

dbt_swap-1.0/src/dbt_swap/cli/common.py

@@ -0,0 +1,24 @@
+import os
+
+from rich.console import Console
+
+console = Console()
+
+
+def init_context(target: str, target_path: str, state: str):
+    """Initialize shared CLI context."""
+    return {
+        "target": target,
+        "target_path": target_path,
+        "state": state,
+    }
+
+
+def set_env(target: str, target_path: str, state: str):
+    """Set environment variables for all dbt-swap commands."""
+    if target:
+        os.environ["DBT_TARGET"] = target
+    if target_path:
+        os.environ["DBT_TARGET_PATH"] = target_path
+    if state:
+        os.environ["DBT_STATE_PATH"] = state

dbt_swap-1.0/src/dbt_swap/cli/main.py

@@ -0,0 +1,25 @@
+import typer
+from dbt_swap.cli import smart_build
+from dbt_swap.cli.common import init_context, set_env
+
+app = typer.Typer(help="dbt-swap CLI — utilities around dbt state and builds")
+
+
+# Shared context init (runs before each command)
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    target: str = typer.Option(None, help="DBT target name."),
+    target_path: str = typer.Option(None, help="Path to dbt target manifest."),
+    state: str = typer.Option("target/", help="Path to dbt state manifest."),
+):
+    """Initialize shared CLI context."""
+    ctx.obj = init_context(target=target, target_path=target_path, state=state)
+    set_env(target=target, target_path=target_path, state=state)
+
+
+# Register subcommands
+app.command("smart-build")(smart_build.smart_build)
+
+if __name__ == "__main__":
+    app()

dbt_swap-1.0/src/dbt_swap/cli/smart_build.py

@@ -0,0 +1,56 @@
+import typer
+import subprocess
+from dbt_swap.core.smart_builder import DbtSmartBuilder
+from dbt_swap.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+app = typer.Typer(help="Build only modified models intelligently.")
+
+
+@app.command("smart-build")
+def smart_build(
+    ctx: typer.Context,
+    dry_run: bool = typer.Option(False, help="Perform a dry run without making changes."),
+    verbose: bool = typer.Option(False, help="Enable verbose output."),
+):
+    """
+    Run a smart build — builds only modified models based on dbt state and column level lineage.
+    Additional arguments after `smart-build` are passed directly to `dbt build`.
+    """
+
+    dbt_smart_build = DbtSmartBuilder()
+    modified_nodes = dbt_smart_build.find_modified_nodes()
+
+    if ctx.args:
+        args = ctx.args
+    else:
+        args = []
+
+    command = f"dbt build -s {' '.join(modified_nodes)} {' '.join(args)}"
+
+    if dry_run:
+        resource_types = {node["resource_type"] for node in dbt_smart_build.nodes.values()}
+        resource_type_counts = {
+            resource_type: {
+                "smart_count": sum(
+                    1 for node in modified_nodes if dbt_smart_build.nodes[node]["resource_type"] == resource_type
+                ),
+                "total_count": sum(
+                    1
+                    for node in dbt_smart_build.modified_and_downstream_node_ids
+                    if dbt_smart_build.nodes[node]["resource_type"] == resource_type
+                ),
+            }
+            for resource_type in resource_types
+        }
+        for resource_type, counts in resource_type_counts.items():
+            if counts["total_count"] > 0:
+                logger.info(f"[DRY RUN] Would build {counts['smart_count']}/{counts['total_count']} {resource_type}(s)")
+        if verbose:
+            logger.info("[DRY RUN] Would build:")
+            for modified_node in modified_nodes:
+                logger.info(modified_node)
+    else:
+        subprocess.run(command, shell=True, check=True)

dbt_swap-1.0/src/dbt_swap/core/smart_builder.py

@@ -0,0 +1,270 @@
+import subprocess
+from pathlib import Path
+from sqlglot import parse_one, exp
+import json
+import os
+from dbt_swap.utils.logging import get_logger
+from functools import cached_property
+
+logger = get_logger(__name__)
+
+
+class DbtSmartBuilder:
+    def __init__(self):
+        # Allow overriding paths via environment for correctness with dbt --state and target-path
+        # DBT_STATE: directory containing a manifest.json representing the comparison state
+        # DBT_TARGET_STATE: directory containing the current target manifest.json (after compile)
+        self.target_dir = os.environ.get("DBT_TARGET_STATE", "./target")
+        self.state_dir = os.environ.get("DBT_STATE", "./state")
+        self.target_manifest_path = str(Path(self.target_dir) / "manifest.json")
+        self.state_manifest_path = str(Path(self.state_dir) / "manifest.json")
+        self.dialect = "postgres"  # Default dialect; could be made configurable
+
+    def list_changed_node_ids(self, downstream: bool = False) -> list[str]:
+        """Get list of changed node unique ids using dbt ls command."""
+        logger.info("Getting changed nodes...")
+        if downstream:
+            selector = "state:modified+"
+        else:
+            selector = "state:modified"
+        output = b""
+        try:
+            output = subprocess.check_output(
+                ["dbt", "ls", "--select", selector, "--output", "json", "--quiet"],
+                stderr=subprocess.STDOUT,
+            )
+        except subprocess.CalledProcessError as e:
+            logger.error(
+                f"Failed to list changed nodes with dbt: {e}\n"
+                f"Command: {e.cmd}\n"
+                f"Return code: {e.returncode}\n"
+                f"Output: {e.output.decode() if e.output else ''}"
+            )
+            # If dbt ls fails (e.g., wrong working directory or missing state), treat as no changes
+            return []
+
+        if not output.strip():
+            return []
+
+        changed_files = [json.loads(line) for line in output.decode().splitlines()]
+        changed_node_ids = [node["unique_id"] for node in changed_files]
+
+        return changed_node_ids
+
+    @cached_property
+    def modified_node_ids(self) -> list[str]:
+        """Get list of changed models using dbt ls command."""
+        return self.list_changed_node_ids(downstream=False)
+
+    @cached_property
+    def modified_and_downstream_node_ids(self) -> list[str]:
+        """Get list of changed models and their downstream dependencies using dbt ls command."""
+        return self.list_changed_node_ids(downstream=True)
+
+    def compile_changed_nodes(self) -> None:
+        """Compile changed nodes and their dependencies using dbt compile command."""
+        logger.info("Compiling changed models and dependencies...")
+        subprocess.check_output(
+            [
+                "dbt",
+                "compile",
+                "--select",
+                "state:modified+",
+                "--quiet",
+                "--state",
+                self.state_dir,
+            ]
+        )
+
+    def load_manifest(self, manifest_path) -> dict:
+        """Load manifest from the given path."""
+        path = Path(manifest_path)
+        if not path.exists():
+            logger.error(f"Manifest not found at {manifest_path}. Run 'dbt compile' first.")
+            raise FileNotFoundError(f"Manifest not found at {manifest_path}. Run 'dbt compile' first.")
+
+        with open(path) as f:
+            manifest = json.load(f)
+
+        return manifest
+
+    @cached_property
+    def manifest(self) -> dict:
+        return self.load_manifest(self.target_manifest_path)
+
+    @cached_property
+    def compare_manifest(self) -> dict:
+        return self.load_manifest(self.state_manifest_path)
+
+    @cached_property
+    def nodes(self) -> dict:
+        """Get nodes from the target manifest."""
+        return self.manifest.get("nodes", {})
+
+    @cached_property
+    def compare_nodes(self) -> dict:
+        """Get nodes from the state manifest."""
+        return self.compare_manifest.get("nodes", {})
+
+    @cached_property
+    def child_map(self) -> dict:
+        """Return the child map from target manifest."""
+        return self.manifest.get("child_map", {})
+
+    def find_changed_columns(self, node: dict) -> list[str]:
+        """Find columns that have changed in a node."""
+        if node["resource_type"] != "model":
+            return []
+
+        sql = node.get("compiled_code", "")
+        compare_sql = self.compare_nodes.get(node["unique_id"], {}).get("compiled_code", "select *")
+
+        try:
+            parsed = [
+                projection
+                for select in parse_one(sql, dialect=self.dialect).find_all(exp.Select)
+                for projection in select.expressions
+            ]
+            compare_parsed = [
+                projection
+                for select in parse_one(compare_sql, dialect=self.dialect).find_all(exp.Select)
+                for projection in select.expressions
+            ]
+
+            diffs = set(parsed) - set(compare_parsed)
+            return [change.alias_or_name for change in diffs]
+        except Exception as e:
+            logger.warning(f"Could not parse SQL for node {node['name']}: {e}")
+            return []
+
+    def find_column_refs(
+        self, node: dict, changed_column: str, refs: list[str] | None = None, visited: set[str] | None = None
+    ) -> list[str]:
+        """Find references to changed columns in a node."""
+        if refs is None:
+            try:
+                sql = node.get("compiled_code", "select *")
+            except Exception as e:
+                logger.warning(f"Could not get SQL for node {node}: {e}")
+                sql = "select *"
+            refs = [
+                projection
+                for select in parse_one(sql, dialect=self.dialect).find_all(exp.Select)
+                for projection in select.expressions
+            ]
+            # If the model selects all columns, consider all changed columns as referenced
+            if set(refs) == {exp.Star()}:
+                return [changed_column]
+
+        if visited is None:
+            visited = set()
+
+        if changed_column in visited:
+            return []
+
+        visited.add(changed_column)
+        column_refs = []
+
+        for ref in refs:
+            if isinstance(ref, exp.Column) and ref.alias_or_name == changed_column:
+                column_refs.append(ref.alias_or_name)
+            elif isinstance(ref, exp.Alias):
+                for col in ref.this.find_all(exp.Column):
+                    if col.alias_or_name == changed_column:
+                        column_refs.append(ref.alias_or_name)
+
+        # If there are no new column references, return an empty set to avoid infinite recursion
+        if column_refs:
+            for column_ref in set(column_refs):
+                column_refs.extend(self.find_column_refs(node, column_ref, refs, visited))
+
+        # Recursively find column references
+        return column_refs
+
+    def search_in_graph(
+        self,
+        changed_node_id: str,
+        changed_columns: list[str],
+        visited: set[str] | None = None,
+        all: bool = False,
+    ) -> list[str]:
+        """Recursively search for models affected by column changes."""
+        if visited is None:
+            visited = set()
+
+        if changed_node_id in visited:
+            return []
+
+        visited.add(changed_node_id)
+        affected = [changed_node_id]
+
+        for child_id in self.child_map.get(changed_node_id, []):
+            if child_id not in visited:
+                child_node = self.nodes[child_id]
+                # If no specific columns changed, all downstream nodes are affected
+                if all:
+                    logger.info(
+                        f"{child_node['resource_type']} {child_id} is affected by changes in {changed_node_id} (node changed entirely)"
+                    )
+                    affected.extend(self.search_in_graph(child_id, changed_columns, visited, all=all))
+                else:
+                    # If the changed_model or child is not a model (e.g., a snapshot or seed), consider it affected
+                    if (
+                        self.nodes[changed_node_id]["resource_type"] != "model"
+                        or child_node["resource_type"] != "model"
+                    ):
+                        logger.info(
+                            f"{child_node['resource_type']} {child_id} is affected by changes in {changed_node_id}"
+                        )
+                        affected.extend(self.search_in_graph(child_id, changed_columns, visited))
+                        continue
+
+                    # If the model selects all columns, consider all changed columns as referenced
+                    column_refs = list(
+                        {
+                            column_ref
+                            for changed_column in changed_columns
+                            for column_ref in self.find_column_refs(child_node, changed_column)
+                        }
+                    )
+
+                    # If there are column references, consider the node affected
+                    if column_refs:
+                        logger.info(
+                            f"{child_node['resource_type']} {child_id} is affected by changes in {changed_node_id}, columns: {column_refs}"
+                        )
+                        affected.extend(self.search_in_graph(child_id, column_refs, visited))
+
+        return list(set(affected))
+
+    def find_modified_nodes(self) -> list[str]:
+        """Find all nodes affected by changes."""
+        logger.info("🧠 Starting smart model selection...")
+
+        if not self.modified_node_ids:
+            logger.info("No modified nodes found.")
+            return []
+
+        self.compile_changed_nodes()
+
+        affected_nodes = set()
+        for changed_node_id in self.modified_node_ids:
+            changed_node = self.nodes[changed_node_id]
+            changed_columns = self.find_changed_columns(changed_node)
+            column_refs = list(
+                {
+                    column_ref
+                    for changed_column in changed_columns
+                    for column_ref in self.find_column_refs(changed_node, changed_column)
+                }
+            )
+            # If no specific columns changed, treat it as a full change
+            all = len(changed_columns) == 0
+            if all:
+                logger.info(f"Changed node: {changed_node['name']}")
+            else:
+                logger.info(f"Changed model: {changed_node['name']} with changed columns: {column_refs}")
+
+            affected_nodes.update(self.search_in_graph(changed_node_id, column_refs, all=all))
+
+        return list([self.nodes[affected_node]["name"] for affected_node in affected_nodes])

dbt_swap-1.0/src/dbt_swap/utils/logging.py

@@ -0,0 +1,17 @@
+import logging
+import sys
+
+
+def get_logger(name: str = "dbt_cli"):
+    """Return a configured logger instance."""
+    logger = logging.getLogger(name)
+
+    if not logger.handlers:  # Prevent duplicate handlers
+        logger.setLevel(logging.INFO)
+
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", "%Y-%m-%d %H:%M:%S")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+    return logger