recce-nightly 1.10.0.20250625__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/adapter/dbt_adapter/__init__.py

@@ -3,6 +3,7 @@ import logging
 import os
 import uuid
 from contextlib import contextmanager
+from copy import deepcopy
 from dataclasses import dataclass, fields
 from errno import ENOENT
 from functools import lru_cache
@@ -25,12 +26,13 @@ from recce.event import log_performance
 from recce.exceptions import RecceException
 from recce.util.cll import CLLPerformanceTracking, cll
 from recce.util.lineage import (
+    build_column_key,
     filter_dependency_maps,
-    filter_lineage_vertices,
-    find_column_dependencies,
     find_downstream,
     find_upstream,
 )
+from recce.util.perf_tracking import LineagePerfTracker
+from recce.util.startup_perf import track_timing
 
 from ...tasks.profile import ProfileTask
 from ...util.breaking import BreakingPerformanceTracking, parse_change_category
@@ -109,6 +111,7 @@ from dbt.config.runtime import RuntimeConfig  # noqa: E402
 from dbt.contracts.graph.manifest import (  # noqa: E402
     MacroManifest,
     Manifest,
+    ManifestMetadata,
     WritableManifest,
 )
 from dbt.contracts.graph.nodes import ManifestNode  # noqa: E402
@@ -208,9 +211,12 @@ def as_manifest(m: WritableManifest) -> Manifest:
         new_data = {k: v for k, v in data.items() if k in all_fields}
         return Manifest(**new_data)
     else:
-        return Manifest.from_writable_manifest(m)
+        result = Manifest.from_writable_manifest(m)
+        result.metadata = ManifestMetadata(**m.metadata.__dict__)
+        return result
 
 
+@track_timing(record_size=True)
 def load_manifest(path: str = None, data: dict = None):
     if path is not None:
         if not os.path.isfile(path):
@@ -220,6 +226,7 @@ def load_manifest(path: str = None, data: dict = None):
         return WritableManifest.upgrade_schema_version(data)
 
 
+@track_timing(record_size=True)
 def load_catalog(path: str = None, data: dict = None):
     if path is not None:
         if not os.path.isfile(path):
@@ -278,7 +285,7 @@ class DbtArgs:
     target_path: Optional[str] = (None,)
     project_only_flags: Optional[Dict[str, Any]] = None
     which: Optional[str] = None
-    state_modified_compare_more_unrendered_values: Optional[bool] = False  # new flag added since dbt v1.9
+    state_modified_compare_more_unrendered_values: Optional[bool] = True  # new flag added since dbt v1.9
 
 
 @dataclass
@@ -407,7 +414,7 @@ class DbtAdapter(BaseAdapter):
 
         if self.adapter.connections.TYPE == "databricks":
             # reference: get_columns_in_relation (dbt/adapters/databricks/impl.py)
-            from dbt.adapters.databricks import DatabricksColumn
+            from dbt.adapters.databricks.column import DatabricksColumn
 
             rows = columns
             columns = []
@@ -472,6 +479,7 @@ class DbtAdapter(BaseAdapter):
 
         return result
 
+    @track_timing("artifact_load")
     def load_artifacts(self):
         """
         Load the artifacts from the 'target' and 'target-base' directory
@@ -487,16 +495,20 @@ class DbtAdapter(BaseAdapter):
 
         # load the artifacts
         path = os.path.join(project_root, target_path, "manifest.json")
-        curr_manifest = load_manifest(path=path)
+        curr_manifest = load_manifest(path=path, timing_name="curr_manifest")
         if curr_manifest is None:
             raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
         path = os.path.join(project_root, target_base_path, "manifest.json")
-        base_manifest = load_manifest(path=path)
+        base_manifest = load_manifest(path=path, timing_name="base_manifest")
         if base_manifest is None:
             raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
 
-        curr_catalog = load_catalog(path=os.path.join(project_root, target_path, "catalog.json"))
-        base_catalog = load_catalog(path=os.path.join(project_root, target_base_path, "catalog.json"))
+        curr_catalog = load_catalog(
+            path=os.path.join(project_root, target_path, "catalog.json"), timing_name="curr_catalog"
+        )
+        base_catalog = load_catalog(
+            path=os.path.join(project_root, target_base_path, "catalog.json"), timing_name="base_catalog"
+        )
 
         # set the value if all the artifacts are loaded successfully
         self.curr_manifest = curr_manifest
@@ -599,7 +611,15 @@ class DbtAdapter(BaseAdapter):
             return node.compiled_code
         else:
             from dbt.clients import jinja
-            from dbt.context.providers import generate_runtime_model_context
+            from dbt.context.providers import (
+                generate_runtime_macro_context,
+                generate_runtime_model_context,
+            )
+
+            # Set up macro resolver for dbt >= 1.8
+            macro_manifest = MacroManifest(manifest.macros)
+            self.adapter.set_macro_resolver(macro_manifest)
+            self.adapter.set_macro_context_generator(generate_runtime_macro_context)
 
             jinja_ctx = generate_runtime_model_context(node, self.runtime_config, manifest)
             jinja_ctx.update(context)
@@ -658,8 +678,8 @@ class DbtAdapter(BaseAdapter):
     @lru_cache(maxsize=2)
     def get_lineage_cached(self, base: Optional[bool] = False, cache_key=0):
         if base is False:
-            cll_tracker = CLLPerformanceTracking()
-            cll_tracker.start_lineage()
+            perf_tracker = LineagePerfTracker()
+            perf_tracker.start_lineage()
 
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
@@ -736,6 +756,7 @@ class DbtAdapter(BaseAdapter):
             nodes[unique_id] = {
                 "id": source["unique_id"],
                 "name": source["name"],
+                "source_name": source["source_name"],
                 "resource_type": source["resource_type"],
                 "package_name": source["package_name"],
                 "config": source["config"],
@@ -777,10 +798,10 @@ class DbtAdapter(BaseAdapter):
         parent_map = self.build_parent_map(nodes, base)
 
         if base is False:
-            cll_tracker.end_lineage()
-            cll_tracker.set_total_nodes(len(nodes))
-            log_performance("model lineage", cll_tracker.to_dict())
-            cll_tracker.reset()
+            perf_tracker.end_lineage()
+            perf_tracker.set_total_nodes(len(nodes))
+            log_performance("model lineage", perf_tracker.to_dict())
+            perf_tracker.reset()
 
         return dict(
             parent_map=parent_map,
@@ -793,15 +814,43 @@ class DbtAdapter(BaseAdapter):
     def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
         base = self.get_lineage(base=True)
         current = self.get_lineage(base=False)
-        keys = {*base.get("nodes", {}).keys(), *current.get("nodes", {}).keys()}
 
-        # Start to diff
-        perf_tracking = BreakingPerformanceTracking()
-        perf_tracking.start_lineage_diff()
+        modified_nodes = self.select_nodes(select="state:modified")
+        diff = {}
+        for node_id in modified_nodes:
+            base_node = base.get("nodes", {}).get(node_id)
+            curr_node = current.get("nodes", {}).get(node_id)
+            if base_node and curr_node:
+                diff[node_id] = NodeDiff(change_status="modified")
+            elif base_node:
+                diff[node_id] = NodeDiff(change_status="removed")
+            elif curr_node:
+                diff[node_id] = NodeDiff(change_status="added")
+
+        return LineageDiff(
+            base=base,
+            current=current,
+            diff=diff,
+        )
+
+    @lru_cache(maxsize=128)
+    def get_change_analysis_cached(self, node_id: str):
+        breaking_perf_tracker = BreakingPerformanceTracking()
+        lineage_diff = self.get_lineage_diff()
+        diff = lineage_diff.diff
+
+        if node_id not in diff or diff[node_id].change_status != "modified":
+            return diff.get(node_id)
+
+        breaking_perf_tracker.increment_modified_nodes()
+        breaking_perf_tracker.start_lineage_diff()
+
+        base = lineage_diff.base
+        current = lineage_diff.current
 
         base_manifest = as_manifest(self.get_manifest(True))
         curr_manifest = as_manifest(self.get_manifest(False))
-        perf_tracking.record_checkpoint("manifest")
+        breaking_perf_tracker.record_checkpoint("manifest")
 
         def ref_func(*args):
             if len(args) == 1:
@@ -821,111 +870,106 @@ class DbtAdapter(BaseAdapter):
             source=source_func,
         )
 
-        # for each node, compare the base and current lineage
-        diff = {}
-        for key in keys:
-            base_node = base.get("nodes", {}).get(key)
-            curr_node = current.get("nodes", {}).get(key)
-            if base_node and curr_node:
-                base_checksum = base_node.get("checksum", {}).get("checksum")
-                curr_checksum = curr_node.get("checksum", {}).get("checksum")
-                change = None
-                if base_checksum is None or curr_checksum is None or base_checksum == curr_checksum:
-                    continue
-
-                if curr_node.get("resource_type") == "model":
-                    try:
-                        perf_tracking.increment_modified_nodes()
-
-                        def _get_schema(lineage):
-                            schema = {}
-                            nodes = lineage["nodes"]
-                            parent_list = lineage["parent_map"].get(key, [])
-                            for parent_id in parent_list:
-                                parent_node = nodes.get(parent_id)
-                                if parent_node is None:
-                                    continue
-                                columns = parent_node.get("columns") or {}
-                                name = parent_node.get("name")
-                                if parent_node.get("resource_type") == "source":
-                                    parts = parent_id.split(".")
-                                    source = parts[2]
-                                    table = parts[3]
-                                    source = source.replace("-", "_")
-                                    name = f"__{source}__{table}"
-                                schema[name] = {name: column.get("type") for name, column in columns.items()}
-                            return schema
-
-                        base_sql = self.generate_sql(
-                            base_node.get("raw_code"),
-                            context=jinja_context,
-                            provided_manifest=base_manifest,
-                        )
-                        curr_sql = self.generate_sql(
-                            curr_node.get("raw_code"),
-                            context=jinja_context,
-                            provided_manifest=curr_manifest,
-                        )
-                        base_schema = _get_schema(base)
-                        curr_schema = _get_schema(current)
-                        dialect = self.adapter.connections.TYPE
-                        if curr_manifest.metadata.adapter_type is not None:
-                            dialect = curr_manifest.metadata.adapter_type
-
-                        change = parse_change_category(
-                            base_sql,
-                            curr_sql,
-                            old_schema=base_schema,
-                            new_schema=curr_schema,
-                            dialect=dialect,
-                            perf_tracking=perf_tracking,
-                        )
-
-                        # Make sure that the case of the column names are the same
-                        changed_columns = {
-                            column.lower(): change_status for column, change_status in (change.columns or {}).items()
-                        }
-                        changed_columns_names = set(changed_columns)
-                        changed_columns_final = {}
-
-                        base_columns = base_node.get("columns") or {}
-                        curr_columns = curr_node.get("columns") or {}
-                        columns_names = set(base_columns) | set(curr_columns)
+        base_node = base.get("nodes", {}).get(node_id)
+        curr_node = current.get("nodes", {}).get(node_id)
+        change = NodeChange(category="unknown")
+        if (
+            curr_node.get("resource_type") in ["model", "snapshot"]
+            and curr_node.get("raw_code") is not None
+            and base_node.get("raw_code") is not None
+        ):
+            try:
+
+                def _get_schema(lineage):
+                    schema = {}
+                    nodes = lineage["nodes"]
+                    parent_list = lineage["parent_map"].get(node_id, [])
+                    for parent_id in parent_list:
+                        parent_node = nodes.get(parent_id)
+                        if parent_node is None:
+                            continue
+                        columns = parent_node.get("columns") or {}
+                        name = parent_node.get("name")
+                        if parent_node.get("resource_type") == "source":
+                            parts = parent_id.split(".")
+                            source = parts[2]
+                            table = parts[3]
+                            source = source.replace("-", "_")
+                            name = f"__{source}__{table}"
+                        schema[name] = {name: column.get("type") for name, column in columns.items()}
+                    return schema
+
+                base_sql = self.generate_sql(
+                    base_node.get("raw_code"),
+                    context=jinja_context,
+                    provided_manifest=base_manifest,
+                )
+                curr_sql = self.generate_sql(
+                    curr_node.get("raw_code"),
+                    context=jinja_context,
+                    provided_manifest=curr_manifest,
+                )
+                base_schema = _get_schema(base)
+                curr_schema = _get_schema(current)
+                dialect = self.adapter.connections.TYPE
+                if curr_manifest.metadata.adapter_type is not None:
+                    dialect = curr_manifest.metadata.adapter_type
+
+                change = parse_change_category(
+                    base_sql,
+                    curr_sql,
+                    old_schema=base_schema,
+                    new_schema=curr_schema,
+                    dialect=dialect,
+                    perf_tracking=breaking_perf_tracker,
+                )
 
-                        for column_name in columns_names:
-                            if column_name.lower() in changed_columns_names:
-                                changed_columns_final[column_name] = changed_columns[column_name.lower()]
+                # Make sure that the case of the column names are the same
+                changed_columns = {
+                    column.lower(): change_status for column, change_status in (change.columns or {}).items()
+                }
+                changed_columns_names = set(changed_columns)
+                changed_columns_final = {}
 
-                        change.columns = changed_columns_final
-                    except Exception:
-                        change = NodeChange(category="unknown")
+                base_columns = base_node.get("columns") or {}
+                curr_columns = curr_node.get("columns") or {}
+                columns_names = set(base_columns) | set(curr_columns)
 
-                diff[key] = NodeDiff(change_status="modified", change=change)
-            elif base_node:
-                diff[key] = NodeDiff(change_status="removed")
-            elif curr_node:
-                diff[key] = NodeDiff(change_status="added")
+                for column_name in columns_names:
+                    if column_name.lower() in changed_columns_names:
+                        changed_columns_final[column_name] = changed_columns[column_name.lower()]
 
-        perf_tracking.end_lineage_diff()
-        log_performance("model lineage diff", perf_tracking.to_dict())
+                change.columns = changed_columns_final
+            except Exception:
+                # TODO: telemetry
+                pass
 
-        return LineageDiff(
-            base=base,
-            current=current,
-            diff=diff,
-        )
+        breaking_perf_tracker.end_lineage_diff()
+        log_performance("change analysis per node", breaking_perf_tracker.to_dict())
+        breaking_perf_tracker.reset()
+        node_diff = diff.get(node_id)
+        node_diff.change = change
+        return node_diff
 
     def get_cll(
         self,
         node_id: Optional[str] = None,
         column: Optional[str] = None,
         change_analysis: Optional[bool] = False,
-        cll: Optional[bool] = True,
-        upstream: Optional[bool] = True,
-        downstream: Optional[bool] = True,
+        no_cll: Optional[bool] = False,
+        no_upstream: Optional[bool] = False,
+        no_downstream: Optional[bool] = False,
         no_filter: Optional[bool] = False,
     ) -> CllData:
-        cll_tracker = CLLPerformanceTracking()
+        cll_tracker = LineagePerfTracker()
+        cll_tracker.set_params(
+            has_node=node_id is not None,
+            has_column=column is not None,
+            change_analysis=change_analysis,
+            no_cll=no_cll,
+            no_upstream=no_upstream,
+            no_downstream=no_downstream,
+        )
         cll_tracker.start_column_lineage()
 
         manifest = self.curr_manifest
@@ -936,47 +980,114 @@ class DbtAdapter(BaseAdapter):
             cll_node_ids = {node_id}
         else:
             lineage_diff = self.get_lineage_diff()
-            cll_node_ids = lineage_diff.diff.keys()
+            cll_node_ids = set(lineage_diff.diff.keys())
+
+        cll_tracker.set_init_nodes(len(cll_node_ids))
 
         nodes = {}
         columns = {}
         parent_map = {}
         child_map = {}
 
-        if upstream:
+        if not no_upstream:
             cll_node_ids = cll_node_ids.union(find_upstream(cll_node_ids, manifest_dict.get("parent_map")))
-        if downstream:
+        if not no_downstream:
             cll_node_ids = cll_node_ids.union(find_downstream(cll_node_ids, manifest_dict.get("child_map")))
 
-        if cll:
+        if not no_cll:
+            allowed_related_nodes = set()
+            for key in ["sources", "nodes", "exposures", "metrics"]:
+                attr = getattr(manifest, key)
+                allowed_related_nodes.update(set(attr.keys()))
+            if hasattr(manifest, "semantic_models"):
+                attr = getattr(manifest, "semantic_models")
+                allowed_related_nodes.update(set(attr.keys()))
             for cll_node_id in cll_node_ids:
-                if (
-                    cll_node_id not in manifest.sources
-                    and cll_node_id not in manifest.nodes
-                    and cll_node_id not in manifest.exposures
-                ):
+                if cll_node_id not in allowed_related_nodes:
                     continue
-                cll_data_one = self.get_cll_cached(cll_node_id, base=False)
+                cll_data_one = deepcopy(self.get_cll_cached(cll_node_id, base=False))
+                cll_tracker.increment_cll_nodes()
                 if cll_data_one is None:
                     continue
 
-                node_diff = self.get_lineage_diff().diff.get(cll_node_id) if change_analysis else None
-                for n_id, n in cll_data_one.nodes.items():
-                    nodes[n_id] = n
-
-                    if node_diff is not None:
-                        n.change_status = node_diff.change_status
-                        if node_diff.change is not None:
-                            n.change_category = node_diff.change.category
+                nodes[cll_node_id] = cll_data_one.nodes.get(cll_node_id)
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
+                if node_diff is not None:
+                    nodes[cll_node_id].change_status = node_diff.change_status
+                    if node_diff.change is not None:
+                        nodes[cll_node_id].change_category = node_diff.change.category
                 for c_id, c in cll_data_one.columns.items():
                     columns[c_id] = c
-                    if node_diff is not None and node_diff.change is not None:
-                        column_diff = node_diff.change.columns.get(c.name)
-                        if column_diff:
-                            c.change_status = column_diff
+                    if node_diff is not None:
+                        if node_diff.change_status == "added":
+                            c.change_status = "added"
+                        elif node_diff.change_status == "removed":
+                            c.change_status = "removed"
+                        elif node_diff.change is not None and node_diff.change.columns is not None:
+                            column_diff = node_diff.change.columns.get(c.name)
+                            if column_diff:
+                                c.change_status = column_diff
 
                 for p_id, parents in cll_data_one.parent_map.items():
                     parent_map[p_id] = parents
+        else:
+            for cll_node_id in cll_node_ids:
+                cll_node = None
+                cll_node_columns: Dict[str, CllColumn] = {}
+
+                if cll_node_id in manifest.sources:
+                    cll_node = CllNode.build_cll_node(manifest, "sources", cll_node_id)
+                    if self.curr_catalog and cll_node_id in self.curr_catalog.sources:
+                        cll_node_columns = {
+                            column.name: CllColumn(
+                                id=f"{cll_node_id}_{column.name}",
+                                table_id=cll_node_id,
+                                name=column.name,
+                                type=column.type,
+                            )
+                            for column in self.curr_catalog.sources[cll_node_id].columns.values()
+                        }
+                elif cll_node_id in manifest.nodes:
+                    cll_node = CllNode.build_cll_node(manifest, "nodes", cll_node_id)
+                    if self.curr_catalog and cll_node_id in self.curr_catalog.nodes:
+                        cll_node_columns = {
+                            column.name: CllColumn(
+                                id=f"{cll_node_id}_{column.name}",
+                                table_id=cll_node_id,
+                                name=column.name,
+                                type=column.type,
+                            )
+                            for column in self.curr_catalog.nodes[cll_node_id].columns.values()
+                        }
+                elif cll_node_id in manifest.exposures:
+                    cll_node = CllNode.build_cll_node(manifest, "exposures", cll_node_id)
+                elif hasattr(manifest, "semantic_models") and cll_node_id in manifest.semantic_models:
+                    cll_node = CllNode.build_cll_node(manifest, "semantic_models", cll_node_id)
+                elif cll_node_id in manifest.metrics:
+                    cll_node = CllNode.build_cll_node(manifest, "metrics", cll_node_id)
+
+                if not cll_node:
+                    continue
+                nodes[cll_node_id] = cll_node
+
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
+                if node_diff is not None:
+                    cll_node.change_status = node_diff.change_status
+                    if node_diff.change is not None:
+                        cll_node.change_category = node_diff.change.category
+                        for c, cll_column in cll_node_columns.items():
+                            cll_node.columns[c] = cll_column
+                            columns[cll_column.id] = cll_column
+                            if node_diff.change.columns and c in node_diff.change.columns:
+                                cll_column.change_status = node_diff.change.columns[c]
+
+                parent_map[cll_node_id] = manifest.parent_map.get(cll_node_id, [])
 
         # build the child map
         for parent_id, parents in parent_map.items():
@@ -987,47 +1098,90 @@ class DbtAdapter(BaseAdapter):
 
         # Find the anchor nodes
         anchor_node_ids = set()
+        extra_node_ids = set()
         if node_id is None and column is None:
             if change_analysis:
                 # If change analysis is requested, we need to find the nodes that have changes
-                for node_id, node_diff in self.get_lineage_diff().diff.items():
-                    if node_diff.change.category == "breaking":
-                        anchor_node_ids.add(node_id)
-                    for column_name in node_diff.change.columns:
-                        anchor_node_ids.add(f"{node_id}_{column_name}")
+                lineage_diff = self.get_lineage_diff()
+                for nid, nd in lineage_diff.diff.items():
+                    if nd.change_status == "added":
+                        anchor_node_ids.add(nid)
+                        n = lineage_diff.current["nodes"].get(nid)
+                        n_columns = n.get("columns", {})
+                        for c in n_columns:
+                            anchor_node_ids.add(build_column_key(nid, c))
+                        continue
+                    if nd.change_status == "removed":
+                        extra_node_ids.add(nid)
+                        continue
+
+                    node_diff = self.get_change_analysis_cached(nid)
+                    if node_diff is not None and node_diff.change is not None:
+                        extra_node_ids.add(nid)
+                        if no_cll:
+                            if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
+                                anchor_node_ids.add(nid)
+                        else:
+                            if node_diff.change.category in ["breaking", "unknown"]:
+                                anchor_node_ids.add(nid)
+                        if node_diff.change.columns is not None:
+                            for column_name in node_diff.change.columns:
+                                anchor_node_ids.add(f"{nid}_{column_name}")
             else:
                 lineage_diff = self.get_lineage_diff()
                 anchor_node_ids = lineage_diff.diff.keys()
         elif node_id is not None and column is None:
             if change_analysis:
                 # If change analysis is requested, we need to find the nodes that have changes
-                node_diff = self.get_lineage_diff().diff.get(node_id)
-                if node_diff:
-                    if node_diff.change.category == "breaking":
-                        anchor_node_ids.add(node_id)
-                    for column_name in node_diff.change.columns:
-                        anchor_node_ids.add(f"{node_id}_{column_name}")
+                node_diff = self.get_change_analysis_cached(node_id)
+                if node_diff is not None and node_diff.change is not None:
+                    extra_node_ids.add(node_id)
+                    if no_cll:
+                        if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
+                            anchor_node_ids.add(node_id)
+                    else:
+                        if node_diff.change.category in ["breaking", "unknown"]:
+                            anchor_node_ids.add(node_id)
+                    if node_diff.change.columns is not None:
+                        for column_name in node_diff.change.columns:
+                            anchor_node_ids.add(f"{node_id}_{column_name}")
                 else:
                     anchor_node_ids.add(node_id)
             else:
                 anchor_node_ids.add(node_id)
+                if not no_cll:
+                    node = nodes.get(node_id)
+                    if node:
+                        for column_name in node.columns:
+                            column_key = build_column_key(node_id, column_name)
+                            anchor_node_ids.add(column_key)
         else:
             anchor_node_ids.add(f"{node_id}_{column}")
 
+        cll_tracker.set_anchor_nodes(len(anchor_node_ids))
         result_node_ids = set(anchor_node_ids)
-        if upstream:
+        if not no_upstream:
             result_node_ids = result_node_ids.union(find_upstream(anchor_node_ids, parent_map))
-        if downstream:
+        if not no_downstream:
             result_node_ids = result_node_ids.union(find_downstream(anchor_node_ids, child_map))
 
         # Filter the nodes and columns based on the anchor nodes
         if not no_filter:
-            nodes = {k: v for k, v in nodes.items() if k in result_node_ids}
-            columns = {k: v for k, v in columns.items() if k in result_node_ids}
+            nodes = {k: v for k, v in nodes.items() if k in result_node_ids or k in extra_node_ids}
+            columns = {k: v for k, v in columns.items() if k in result_node_ids or k in extra_node_ids}
+
+            for node in nodes.values():
+                node.columns = {
+                    k: v for k, v in node.columns.items() if v.id in result_node_ids or v.id in extra_node_ids
+                }
+
+                if change_analysis:
+                    node.impacted = node.id in result_node_ids
+
             parent_map, child_map = filter_dependency_maps(parent_map, child_map, result_node_ids)
 
         cll_tracker.end_column_lineage()
-        cll_tracker.set_total_nodes(len(nodes))
+        cll_tracker.set_total_nodes(len(nodes) + len(columns))
         log_performance("column level lineage", cll_tracker.to_dict())
         cll_tracker.reset()
 
@@ -1046,6 +1200,9 @@ class DbtAdapter(BaseAdapter):
         if node is None:
             return None
 
+        cll_tracker.set_total_nodes(1)
+        cll_tracker.start_column_lineage()
+
         def _apply_all_columns(node: CllNode, transformation_type):
             cll_data = CllData()
             cll_data.nodes[node.id] = node
@@ -1170,6 +1327,10 @@ class DbtAdapter(BaseAdapter):
                     depends_on.add(parent_key)
                 column.transformation_type = c2c_map[name].transformation_type
             cll_data.parent_map[column_id] = set(depends_on)
+
+        cll_tracker.end_column_lineage()
+        log_performance("column level lineage per node", cll_tracker.to_dict())
+        cll_tracker.reset()
         return cll_data
 
     def get_cll_node(self, node_id: str, base: Optional[bool] = False) -> Tuple[Optional[CllNode], list[str]]:
@@ -1181,21 +1342,12 @@ class DbtAdapter(BaseAdapter):
         # model, seed, snapshot
         if node_id in manifest.nodes:
             found = manifest.nodes[node_id]
-            if found.resource_type not in ["model", "seed", "snapshot"]:
-                return None, []
-
             unique_id = found.unique_id
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-                raw_code=found.raw_code,
-            )
+            node = CllNode.build_cll_node(manifest, "nodes", node_id)
             if hasattr(found.depends_on, "nodes"):
                 parent_list = found.depends_on.nodes
 
-            if catalog is not None and unique_id in catalog.nodes:
+            if catalog is not None and node is not None and unique_id in catalog.nodes:
                 columns = {}
                 for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
                     column_id = f"{unique_id}_{col_name}"
@@ -1207,17 +1359,10 @@ class DbtAdapter(BaseAdapter):
         if node_id in manifest.sources:
             found = manifest.sources[node_id]
             unique_id = found.unique_id
-
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-                source_name=found.source_name,
-            )
+            node = CllNode.build_cll_node(manifest, "sources", node_id)
             parent_list = []
 
-            if catalog is not None and unique_id in catalog.sources:
+            if catalog is not None and node is not None and unique_id in catalog.sources:
                 columns = {}
                 for col_name, col_metadata in catalog.sources[unique_id].columns.items():
                     column_id = f"{unique_id}_{col_name}"
@@ -1228,13 +1373,19 @@ class DbtAdapter(BaseAdapter):
         # exposure
         if node_id in manifest.exposures:
             found = manifest.exposures[node_id]
+            node = CllNode.build_cll_node(manifest, "exposures", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-            )
+        if hasattr(manifest, "semantic_models") and node_id in manifest.semantic_models:
+            found = manifest.semantic_models[node_id]
+            node = CllNode.build_cll_node(manifest, "semantic_models", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
+
+        if node_id in manifest.metrics:
+            found = manifest.metrics[node_id]
+            node = CllNode.build_cll_node(manifest, "metrics", node_id)
             if hasattr(found.depends_on, "nodes"):
                 parent_list = found.depends_on.nodes
 
@@ -1250,73 +1401,6 @@ class DbtAdapter(BaseAdapter):
             }
         return None
 
-    def get_impact_radius(self, node_id: str) -> CllData:
-        impacted_nodes = self.get_impacted_nodes(node_id)
-        impacted_cll = self.get_impacted_cll(node_id)
-
-        # merge impact radius
-        return self._merge_cll_data(impacted_nodes, impacted_cll)
-
-    def get_impacted_nodes(self, node_id: str) -> CllData:
-        lineage_diff = self.get_lineage_diff()
-        diff_info = lineage_diff.diff.get(node_id)
-        if diff_info is None:
-            return CllData()
-        change_category = diff_info.change.category
-
-        if change_category == "breaking":
-            cll = self.get_cll(node_id, no_filter=True)
-            _, downstream = find_column_dependencies(node_id, cll.parent_map, cll.child_map)
-            relevant_columns = {node_id}
-            relevant_columns.update(downstream)
-            nodes, columns = filter_lineage_vertices(cll.nodes, cll.columns, relevant_columns)
-            p_map, c_map = filter_dependency_maps(cll.parent_map, cll.child_map, relevant_columns)
-
-            return CllData(nodes=nodes, columns=columns, parent_map=p_map, child_map=c_map)
-
-        return CllData()
-
-    def get_impacted_cll(self, node_id: str) -> CllData:
-        lineage_diff = self.get_lineage_diff()
-        diff_info = lineage_diff.diff.get(node_id)
-        if diff_info is None:
-            return CllData()
-        change_columns = diff_info.change.columns
-
-        cll = self.get_cll(node_id, no_filter=True)
-        relevant_columns = set()
-        for col, change_status in change_columns.items():
-            if change_status == "removed":
-                continue
-            target_column = f"{node_id}_{col}"
-            _, downstream = find_column_dependencies(target_column, cll.parent_map, cll.child_map)
-            relevant_columns.add(target_column)
-            relevant_columns.update(downstream)
-
-        nodes, columns = filter_lineage_vertices(cll.nodes, cll.columns, relevant_columns)
-        p_map, c_map = filter_dependency_maps(cll.parent_map, cll.child_map, relevant_columns)
-
-        return CllData(nodes=nodes, columns=columns, parent_map=p_map, child_map=c_map)
-
-    @staticmethod
-    def _merge_cll_data(base: CllData, target: CllData) -> CllData:
-        merged_nodes = {**base.nodes, **target.nodes}
-        merged_columns = {**base.columns, **target.columns}
-
-        merged_parent_map = {}
-        merged_keys = set(base.parent_map.keys()).union(set(target.parent_map.keys()))
-        for key in merged_keys:
-            merged_parent_map[key] = base.parent_map.get(key, set()).union(target.parent_map.get(key, set()))
-
-        merged_child_map = {}
-        merged_keys = set(base.child_map.keys()).union(set(target.child_map.keys()))
-        for key in merged_keys:
-            merged_child_map[key] = base.child_map.get(key, set()).union(target.child_map.get(key, set()))
-
-        return CllData(
-            nodes=merged_nodes, columns=merged_columns, parent_map=merged_parent_map, child_map=merged_child_map
-        )
-
     def build_name_to_unique_id_index(self) -> Dict[str, str]:
         name_to_unique_id = {}
         curr_manifest = self.get_manifest(base=False)
@@ -1404,13 +1488,18 @@ class DbtAdapter(BaseAdapter):
                 self.curr_manifest = load_manifest(path=refresh_file_path)
                 self.manifest = as_manifest(self.curr_manifest)
                 self.get_cll_cached.cache_clear()
+                self.get_change_analysis_cached.cache_clear()
             elif refresh_file_path.endswith("catalog.json"):
                 self.curr_catalog = load_catalog(path=refresh_file_path)
+                self.get_cll_cached.cache_clear()
+                self.get_change_analysis_cached.cache_clear()
         elif self.base_path and target_type == os.path.basename(self.base_path):
             if refresh_file_path.endswith("manifest.json"):
                 self.base_manifest = load_manifest(path=refresh_file_path)
+                self.get_change_analysis_cached.cache_clear()
             elif refresh_file_path.endswith("catalog.json"):
                 self.base_catalog = load_catalog(path=refresh_file_path)
+                self.get_change_analysis_cached.cache_clear()
 
     def create_relation(self, model, base=False):
         node = self.find_node_by_name(model, base)
@@ -1446,6 +1535,15 @@ class DbtAdapter(BaseAdapter):
 
         specs = [_parse_difference(select_list, exclude_list)]
 
+        # If packages is not provided, use the project name from manifest metadata as default
+        if packages is None:
+            if (
+                self.manifest.metadata
+                and hasattr(self.manifest.metadata, "project_name")
+                and self.manifest.metadata.project_name
+            ):
+                packages = [self.manifest.metadata.project_name]
+
         if packages is not None:
             package_spec = SelectionUnion([_parse_difference([f"package:{p}"], None) for p in packages])
             specs.append(package_spec)
@@ -1522,7 +1620,7 @@ class DbtAdapter(BaseAdapter):
             if not os.path.isfile(path):
                 return None
 
-            with open(path, "r") as f:
+            with open(path, "r", encoding="utf-8") as f:
                 json_content = f.read()
                 return json.loads(json_content)