recce-nightly 0.62.0.20250417__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/adapter/dbt_adapter/__init__.py

@@ -8,14 +8,34 @@ from dataclasses import dataclass, fields
 from errno import ENOENT
 from functools import lru_cache
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Tuple, Iterator, Any, Set, Union, Literal, Type
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
 
 from recce.event import log_performance
 from recce.exceptions import RecceException
-from recce.util.cll import cll, CLLPerformanceTracking
-from recce.util.lineage import find_upstream, find_downstream
+from recce.util.cll import CLLPerformanceTracking, cll
+from recce.util.lineage import (
+    build_column_key,
+    filter_dependency_maps,
+    find_downstream,
+    find_upstream,
+)
+from recce.util.perf_tracking import LineagePerfTracker
+from recce.util.startup_perf import track_timing
+
 from ...tasks.profile import ProfileTask
-from ...util.breaking import parse_change_category, BreakingPerformanceTracking
+from ...util.breaking import BreakingPerformanceTracking, parse_change_category
 
 try:
     import agate
@@ -30,11 +50,30 @@ from watchdog.observers import Observer
 
 from recce.adapter.base import BaseAdapter
 from recce.state import ArtifactsRoot
-from .dbt_version import DbtVersion
+
 from ...models import RunType
-from ...models.types import LineageDiff, NodeDiff, NodeChange
-from ...tasks import Task, QueryTask, QueryBaseTask, QueryDiffTask, ValueDiffTask, ValueDiffDetailTask, ProfileDiffTask, \
-    RowCountTask, RowCountDiffTask, TopKDiffTask, HistogramDiffTask
+from ...models.types import (
+    CllColumn,
+    CllData,
+    CllNode,
+    LineageDiff,
+    NodeChange,
+    NodeDiff,
+)
+from ...tasks import (
+    HistogramDiffTask,
+    ProfileDiffTask,
+    QueryBaseTask,
+    QueryDiffTask,
+    QueryTask,
+    RowCountDiffTask,
+    RowCountTask,
+    Task,
+    TopKDiffTask,
+    ValueDiffDetailTask,
+    ValueDiffTask,
+)
+from .dbt_version import DbtVersion
 
 dbt_supported_registry: Dict[RunType, Type[Task]] = {
     RunType.QUERY: QueryTask,
@@ -56,7 +95,7 @@ get_adapter_orig = dbt.adapters.factory.get_adapter
 
 
 def get_adapter(config):
-    if hasattr(config, 'adapter'):
+    if hasattr(config, "adapter"):
         return config.adapter
     else:
         return get_adapter_orig(config)
@@ -69,7 +108,12 @@ from dbt.adapters.base import Column  # noqa: E402
 from dbt.adapters.factory import get_adapter_class_by_name  # noqa: E402
 from dbt.adapters.sql import SQLAdapter  # noqa: E402
 from dbt.config.runtime import RuntimeConfig  # noqa: E402
-from dbt.contracts.graph.manifest import Manifest, WritableManifest, MacroManifest  # noqa: E402
+from dbt.contracts.graph.manifest import (  # noqa: E402
+    MacroManifest,
+    Manifest,
+    ManifestMetadata,
+    WritableManifest,
+)
 from dbt.contracts.graph.nodes import ManifestNode  # noqa: E402
 from dbt.contracts.results import CatalogArtifact  # noqa: E402
 from dbt.flags import set_from_args  # noqa: E402
@@ -78,7 +122,7 @@ from dbt.parser.sql import SqlBlockParser  # noqa: E402
 
 dbt_version = DbtVersion()
 
-if dbt_version < 'v1.8':
+if dbt_version < "v1.8":
     from dbt.contracts.connection import Connection
 else:
     from dbt.adapters.contracts.connection import Connection
@@ -86,19 +130,22 @@ else:
 
 @contextmanager
 def silence_no_nodes_warning():
-    if dbt_version >= 'v1.8':
+    if dbt_version >= "v1.8":
         from dbt.events.types import NoNodesForSelectionCriteria
         from dbt_common.events.functions import WARN_ERROR_OPTIONS
+
         WARN_ERROR_OPTIONS.silence.append(NoNodesForSelectionCriteria.__name__)
     try:
         yield
     finally:
-        if dbt_version >= 'v1.8':
+        if dbt_version >= "v1.8":
             from dbt_common.events.functions import WARN_ERROR_OPTIONS
+
             WARN_ERROR_OPTIONS.silence.pop()
 
 
-logger = logging.getLogger('uvicorn')
+logger = logging.getLogger("uvicorn")
+MIN_DBT_NODE_COMPOSITION = 3
 
 
 class ArtifactsEventHandler(FileSystemEventHandler):
@@ -147,24 +194,29 @@ class EnvironmentEventHandler(FileSystemEventHandler):
 
 
 def merge_tables(tables: List[agate.Table]) -> agate.Table:
-    if dbt_version < 'v1.8':
+    if dbt_version < "v1.8":
         from dbt.clients.agate_helper import merge_tables
+
         return merge_tables(tables)
     else:
         from dbt_common.clients.agate_helper import merge_tables
+
         return merge_tables(tables)
 
 
 def as_manifest(m: WritableManifest) -> Manifest:
-    if dbt_version < 'v1.8':
+    if dbt_version < "v1.8":
         data = m.__dict__
         all_fields = set([x.name for x in fields(Manifest)])
         new_data = {k: v for k, v in data.items() if k in all_fields}
         return Manifest(**new_data)
     else:
-        return Manifest.from_writable_manifest(m)
+        result = Manifest.from_writable_manifest(m)
+        result.metadata = ManifestMetadata(**m.metadata.__dict__)
+        return result
 
 
+@track_timing(record_size=True)
 def load_manifest(path: str = None, data: dict = None):
     if path is not None:
         if not os.path.isfile(path):
@@ -174,6 +226,7 @@ def load_manifest(path: str = None, data: dict = None):
         return WritableManifest.upgrade_schema_version(data)
 
 
+@track_timing(record_size=True)
 def load_catalog(path: str = None, data: dict = None):
     if path is not None:
         if not os.path.isfile(path):
@@ -184,12 +237,13 @@ def load_catalog(path: str = None, data: dict = None):
 
 
 def previous_state(state_path: Path, target_path: Path, project_root: Path) -> PreviousState:
-    if dbt_version < 'v1.5.2':
+    if dbt_version < "v1.5.2":
         return PreviousState(state_path, target_path)
     else:
         try:
             # Overwrite the level_tag method temporarily to avoid the warning message
-            from dbt.events.types import WarnStateTargetEqual, EventLevel
+            from dbt.events.types import EventLevel, WarnStateTargetEqual
+
             original_level_tag_func = WarnStateTargetEqual.level_tag
             WarnStateTargetEqual.level_tag = lambda x: EventLevel.DEBUG
         except ImportError:
@@ -209,12 +263,12 @@ def previous_state(state_path: Path, target_path: Path, project_root: Path) -> P
 def default_profiles_dir():
     # Precedence: DBT_PROFILES_DIR > current working directory > ~/.dbt/
     # https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles#advanced-customizing-a-profile-directory
-    if os.getenv('DBT_PROFILES_DIR'):
-        return os.getenv('DBT_PROFILES_DIR')
-    elif os.path.exists(os.path.join(os.getcwd(), 'profiles.yml')):
+    if os.getenv("DBT_PROFILES_DIR"):
+        return os.getenv("DBT_PROFILES_DIR")
+    elif os.path.exists(os.path.join(os.getcwd(), "profiles.yml")):
         return os.getcwd()
     else:
-        return os.path.expanduser('~/.dbt/')
+        return os.path.expanduser("~/.dbt/")
 
 
 @dataclass()
@@ -222,15 +276,16 @@ class DbtArgs:
     """
     Used for RuntimeConfig.from_args
     """
-    threads: Optional[int] = 1,
-    target: Optional[str] = None,
-    profiles_dir: Optional[str] = None,
-    project_dir: Optional[str] = None,
-    profile: Optional[str] = None,
-    target_path: Optional[str] = None,
+
+    threads: Optional[int] = (1,)
+    target: Optional[str] = (None,)
+    profiles_dir: Optional[str] = (None,)
+    project_dir: Optional[str] = (None,)
+    profile: Optional[str] = (None,)
+    target_path: Optional[str] = (None,)
     project_only_flags: Optional[Dict[str, Any]] = None
     which: Optional[str] = None
-    state_modified_compare_more_unrendered_values: Optional[bool] = False  # new flag added since dbt v1.9
+    state_modified_compare_more_unrendered_values: Optional[bool] = True  # new flag added since dbt v1.9
 
 
 @dataclass
@@ -258,32 +313,18 @@ class DbtAdapter(BaseAdapter):
 
     def support_tasks(self):
         support_map = {run_type.value: True for run_type in dbt_supported_registry}
-        supported_dbt_packages = set([package.package_name for package in self.manifest.macros.values()])
-
-        if 'dbt_profiler' not in supported_dbt_packages:
-            support_map[RunType.PROFILE_DIFF.value] = False
-            support_map[RunType.PROFILE.value] = False
-
-        if 'audit_helper' not in supported_dbt_packages:
-            support_map[RunType.VALUE_DIFF.value] = False
-            support_map[RunType.VALUE_DIFF_DETAIL.value] = False
-            support_map['query_diff_with_primary_key'] = False
 
         return support_map
 
     @classmethod
-    def load(cls,
-             no_artifacts=False,
-             review=False,
-             **kwargs):
-
-        target = kwargs.get('target')
-        target_path = kwargs.get('target_path', 'target')
-        target_base_path = kwargs.get('target_base_path', 'target-base')
+    def load(cls, no_artifacts=False, review=False, **kwargs):
+        target = kwargs.get("target")
+        target_path = kwargs.get("target_path", "target")
+        target_base_path = kwargs.get("target_base_path", "target-base")
 
-        profile_name = kwargs.get('profile')
-        project_dir = kwargs.get('project_dir')
-        profiles_dir = kwargs.get('profiles_dir')
+        profile_name = kwargs.get("profile")
+        project_dir = kwargs.get("project_dir")
+        profiles_dir = kwargs.get("profiles_dir")
 
         if profiles_dir is None:
             profiles_dir = default_profiles_dir()
@@ -297,21 +338,25 @@ class DbtAdapter(BaseAdapter):
             profiles_dir=profiles_dir,
             profile=profile_name,
             project_only_flags={},
-            which='list'
+            which="list",
         )
         set_from_args(args, args)
 
         from dbt.exceptions import DbtProjectError
+
         try:
             # adapter
-            if dbt_version < 'v1.8':
+            if dbt_version < "v1.8":
                 runtime_config = RuntimeConfig.from_args(args)
                 adapter_name = runtime_config.credentials.type
                 adapter_cls = get_adapter_class_by_name(adapter_name)
                 adapter: SQLAdapter = adapter_cls(runtime_config)
             else:
-                from dbt_common.context import set_invocation_context, get_invocation_context
                 from dbt.mp_context import get_mp_context
+                from dbt_common.context import (
+                    get_invocation_context,
+                    set_invocation_context,
+                )
 
                 set_invocation_context({})
                 get_invocation_context()._env = dict(os.environ)
@@ -319,6 +364,9 @@ class DbtAdapter(BaseAdapter):
                 adapter_name = runtime_config.credentials.type
                 adapter_cls = get_adapter_class_by_name(adapter_name)
                 adapter: SQLAdapter = adapter_cls(runtime_config, get_mp_context())
+                from dbt.adapters.factory import FACTORY
+
+                FACTORY.adapters[adapter_name] = adapter
 
             adapter.connections.set_connection_name()
             runtime_config.adapter = adapter
@@ -327,7 +375,7 @@ class DbtAdapter(BaseAdapter):
                 runtime_config=runtime_config,
                 adapter=adapter,
                 review_mode=review,
-                base_path=target_base_path
+                base_path=target_base_path,
             )
         except DbtProjectError as e:
             raise e
@@ -348,27 +396,26 @@ class DbtAdapter(BaseAdapter):
 
     def get_columns(self, model: str, base=False) -> List[Column]:
         relation = self.create_relation(model, base)
-        get_columns_macro = 'get_columns_in_relation'
-        if self.adapter.connections.TYPE == 'databricks':
-            get_columns_macro = 'get_columns_comments'
+        get_columns_macro = "get_columns_in_relation"
+        if self.adapter.connections.TYPE == "databricks":
+            get_columns_macro = "get_columns_comments"
 
-        if dbt_version < 'v1.8':
+        if dbt_version < "v1.8":
             columns = self.adapter.execute_macro(
-                get_columns_macro,
-                kwargs={"relation": relation},
-                manifest=self.manifest)
+                get_columns_macro, kwargs={"relation": relation}, manifest=self.manifest
+            )
         else:
             from dbt.context.providers import generate_runtime_macro_context
+
             macro_manifest = MacroManifest(self.manifest.macros)
             self.adapter.set_macro_resolver(macro_manifest)
             self.adapter.set_macro_context_generator(generate_runtime_macro_context)
-            columns = self.adapter.execute_macro(
-                get_columns_macro,
-                kwargs={"relation": relation})
+            columns = self.adapter.execute_macro(get_columns_macro, kwargs={"relation": relation})
 
-        if self.adapter.connections.TYPE == 'databricks':
+        if self.adapter.connections.TYPE == "databricks":
             # reference: get_columns_in_relation (dbt/adapters/databricks/impl.py)
-            from dbt.adapters.databricks import DatabricksColumn
+            from dbt.adapters.databricks.column import DatabricksColumn
+
             rows = columns
             columns = []
             for row in rows:
@@ -376,7 +423,9 @@ class DbtAdapter(BaseAdapter):
                     break
                 columns.append(
                     DatabricksColumn(
-                        column=row["col_name"], dtype=row["data_type"], comment=row["comment"]
+                        column=row["col_name"],
+                        dtype=row["data_type"],
+                        comment=row["comment"],
                     )
                 )
             return columns
@@ -387,29 +436,29 @@ class DbtAdapter(BaseAdapter):
         manifest = self.curr_manifest if base is False else self.base_manifest
         manifest_dict = manifest.to_dict()
 
-        node = manifest_dict['nodes'].get(model_id)
+        node = manifest_dict["nodes"].get(model_id)
         if node is None:
             return {}
 
-        node_name = node['name']
-        with self.adapter.connection_named('model'):
+        node_name = node["name"]
+        with self.adapter.connection_named("model"):
             columns = [column for column in self.get_columns(node_name, base=base)]
 
-        child_map: List[str] = manifest_dict['child_map'][model_id]
+        child_map: List[str] = manifest_dict["child_map"][model_id]
         cols_not_null = []
         cols_unique = []
 
         for child in child_map:
-            comps = child.split('.')
+            comps = child.split(".")
             child_type = comps[0]
             child_name = comps[2]
 
-            not_null_prefix = f'not_null_{node_name}_'
-            if child_type == 'test' and child_name.startswith(not_null_prefix):
-                cols_not_null.append(child_name[len(not_null_prefix):])
-            unique_prefix = f'unique_{node_name}_'
-            if child_type == 'test' and child_name.startswith(unique_prefix):
-                cols_unique.append(child_name[len(unique_prefix):])
+            not_null_prefix = f"not_null_{node_name}_"
+            if child_type == "test" and child_name.startswith(not_null_prefix):
+                cols_not_null.append(child_name[len(not_null_prefix) :])
+            unique_prefix = f"unique_{node_name}_"
+            if child_type == "test" and child_name.startswith(unique_prefix):
+                cols_unique.append(child_name[len(unique_prefix) :])
 
         columns_info = {}
         primary_key = None
@@ -417,25 +466,26 @@ class DbtAdapter(BaseAdapter):
             col_name = c.column
             col = dict(name=col_name, type=c.dtype)
             if col_name in cols_not_null:
-                col['not_null'] = True
+                col["not_null"] = True
             if col_name in cols_unique:
-                col['unique'] = True
+                col["unique"] = True
                 if not primary_key:
                     primary_key = col_name
             columns_info[col_name] = col
 
         result = dict(columns=columns_info)
         if primary_key:
-            result['primary_key'] = primary_key
+            result["primary_key"] = primary_key
 
         return result
 
+    @track_timing("artifact_load")
     def load_artifacts(self):
         """
         Load the artifacts from the 'target' and 'target-base' directory
         """
         if self.runtime_config is None:
-            raise Exception('Cannot find the dbt project configuration')
+            raise Exception("Cannot find the dbt project configuration")
 
         project_root = self.runtime_config.project_root
         target_path = self.runtime_config.target_path
@@ -444,17 +494,21 @@ class DbtAdapter(BaseAdapter):
         self.base_path = os.path.join(project_root, target_base_path)
 
         # load the artifacts
-        path = os.path.join(project_root, target_path, 'manifest.json')
-        curr_manifest = load_manifest(path=path)
+        path = os.path.join(project_root, target_path, "manifest.json")
+        curr_manifest = load_manifest(path=path, timing_name="curr_manifest")
         if curr_manifest is None:
             raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
-        path = os.path.join(project_root, target_base_path, 'manifest.json')
-        base_manifest = load_manifest(path=path)
+        path = os.path.join(project_root, target_base_path, "manifest.json")
+        base_manifest = load_manifest(path=path, timing_name="base_manifest")
         if base_manifest is None:
             raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
 
-        curr_catalog = load_catalog(path=os.path.join(project_root, target_path, 'catalog.json'))
-        base_catalog = load_catalog(path=os.path.join(project_root, target_base_path, 'catalog.json'))
+        curr_catalog = load_catalog(
+            path=os.path.join(project_root, target_path, "catalog.json"), timing_name="curr_catalog"
+        )
+        base_catalog = load_catalog(
+            path=os.path.join(project_root, target_base_path, "catalog.json"), timing_name="base_catalog"
+        )
 
         # set the value if all the artifacts are loaded successfully
         self.curr_manifest = curr_manifest
@@ -472,22 +526,21 @@ class DbtAdapter(BaseAdapter):
 
         # set the file paths to watch
         self.artifacts_files = [
-            os.path.join(project_root, target_path, 'manifest.json'),
-            os.path.join(project_root, target_path, 'catalog.json'),
-            os.path.join(project_root, target_base_path, 'manifest.json'),
-            os.path.join(project_root, target_base_path, 'catalog.json'),
+            os.path.join(project_root, target_path, "manifest.json"),
+            os.path.join(project_root, target_path, "catalog.json"),
+            os.path.join(project_root, target_base_path, "manifest.json"),
+            os.path.join(project_root, target_base_path, "catalog.json"),
         ]
 
     def is_python_model(self, node_id: str, base: Optional[bool] = False):
         manifest = self.curr_manifest if base is False else self.base_manifest
         model = manifest.nodes.get(node_id)
-        if hasattr(model, 'language'):
-            return model.language == 'python'
+        if hasattr(model, "language"):
+            return model.language == "python"
 
         return False
 
     def find_node_by_name(self, node_name, base=False) -> Optional[ManifestNode]:
-
         manifest = self.curr_manifest if base is False else self.base_manifest
 
         for key, node in manifest.nodes.items():
@@ -497,22 +550,22 @@ class DbtAdapter(BaseAdapter):
         return None
 
     def get_node_name_by_id(self, unique_id):
-        if unique_id.startswith('source.'):
+        if unique_id.startswith("source."):
             if unique_id in self.curr_manifest.sources:
                 return self.curr_manifest.sources[unique_id].name
             elif unique_id in self.base_manifest.sources:
                 return self.base_manifest.sources[unique_id].name
-        elif unique_id.startswith('metric.'):
+        elif unique_id.startswith("metric."):
             if unique_id in self.curr_manifest.metrics:
                 return self.curr_manifest.metrics[unique_id].name
             elif unique_id in self.base_manifest.metrics:
                 return self.base_manifest.metrics[unique_id].name
-        elif unique_id.startswith('exposure.'):
+        elif unique_id.startswith("exposure."):
             if unique_id in self.curr_manifest.exposures:
                 return self.curr_manifest.exposures[unique_id].name
             elif unique_id in self.base_manifest.exposures:
                 return self.base_manifest.exposures[unique_id].name
-        elif unique_id.startswith('semantic_model.'):
+        elif unique_id.startswith("semantic_model."):
             if unique_id in self.curr_manifest.semantic_models:
                 return self.curr_manifest.semantic_models[unique_id].name
             elif unique_id in self.base_manifest.semantic_models:
@@ -527,14 +580,24 @@ class DbtAdapter(BaseAdapter):
     def get_manifest(self, base: bool):
         return self.curr_manifest if base is False else self.base_manifest
 
-    def generate_sql(self, sql_template: str, base: bool = False, context=None, provided_manifest=None):
+    def generate_sql(
+        self,
+        sql_template: str,
+        base: bool = False,
+        context=None,
+        provided_manifest=None,
+    ):
         if context is None:
             context = {}
         manifest = provided_manifest if provided_manifest is not None else as_manifest(self.get_manifest(base))
         parser = SqlBlockParser(self.runtime_config, manifest, self.runtime_config)
 
-        if dbt_version >= dbt_version.parse('v1.8'):
-            from dbt_common.context import set_invocation_context, get_invocation_context
+        if dbt_version >= dbt_version.parse("v1.8"):
+            from dbt_common.context import (
+                get_invocation_context,
+                set_invocation_context,
+            )
+
             set_invocation_context({})
             get_invocation_context()._env = dict(os.environ)
 
@@ -542,21 +605,35 @@ class DbtAdapter(BaseAdapter):
         node = parser.parse_remote(sql_template, node_id)
         process_node(self.runtime_config, manifest, node)
 
-        if dbt_version < dbt_version.parse('v1.8'):
+        if dbt_version < dbt_version.parse("v1.8"):
             compiler = self.adapter.get_compiler()
             compiler.compile_node(node, manifest, context)
             return node.compiled_code
         else:
-            from dbt.context.providers import generate_runtime_model_context
             from dbt.clients import jinja
+            from dbt.context.providers import (
+                generate_runtime_macro_context,
+                generate_runtime_model_context,
+            )
+
+            # Set up macro resolver for dbt >= 1.8
+            macro_manifest = MacroManifest(manifest.macros)
+            self.adapter.set_macro_resolver(macro_manifest)
+            self.adapter.set_macro_context_generator(generate_runtime_macro_context)
+
             jinja_ctx = generate_runtime_model_context(node, self.runtime_config, manifest)
             jinja_ctx.update(context)
             compiled_code = jinja.get_rendered(sql_template, jinja_ctx, node)
             return compiled_code
 
-    def execute(self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None) -> Tuple[
-        any, agate.Table]:
-        if dbt_version < dbt_version.parse('v1.6'):
+    def execute(
+        self,
+        sql: str,
+        auto_begin: bool = False,
+        fetch: bool = False,
+        limit: Optional[int] = None,
+    ) -> Tuple[any, agate.Table]:
+        if dbt_version < dbt_version.parse("v1.6"):
             return self.adapter.execute(sql, auto_begin=auto_begin, fetch=fetch)
 
         return self.adapter.execute(sql, auto_begin=auto_begin, fetch=fetch, limit=limit)
@@ -567,7 +644,7 @@ class DbtAdapter(BaseAdapter):
 
         node_ids = nodes.keys()
         parent_map = {}
-        for k, parents in manifest_dict['parent_map'].items():
+        for k, parents in manifest_dict["parent_map"].items():
             if k not in node_ids:
                 continue
             parent_map[k] = [parent for parent in parents if parent in node_ids]
@@ -578,8 +655,8 @@ class DbtAdapter(BaseAdapter):
         manifest = self.curr_manifest if base is False else self.base_manifest
         manifest_dict = manifest.to_dict()
 
-        if node_id in manifest_dict['parent_map']:
-            return manifest_dict['parent_map'][node_id]
+        if node_id in manifest_dict["parent_map"]:
+            return manifest_dict["parent_map"][node_id]
 
     def get_lineage(self, base: Optional[bool] = False):
         manifest = self.curr_manifest if base is False else self.base_manifest
@@ -588,19 +665,21 @@ class DbtAdapter(BaseAdapter):
         return self.get_lineage_cached(base, cache_key)
 
     def get_lineage_diff(self) -> LineageDiff:
-        cache_key = hash((
-            id(self.base_manifest),
-            id(self.base_catalog),
-            id(self.curr_manifest),
-            id(self.curr_catalog),
-        ))
+        cache_key = hash(
+            (
+                id(self.base_manifest),
+                id(self.base_catalog),
+                id(self.curr_manifest),
+                id(self.curr_catalog),
+            )
+        )
         return self._get_lineage_diff_cached(cache_key)
 
     @lru_cache(maxsize=2)
     def get_lineage_cached(self, base: Optional[bool] = False, cache_key=0):
         if base is False:
-            cll_tracker = CLLPerformanceTracking()
-            cll_tracker.start_lineage()
+            perf_tracker = LineagePerfTracker()
+            perf_tracker.start_lineage()
 
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
@@ -612,48 +691,48 @@ class DbtAdapter(BaseAdapter):
 
         nodes = {}
 
-        for node in manifest_dict['nodes'].values():
-            unique_id = node['unique_id']
-            resource_type = node['resource_type']
+        for node in manifest_dict["nodes"].values():
+            unique_id = node["unique_id"]
+            resource_type = node["resource_type"]
 
-            if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
+            if resource_type not in ["model", "seed", "exposure", "snapshot"]:
                 continue
 
             nodes[unique_id] = {
-                'id': node['unique_id'],
-                'name': node['name'],
-                'resource_type': node['resource_type'],
-                'package_name': node['package_name'],
-                'schema': node['schema'],
-                'config': node['config'],
-                'checksum': node['checksum'],
-                'raw_code': node['raw_code'],
+                "id": node["unique_id"],
+                "name": node["name"],
+                "resource_type": node["resource_type"],
+                "package_name": node["package_name"],
+                "schema": node["schema"],
+                "config": node["config"],
+                "checksum": node["checksum"],
+                "raw_code": node["raw_code"],
             }
 
             # List of <type>.<package_name>.<node_name>.<hash>
             # model.jaffle_shop.customer_segments
             # test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d
             # test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1
-            child_map: List[str] = manifest_dict['child_map'][unique_id]
+            child_map: List[str] = manifest_dict["child_map"][unique_id]
             cols_not_null = []
             cols_unique = []
 
             for child in child_map:
-                node_name = node['name']
-                comps = child.split('.')
-                if len(comps) < 2:
+                node_name = node["name"]
+                comps = child.split(".")
+                if len(comps) < MIN_DBT_NODE_COMPOSITION:
                     # only happens in unittest
                     continue
 
                 child_type = comps[0]
                 child_name = comps[2]
 
-                not_null_prefix = f'not_null_{node_name}_'
-                if child_type == 'test' and child_name.startswith(not_null_prefix):
-                    cols_not_null.append(child_name[len(not_null_prefix):])
-                unique_prefix = f'unique_{node_name}_'
-                if child_type == 'test' and child_name.startswith(unique_prefix):
-                    cols_unique.append(child_name[len(unique_prefix):])
+                not_null_prefix = f"not_null_{node_name}_"
+                if child_type == "test" and child_name.startswith(not_null_prefix):
+                    cols_not_null.append(child_name[len(not_null_prefix) :])
+                unique_prefix = f"unique_{node_name}_"
+                if child_type == "test" and child_name.startswith(unique_prefix):
+                    cols_unique.append(child_name[len(unique_prefix) :])
 
             if catalog is not None and unique_id in catalog.nodes:
                 columns = {}
@@ -661,70 +740,68 @@ class DbtAdapter(BaseAdapter):
                 for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
                     col = dict(name=col_name, type=col_metadata.type)
                     if col_name in cols_not_null:
-                        col['not_null'] = True
+                        col["not_null"] = True
                     if col_name in cols_unique:
-                        col['unique'] = True
+                        col["unique"] = True
                         if not primary_key:
                             primary_key = col_name
                     columns[col_name] = col
-                nodes[unique_id]['columns'] = columns
+                nodes[unique_id]["columns"] = columns
                 if primary_key:
-                    nodes[unique_id]['primary_key'] = primary_key
+                    nodes[unique_id]["primary_key"] = primary_key
 
-        for source in manifest_dict['sources'].values():
-            unique_id = source['unique_id']
+        for source in manifest_dict["sources"].values():
+            unique_id = source["unique_id"]
 
             nodes[unique_id] = {
-                'id': source['unique_id'],
-                'name': source['name'],
-                'resource_type': source['resource_type'],
-                'package_name': source['package_name'],
-                'config': source['config'],
+                "id": source["unique_id"],
+                "name": source["name"],
+                "source_name": source["source_name"],
+                "resource_type": source["resource_type"],
+                "package_name": source["package_name"],
+                "config": source["config"],
             }
 
             if catalog is not None and unique_id in catalog.sources:
-                nodes[unique_id]['columns'] = {
-                    col_name: {
-                        'name': col_name,
-                        'type': col_metadata.type
-                    }
+                nodes[unique_id]["columns"] = {
+                    col_name: {"name": col_name, "type": col_metadata.type}
                     for col_name, col_metadata in catalog.sources[unique_id].columns.items()
                 }
 
-        for exposure in manifest_dict['exposures'].values():
-            nodes[exposure['unique_id']] = {
-                'id': exposure['unique_id'],
-                'name': exposure['name'],
-                'resource_type': exposure['resource_type'],
-                'package_name': exposure['package_name'],
-                'config': exposure['config'],
+        for exposure in manifest_dict["exposures"].values():
+            nodes[exposure["unique_id"]] = {
+                "id": exposure["unique_id"],
+                "name": exposure["name"],
+                "resource_type": exposure["resource_type"],
+                "package_name": exposure["package_name"],
+                "config": exposure["config"],
             }
-        for metric in manifest_dict['metrics'].values():
-            nodes[metric['unique_id']] = {
-                'id': metric['unique_id'],
-                'name': metric['name'],
-                'resource_type': metric['resource_type'],
-                'package_name': metric['package_name'],
-                'config': metric['config'],
+        for metric in manifest_dict["metrics"].values():
+            nodes[metric["unique_id"]] = {
+                "id": metric["unique_id"],
+                "name": metric["name"],
+                "resource_type": metric["resource_type"],
+                "package_name": metric["package_name"],
+                "config": metric["config"],
             }
 
-        if 'semantic_models' in manifest_dict:
-            for semantic_models in manifest_dict['semantic_models'].values():
-                nodes[semantic_models['unique_id']] = {
-                    'id': semantic_models['unique_id'],
-                    'name': semantic_models['name'],
-                    'resource_type': semantic_models['resource_type'],
-                    'package_name': semantic_models['package_name'],
-                    'config': semantic_models['config'],
+        if "semantic_models" in manifest_dict:
+            for semantic_models in manifest_dict["semantic_models"].values():
+                nodes[semantic_models["unique_id"]] = {
+                    "id": semantic_models["unique_id"],
+                    "name": semantic_models["name"],
+                    "resource_type": semantic_models["resource_type"],
+                    "package_name": semantic_models["package_name"],
+                    "config": semantic_models["config"],
                 }
 
         parent_map = self.build_parent_map(nodes, base)
 
         if base is False:
-            cll_tracker.end_lineage()
-            cll_tracker.set_total_nodes(len(nodes))
-            log_performance('model lineage', cll_tracker.to_dict())
-            cll_tracker.reset()
+            perf_tracker.end_lineage()
+            perf_tracker.set_total_nodes(len(nodes))
+            log_performance("model lineage", perf_tracker.to_dict())
+            perf_tracker.reset()
 
         return dict(
             parent_map=parent_map,
@@ -737,18 +814,43 @@ class DbtAdapter(BaseAdapter):
     def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
         base = self.get_lineage(base=True)
         current = self.get_lineage(base=False)
-        keys = {
-            *base.get('nodes', {}).keys(),
-            *current.get('nodes', {}).keys()
-        }
 
-        # Start to diff
-        perf_tracking = BreakingPerformanceTracking()
-        perf_tracking.start_lineage_diff()
+        modified_nodes = self.select_nodes(select="state:modified")
+        diff = {}
+        for node_id in modified_nodes:
+            base_node = base.get("nodes", {}).get(node_id)
+            curr_node = current.get("nodes", {}).get(node_id)
+            if base_node and curr_node:
+                diff[node_id] = NodeDiff(change_status="modified")
+            elif base_node:
+                diff[node_id] = NodeDiff(change_status="removed")
+            elif curr_node:
+                diff[node_id] = NodeDiff(change_status="added")
+
+        return LineageDiff(
+            base=base,
+            current=current,
+            diff=diff,
+        )
+
+    @lru_cache(maxsize=128)
+    def get_change_analysis_cached(self, node_id: str):
+        breaking_perf_tracker = BreakingPerformanceTracking()
+        lineage_diff = self.get_lineage_diff()
+        diff = lineage_diff.diff
+
+        if node_id not in diff or diff[node_id].change_status != "modified":
+            return diff.get(node_id)
+
+        breaking_perf_tracker.increment_modified_nodes()
+        breaking_perf_tracker.start_lineage_diff()
+
+        base = lineage_diff.base
+        current = lineage_diff.current
 
         base_manifest = as_manifest(self.get_manifest(True))
         curr_manifest = as_manifest(self.get_manifest(False))
-        perf_tracking.record_checkpoint('manifest')
+        breaking_perf_tracker.record_checkpoint("manifest")
 
         def ref_func(*args):
             if len(args) == 1:
@@ -760,7 +862,7 @@ class DbtAdapter(BaseAdapter):
             return node
 
         def source_func(source_name, table_name):
-            source_name = source_name.replace('-', '_')
+            source_name = source_name.replace("-", "_")
             return f"__{source_name}__{table_name}"
 
         jinja_context = dict(
@@ -768,284 +870,534 @@ class DbtAdapter(BaseAdapter):
             source=source_func,
         )
 
-        # for each node, compare the base and current lineage
-        diff = {}
-        for key in keys:
-            base_node = base.get('nodes', {}).get(key)
-            curr_node = current.get('nodes', {}).get(key)
-            if base_node and curr_node:
-                base_checksum = base_node.get('checksum', {}).get('checksum')
-                curr_checksum = curr_node.get('checksum', {}).get('checksum')
-                if base_checksum is None or curr_checksum is None or base_checksum == curr_checksum:
-                    continue
-
-                if curr_node.get('resource_type') == 'model':
-                    try:
-                        perf_tracking.increment_modified_nodes()
-
-                        def _get_schema(lineage):
-                            schema = {}
-                            nodes = lineage['nodes']
-                            parent_list = lineage['parent_map'].get(key, [])
-                            for parent_id in parent_list:
-                                parent_node = nodes.get(parent_id)
-                                if parent_node is None:
-                                    continue
-                                columns = parent_node.get('columns') or {}
-                                name = parent_node.get('name')
-                                if parent_node.get('resource_type') == 'source':
-                                    parts = parent_id.split('.')
-                                    source = parts[2]
-                                    table = parts[3]
-                                    source = source.replace('-', '_')
-                                    name = f"__{source}__{table}"
-                                schema[name] = {
-                                    name: column.get('type') for name, column in columns.items()
-                                }
-                            return schema
-
-                        base_sql = self.generate_sql(
-                            base_node.get('raw_code'),
-                            context=jinja_context,
-                            provided_manifest=base_manifest
-                        )
-                        curr_sql = self.generate_sql(
-                            curr_node.get('raw_code'),
-                            context=jinja_context,
-                            provided_manifest=curr_manifest
-                        )
-                        base_schema = _get_schema(base)
-                        curr_schema = _get_schema(current)
-                        dialect = self.adapter.connections.TYPE
-
-                        change = parse_change_category(
-                            base_sql,
-                            curr_sql,
-                            old_schema=base_schema,
-                            new_schema=curr_schema,
-                            dialect=dialect,
-                            perf_tracking=perf_tracking,
-                        )
-                    except Exception:
-                        change = NodeChange(category='unknown')
-
-                diff[key] = NodeDiff(change_status='modified', change=change)
-            elif base_node:
-                diff[key] = NodeDiff(change_status='removed')
-            elif curr_node:
-                diff[key] = NodeDiff(change_status='added')
-
-        perf_tracking.end_lineage_diff()
-        log_performance('model lineage diff', perf_tracking.to_dict())
+        base_node = base.get("nodes", {}).get(node_id)
+        curr_node = current.get("nodes", {}).get(node_id)
+        change = NodeChange(category="unknown")
+        if (
+            curr_node.get("resource_type") in ["model", "snapshot"]
+            and curr_node.get("raw_code") is not None
+            and base_node.get("raw_code") is not None
+        ):
+            try:
+
+                def _get_schema(lineage):
+                    schema = {}
+                    nodes = lineage["nodes"]
+                    parent_list = lineage["parent_map"].get(node_id, [])
+                    for parent_id in parent_list:
+                        parent_node = nodes.get(parent_id)
+                        if parent_node is None:
+                            continue
+                        columns = parent_node.get("columns") or {}
+                        name = parent_node.get("name")
+                        if parent_node.get("resource_type") == "source":
+                            parts = parent_id.split(".")
+                            source = parts[2]
+                            table = parts[3]
+                            source = source.replace("-", "_")
+                            name = f"__{source}__{table}"
+                        schema[name] = {name: column.get("type") for name, column in columns.items()}
+                    return schema
+
+                base_sql = self.generate_sql(
+                    base_node.get("raw_code"),
+                    context=jinja_context,
+                    provided_manifest=base_manifest,
+                )
+                curr_sql = self.generate_sql(
+                    curr_node.get("raw_code"),
+                    context=jinja_context,
+                    provided_manifest=curr_manifest,
+                )
+                base_schema = _get_schema(base)
+                curr_schema = _get_schema(current)
+                dialect = self.adapter.connections.TYPE
+                if curr_manifest.metadata.adapter_type is not None:
+                    dialect = curr_manifest.metadata.adapter_type
+
+                change = parse_change_category(
+                    base_sql,
+                    curr_sql,
+                    old_schema=base_schema,
+                    new_schema=curr_schema,
+                    dialect=dialect,
+                    perf_tracking=breaking_perf_tracker,
+                )
 
-        return LineageDiff(
-            base=base,
-            current=current,
-            diff=diff,
+                # Make sure that the case of the column names are the same
+                changed_columns = {
+                    column.lower(): change_status for column, change_status in (change.columns or {}).items()
+                }
+                changed_columns_names = set(changed_columns)
+                changed_columns_final = {}
+
+                base_columns = base_node.get("columns") or {}
+                curr_columns = curr_node.get("columns") or {}
+                columns_names = set(base_columns) | set(curr_columns)
+
+                for column_name in columns_names:
+                    if column_name.lower() in changed_columns_names:
+                        changed_columns_final[column_name] = changed_columns[column_name.lower()]
+
+                change.columns = changed_columns_final
+            except Exception:
+                # TODO: telemetry
+                pass
+
+        breaking_perf_tracker.end_lineage_diff()
+        log_performance("change analysis per node", breaking_perf_tracker.to_dict())
+        breaking_perf_tracker.reset()
+        node_diff = diff.get(node_id)
+        node_diff.change = change
+        return node_diff
+
+    def get_cll(
+        self,
+        node_id: Optional[str] = None,
+        column: Optional[str] = None,
+        change_analysis: Optional[bool] = False,
+        no_cll: Optional[bool] = False,
+        no_upstream: Optional[bool] = False,
+        no_downstream: Optional[bool] = False,
+        no_filter: Optional[bool] = False,
+    ) -> CllData:
+        cll_tracker = LineagePerfTracker()
+        cll_tracker.set_params(
+            has_node=node_id is not None,
+            has_column=column is not None,
+            change_analysis=change_analysis,
+            no_cll=no_cll,
+            no_upstream=no_upstream,
+            no_downstream=no_downstream,
         )
-
-    def get_cll_by_node_id(self, node_id: str, base: Optional[bool] = False):
-        cll_tracker = CLLPerformanceTracking()
         cll_tracker.start_column_lineage()
 
-        manifest = self.curr_manifest if base is False else self.base_manifest
+        manifest = self.curr_manifest
         manifest_dict = manifest.to_dict()
 
-        parent_ids = find_upstream(node_id, manifest_dict.get('parent_map'))
-        child_ids = find_downstream(node_id, manifest_dict.get('child_map'))
-        cll_node_ids = parent_ids.union(child_ids)
-        cll_node_ids.add(node_id)
+        # Find related model nodes
+        if node_id is not None:
+            cll_node_ids = {node_id}
+        else:
+            lineage_diff = self.get_lineage_diff()
+            cll_node_ids = set(lineage_diff.diff.keys())
+
+        cll_tracker.set_init_nodes(len(cll_node_ids))
 
-        node_manifest = self.get_lineage_nodes_metadata(base=base)
         nodes = {}
-        for node_id in cll_node_ids:
-            if node_id not in node_manifest:
-                continue
-            nodes[node_id] = self.get_cll_cached(node_id, base=base)
+        columns = {}
+        parent_map = {}
+        child_map = {}
+
+        if not no_upstream:
+            cll_node_ids = cll_node_ids.union(find_upstream(cll_node_ids, manifest_dict.get("parent_map")))
+        if not no_downstream:
+            cll_node_ids = cll_node_ids.union(find_downstream(cll_node_ids, manifest_dict.get("child_map")))
+
+        if not no_cll:
+            allowed_related_nodes = set()
+            for key in ["sources", "nodes", "exposures", "metrics"]:
+                attr = getattr(manifest, key)
+                allowed_related_nodes.update(set(attr.keys()))
+            if hasattr(manifest, "semantic_models"):
+                attr = getattr(manifest, "semantic_models")
+                allowed_related_nodes.update(set(attr.keys()))
+            for cll_node_id in cll_node_ids:
+                if cll_node_id not in allowed_related_nodes:
+                    continue
+                cll_data_one = deepcopy(self.get_cll_cached(cll_node_id, base=False))
+                cll_tracker.increment_cll_nodes()
+                if cll_data_one is None:
+                    continue
+
+                nodes[cll_node_id] = cll_data_one.nodes.get(cll_node_id)
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
+                if node_diff is not None:
+                    nodes[cll_node_id].change_status = node_diff.change_status
+                    if node_diff.change is not None:
+                        nodes[cll_node_id].change_category = node_diff.change.category
+                for c_id, c in cll_data_one.columns.items():
+                    columns[c_id] = c
+                    if node_diff is not None:
+                        if node_diff.change_status == "added":
+                            c.change_status = "added"
+                        elif node_diff.change_status == "removed":
+                            c.change_status = "removed"
+                        elif node_diff.change is not None and node_diff.change.columns is not None:
+                            column_diff = node_diff.change.columns.get(c.name)
+                            if column_diff:
+                                c.change_status = column_diff
+
+                for p_id, parents in cll_data_one.parent_map.items():
+                    parent_map[p_id] = parents
+        else:
+            for cll_node_id in cll_node_ids:
+                cll_node = None
+                cll_node_columns: Dict[str, CllColumn] = {}
+
+                if cll_node_id in manifest.sources:
+                    cll_node = CllNode.build_cll_node(manifest, "sources", cll_node_id)
+                    if self.curr_catalog and cll_node_id in self.curr_catalog.sources:
+                        cll_node_columns = {
+                            column.name: CllColumn(
+                                id=f"{cll_node_id}_{column.name}",
+                                table_id=cll_node_id,
+                                name=column.name,
+                                type=column.type,
+                            )
+                            for column in self.curr_catalog.sources[cll_node_id].columns.values()
+                        }
+                elif cll_node_id in manifest.nodes:
+                    cll_node = CllNode.build_cll_node(manifest, "nodes", cll_node_id)
+                    if self.curr_catalog and cll_node_id in self.curr_catalog.nodes:
+                        cll_node_columns = {
+                            column.name: CllColumn(
+                                id=f"{cll_node_id}_{column.name}",
+                                table_id=cll_node_id,
+                                name=column.name,
+                                type=column.type,
+                            )
+                            for column in self.curr_catalog.nodes[cll_node_id].columns.values()
+                        }
+                elif cll_node_id in manifest.exposures:
+                    cll_node = CllNode.build_cll_node(manifest, "exposures", cll_node_id)
+                elif hasattr(manifest, "semantic_models") and cll_node_id in manifest.semantic_models:
+                    cll_node = CllNode.build_cll_node(manifest, "semantic_models", cll_node_id)
+                elif cll_node_id in manifest.metrics:
+                    cll_node = CllNode.build_cll_node(manifest, "metrics", cll_node_id)
+
+                if not cll_node:
+                    continue
+                nodes[cll_node_id] = cll_node
+
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
+                if node_diff is not None:
+                    cll_node.change_status = node_diff.change_status
+                    if node_diff.change is not None:
+                        cll_node.change_category = node_diff.change.category
+                        for c, cll_column in cll_node_columns.items():
+                            cll_node.columns[c] = cll_column
+                            columns[cll_column.id] = cll_column
+                            if node_diff.change.columns and c in node_diff.change.columns:
+                                cll_column.change_status = node_diff.change.columns[c]
+
+                parent_map[cll_node_id] = manifest.parent_map.get(cll_node_id, [])
+
+        # build the child map
+        for parent_id, parents in parent_map.items():
+            for parent in parents:
+                if parent not in child_map:
+                    child_map[parent] = set()
+                child_map[parent].add(parent_id)
+
+        # Find the anchor nodes
+        anchor_node_ids = set()
+        extra_node_ids = set()
+        if node_id is None and column is None:
+            if change_analysis:
+                # If change analysis is requested, we need to find the nodes that have changes
+                lineage_diff = self.get_lineage_diff()
+                for nid, nd in lineage_diff.diff.items():
+                    if nd.change_status == "added":
+                        anchor_node_ids.add(nid)
+                        n = lineage_diff.current["nodes"].get(nid)
+                        n_columns = n.get("columns", {})
+                        for c in n_columns:
+                            anchor_node_ids.add(build_column_key(nid, c))
+                        continue
+                    if nd.change_status == "removed":
+                        extra_node_ids.add(nid)
+                        continue
+
+                    node_diff = self.get_change_analysis_cached(nid)
+                    if node_diff is not None and node_diff.change is not None:
+                        extra_node_ids.add(nid)
+                        if no_cll:
+                            if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
+                                anchor_node_ids.add(nid)
+                        else:
+                            if node_diff.change.category in ["breaking", "unknown"]:
+                                anchor_node_ids.add(nid)
+                        if node_diff.change.columns is not None:
+                            for column_name in node_diff.change.columns:
+                                anchor_node_ids.add(f"{nid}_{column_name}")
+            else:
+                lineage_diff = self.get_lineage_diff()
+                anchor_node_ids = lineage_diff.diff.keys()
+        elif node_id is not None and column is None:
+            if change_analysis:
+                # If change analysis is requested, we need to find the nodes that have changes
+                node_diff = self.get_change_analysis_cached(node_id)
+                if node_diff is not None and node_diff.change is not None:
+                    extra_node_ids.add(node_id)
+                    if no_cll:
+                        if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
+                            anchor_node_ids.add(node_id)
+                    else:
+                        if node_diff.change.category in ["breaking", "unknown"]:
+                            anchor_node_ids.add(node_id)
+                    if node_diff.change.columns is not None:
+                        for column_name in node_diff.change.columns:
+                            anchor_node_ids.add(f"{node_id}_{column_name}")
+                else:
+                    anchor_node_ids.add(node_id)
+            else:
+                anchor_node_ids.add(node_id)
+                if not no_cll:
+                    node = nodes.get(node_id)
+                    if node:
+                        for column_name in node.columns:
+                            column_key = build_column_key(node_id, column_name)
+                            anchor_node_ids.add(column_key)
+        else:
+            anchor_node_ids.add(f"{node_id}_{column}")
+
+        cll_tracker.set_anchor_nodes(len(anchor_node_ids))
+        result_node_ids = set(anchor_node_ids)
+        if not no_upstream:
+            result_node_ids = result_node_ids.union(find_upstream(anchor_node_ids, parent_map))
+        if not no_downstream:
+            result_node_ids = result_node_ids.union(find_downstream(anchor_node_ids, child_map))
+
+        # Filter the nodes and columns based on the anchor nodes
+        if not no_filter:
+            nodes = {k: v for k, v in nodes.items() if k in result_node_ids or k in extra_node_ids}
+            columns = {k: v for k, v in columns.items() if k in result_node_ids or k in extra_node_ids}
+
+            for node in nodes.values():
+                node.columns = {
+                    k: v for k, v in node.columns.items() if v.id in result_node_ids or v.id in extra_node_ids
+                }
+
+                if change_analysis:
+                    node.impacted = node.id in result_node_ids
+
+            parent_map, child_map = filter_dependency_maps(parent_map, child_map, result_node_ids)
 
         cll_tracker.end_column_lineage()
-        cll_tracker.set_total_nodes(len(nodes))
-        log_performance('column level lineage', cll_tracker.to_dict())
+        cll_tracker.set_total_nodes(len(nodes) + len(columns))
+        log_performance("column level lineage", cll_tracker.to_dict())
         cll_tracker.reset()
 
-        return dict(nodes=nodes)
+        return CllData(
+            nodes=nodes,
+            columns=columns,
+            parent_map=parent_map,
+            child_map=child_map,
+        )
 
     @lru_cache(maxsize=128)
-    def get_cll_cached(self, node_id: str, base: Optional[bool] = False):
-        nodes = self.get_lineage_nodes_metadata(base=base)
+    def get_cll_cached(self, node_id: str, base: Optional[bool] = False) -> Optional[CllData]:
+        cll_tracker = CLLPerformanceTracking()
 
-        manifest = self.curr_manifest if base is False else self.base_manifest
-        manifest_dict = manifest.to_dict()
-        parent_list = []
-        if node_id in manifest_dict['parent_map']:
-            parent_list = manifest_dict['parent_map'][node_id]
-
-        node = deepcopy(nodes[node_id])
-        self.append_column_lineage(node, parent_list, base)
-        return node
-
-    def append_column_lineage(self, node: Dict, parent_list: List, base: Optional[bool] = False):
-        def _apply_all_columns(node, trans_type, depends_on):
-            for col in node.get('columns', {}).values():
-                col['transformation_type'] = trans_type
-                col['depends_on'] = depends_on
-
-        def _depend_node_to_id(column_lineage, nodes):
-            for cl in column_lineage.values():
-                for depend_on in cl.depends_on:
-                    if depend_on.node.startswith('__'):
-                        for n in nodes.values():
-                            if n.get('resource_type') != 'source':
-                                continue
-                            # __source__table -> source.table
-                            source_table = depend_on.node.lstrip("_").replace("__", ".", 1).lower()
-                            if source_table in n.get('id'):
-                                depend_on.node = n.get('id')
-                                break
-                    else:
-                        for n in nodes.values():
-                            if n.get('name') == depend_on.node.lower():
-                                depend_on.node = n.get('id')
-                                break
+        node, parent_list = self.get_cll_node(node_id, base=base)
+        if node is None:
+            return None
+
+        cll_tracker.set_total_nodes(1)
+        cll_tracker.start_column_lineage()
+
+        def _apply_all_columns(node: CllNode, transformation_type):
+            cll_data = CllData()
+            cll_data.nodes[node.id] = node
+            cll_data.parent_map[node.id] = set(parent_list)
+            for col in node.columns.values():
+                column_id = f"{node.id}_{col.name}"
+                col.transformation_type = transformation_type
+                cll_data.columns[column_id] = col
+                cll_data.parent_map[column_id] = set()
+            return cll_data
 
-        cll_tracker = CLLPerformanceTracking()
-        nodes = self.get_lineage_nodes_metadata(base=base)
         manifest = as_manifest(self.get_manifest(base))
-        resource_type = node.get('resource_type')
-        if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
-            return
+        catalog = self.curr_catalog if base is False else self.base_catalog
+        resource_type = node.resource_type
+        if resource_type not in {"model", "seed", "source", "snapshot"}:
+            return _apply_all_columns(node, "unknown")
 
-        if resource_type == 'source' or resource_type == 'seed':
-            _apply_all_columns(node, 'source', [])
-            return
+        if resource_type == "source" or resource_type == "seed":
+            return _apply_all_columns(node, "source")
 
-        if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
-            _apply_all_columns(node, 'unknown', [])
-            return
+        if node.raw_code is None or self.is_python_model(node.id, base=base):
+            return _apply_all_columns(node, "unknown")
 
-        # dbt <= 1.8, MetricFlow expects the time spine table to be named metricflow_time_spine
-        if node.get('name') == 'metricflow_time_spine':
-            _apply_all_columns(node, 'source', [])
-            return
+        if node.name == "metricflow_time_spine":
+            return _apply_all_columns(node, "source")
 
-        if not node.get('columns', {}):
-            # no catalog
-            return
+        if not node.columns:
+            return _apply_all_columns(node, "unknown")
+
+        table_id_map = {}
 
         def ref_func(*args):
+            node_name: str = None
+            project_or_package: str = None
+
             if len(args) == 1:
-                node = args[0]
-            elif len(args) > 1:
-                node = args[1]
+                node_name = args[0]
             else:
-                return None
-            return node
+                project_or_package = args[0]
+                node_name = args[1]
 
-        def source_func(source_name, table_name):
-            return f"__{source_name}__{table_name}"
+            for key, n in manifest.nodes.items():
+                if n.name != node_name:
+                    continue
+                if project_or_package is not None and n.package_name != project_or_package:
+                    continue
+
+                # replace id "." to "_"
+                unique_id = n.unique_id
+                table_name = unique_id.replace(".", "_")
+                table_id_map[table_name.lower()] = unique_id
+                return table_name
+
+            raise ValueError(f"Cannot find node {node_name} in the manifest")
+
+        def source_func(source_name, name):
+            for key, n in manifest.sources.items():
+                if n.source_name != source_name:
+                    continue
+                if n.name != name:
+                    continue
+
+                # replace id "." to "_"
+                unique_id = n.unique_id
+                table_name = unique_id.replace(".", "_")
+                table_id_map[table_name.lower()] = unique_id
+                return table_name
 
-        raw_code = node.get('raw_code')
+            raise ValueError(f"Cannot find source {source_name}.{name} in the manifest")
+
+        raw_code = node.raw_code
         jinja_context = dict(
             ref=ref_func,
             source=source_func,
         )
 
         schema = {}
-        for parent_id in parent_list:
-            parent_node = nodes.get(parent_id)
-            if parent_node is None:
-                continue
-            columns = parent_node.get('columns') or {}
-            name = parent_node.get('name')
-            if parent_node.get('resource_type') == 'source':
-                parts = parent_id.split('.')
-                source = parts[2]
-                table = parts[3]
-                name = f"__{source}__{table}"
-            schema[name] = {
-                name: column.get('type') for name, column in columns.items()
-            }
+        if catalog is not None:
+            for parent_id in parent_list:
+                table_name = parent_id.replace(".", "_")
+                columns = {}
+                if parent_id in catalog.nodes:
+                    for col_name, col_metadata in catalog.nodes[parent_id].columns.items():
+                        columns[col_name] = col_metadata.type
+                if parent_id in catalog.sources:
+                    for col_name, col_metadata in catalog.sources[parent_id].columns.items():
+                        columns[col_name] = col_metadata.type
+                schema[table_name] = columns
 
         try:
-            # provide a manifest to speedup and not pollute the manifest
             compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
             dialect = self.adapter.type()
-            # find adapter type from the manifest, otherwise we use the adapter type from the adapter
             if self.get_manifest(base).metadata.adapter_type is not None:
                 dialect = self.get_manifest(base).metadata.adapter_type
-            column_lineage = cll(compiled_sql, schema=schema, dialect=dialect)
+            m2c, c2c_map = cll(compiled_sql, schema=schema, dialect=dialect)
         except RecceException:
-            # TODO: provide parsing error message if needed
-            _apply_all_columns(node, 'unknown', [])
             cll_tracker.increment_sqlglot_error_nodes()
-            return
+            return _apply_all_columns(node, "unknown")
         except Exception:
-            _apply_all_columns(node, 'unknown', [])
             cll_tracker.increment_other_error_nodes()
-            return
-
-        _depend_node_to_id(column_lineage, nodes)
+            return _apply_all_columns(node, "unknown")
+
+        # Add cll dependency to the node.
+        cll_data = CllData()
+        cll_data.nodes[node.id] = node
+        cll_data.columns = {f"{node.id}_{col.name}": col for col in node.columns.values()}
+
+        # parent map for node
+        depends_on = set(parent_list)
+        for d in m2c:
+            parent_key = f"{table_id_map[d.node.lower()]}_{d.column}"
+            depends_on.add(parent_key)
+        cll_data.parent_map[node_id] = depends_on
+
+        # parent map for columns
+        for name, column in node.columns.items():
+            depends_on = set()
+            column_id = f"{node.id}_{name}"
+            if name in c2c_map:
+                for d in c2c_map[name].depends_on:
+                    parent_key = f"{table_id_map[d.node.lower()]}_{d.column}"
+                    depends_on.add(parent_key)
+                column.transformation_type = c2c_map[name].transformation_type
+            cll_data.parent_map[column_id] = set(depends_on)
 
-        for name, column in node.get('columns', {}).items():
-            if name in column_lineage:
-                column['depends_on'] = column_lineage[name].depends_on
-                column['transformation_type'] = column_lineage[name].type
+        cll_tracker.end_column_lineage()
+        log_performance("column level lineage per node", cll_tracker.to_dict())
+        cll_tracker.reset()
+        return cll_data
 
-    @lru_cache(maxsize=2)
-    def get_lineage_nodes_metadata(self, base: Optional[bool] = False):
+    def get_cll_node(self, node_id: str, base: Optional[bool] = False) -> Tuple[Optional[CllNode], list[str]]:
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
-        manifest_dict = manifest.to_dict()
+        parent_list = []
+        node = None
 
-        nodes = {}
-        for node in manifest_dict['nodes'].values():
-            unique_id = node['unique_id']
-            resource_type = node['resource_type']
+        # model, seed, snapshot
+        if node_id in manifest.nodes:
+            found = manifest.nodes[node_id]
+            unique_id = found.unique_id
+            node = CllNode.build_cll_node(manifest, "nodes", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-            if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
-                continue
+            if catalog is not None and node is not None and unique_id in catalog.nodes:
+                columns = {}
+                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
+                    column_id = f"{unique_id}_{col_name}"
+                    col = CllColumn(id=column_id, name=col_name, table_id=unique_id, type=col_metadata.type)
+                    columns[col_name] = col
+                node.columns = columns
 
-            nodes[unique_id] = {
-                'id': node['unique_id'],
-                'name': node['name'],
-                'resource_type': node['resource_type'],
-                'raw_code': node['raw_code'],
-            }
+        # source
+        if node_id in manifest.sources:
+            found = manifest.sources[node_id]
+            unique_id = found.unique_id
+            node = CllNode.build_cll_node(manifest, "sources", node_id)
+            parent_list = []
 
-            if catalog is not None and unique_id in catalog.nodes:
+            if catalog is not None and node is not None and unique_id in catalog.sources:
                 columns = {}
-                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
-                    col = dict(name=col_name, type=col_metadata.type)
+                for col_name, col_metadata in catalog.sources[unique_id].columns.items():
+                    column_id = f"{unique_id}_{col_name}"
+                    col = CllColumn(id=column_id, name=col_name, table_id=unique_id, type=col_metadata.type)
                     columns[col_name] = col
-                nodes[unique_id]['columns'] = columns
+                node.columns = columns
 
-        for source in manifest_dict['sources'].values():
-            unique_id = source['unique_id']
+        # exposure
+        if node_id in manifest.exposures:
+            found = manifest.exposures[node_id]
+            node = CllNode.build_cll_node(manifest, "exposures", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-            nodes[unique_id] = {
-                'id': source['unique_id'],
-                'name': source['name'],
-                'resource_type': source['resource_type'],
-            }
+        if hasattr(manifest, "semantic_models") and node_id in manifest.semantic_models:
+            found = manifest.semantic_models[node_id]
+            node = CllNode.build_cll_node(manifest, "semantic_models", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-            if catalog is not None and unique_id in catalog.sources:
-                nodes[unique_id]['columns'] = {
-                    col_name: {
-                        'name': col_name,
-                        'type': col_metadata.type
-                    }
-                    for col_name, col_metadata in catalog.sources[unique_id].columns.items()
-                }
+        if node_id in manifest.metrics:
+            found = manifest.metrics[node_id]
+            node = CllNode.build_cll_node(manifest, "metrics", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-        return nodes
+        return node, parent_list
 
     def get_manifests_by_id(self, unique_id: str):
         curr_manifest = self.get_manifest(base=False)
         base_manifest = self.get_manifest(base=True)
         if unique_id in curr_manifest.nodes.keys() or unique_id in base_manifest.nodes.keys():
             return {
-                'current': curr_manifest.nodes.get(unique_id),
-                'base': base_manifest.nodes.get(unique_id)
+                "current": curr_manifest.nodes.get(unique_id),
+                "base": base_manifest.nodes.get(unique_id),
             }
         return None
 
@@ -1068,39 +1420,40 @@ class DbtAdapter(BaseAdapter):
             if self.base_path:
                 self.artifacts_observer.schedule(event_handler, self.base_path, recursive=False)
             self.artifacts_observer.start()
-            logger.info('Start monitoring dbt artifacts')
+            logger.info("Start monitoring dbt artifacts")
 
     def stop_monitor_artifacts(self):
         if self.artifacts_files:
             self.artifacts_observer.stop()
             self.artifacts_observer.join()
-            logger.info('Stop monitoring artifacts')
+            logger.info("Stop monitoring artifacts")
 
     def start_monitor_base_env(self, callback: Callable = None):
-        target_base_dir = os.path.join(self.runtime_config.project_root, 'target-base')
+        target_base_dir = os.path.join(self.runtime_config.project_root, "target-base")
         base_env_files = {
-            os.path.join(target_base_dir, 'manifest.json'),
-            os.path.join(target_base_dir, 'catalog.json'),
+            os.path.join(target_base_dir, "manifest.json"),
+            os.path.join(target_base_dir, "catalog.json"),
         }
         event_handler = EnvironmentEventHandler(self.base_env_observer, base_env_files, callback=callback)
         self.base_env_observer.schedule(event_handler, self.runtime_config.project_root, recursive=True)
         self.base_env_observer.start()
-        logger.info('Start monitoring base environment')
+        logger.info("Start monitoring base environment")
 
     def stop_monitor_base_env(self):
         if self.base_env_observer.is_alive():
             self.base_env_observer.stop()
         self.base_env_observer.join()
-        logger.info('Stop monitoring base environment')
-
-    def set_artifacts(self,
-                      base_manifest: WritableManifest,
-                      curr_manifest: WritableManifest,
-                      manifest: Manifest,
-                      previous_manifest: Manifest,
-                      base_catalog: CatalogArtifact,
-                      curr_catalog: CatalogArtifact,
-                      ):
+        logger.info("Stop monitoring base environment")
+
+    def set_artifacts(
+        self,
+        base_manifest: WritableManifest,
+        curr_manifest: WritableManifest,
+        manifest: Manifest,
+        previous_manifest: Manifest,
+        base_catalog: CatalogArtifact,
+        curr_catalog: CatalogArtifact,
+    ):
         self.curr_manifest = curr_manifest
         self.base_manifest = base_manifest
         self.manifest = manifest
@@ -1109,7 +1462,7 @@ class DbtAdapter(BaseAdapter):
         self.previous_state = previous_state(
             Path(self.base_path),
             Path(self.runtime_config.target_path),
-            Path(self.runtime_config.project_root)
+            Path(self.runtime_config.project_root),
         )
         self.previous_state.manifest = previous_manifest
 
@@ -1131,19 +1484,22 @@ class DbtAdapter(BaseAdapter):
         # we capture the original manifest as base and only update the current
         target_type = os.path.basename(os.path.dirname(refresh_file_path))
         if self.target_path and target_type == os.path.basename(self.target_path):
-            if refresh_file_path.endswith('manifest.json'):
+            if refresh_file_path.endswith("manifest.json"):
                 self.curr_manifest = load_manifest(path=refresh_file_path)
                 self.manifest = as_manifest(self.curr_manifest)
                 self.get_cll_cached.cache_clear()
-                self.get_lineage_nodes_metadata.cache_clear()
-            elif refresh_file_path.endswith('catalog.json'):
+                self.get_change_analysis_cached.cache_clear()
+            elif refresh_file_path.endswith("catalog.json"):
                 self.curr_catalog = load_catalog(path=refresh_file_path)
-                self.get_lineage_nodes_metadata.cache_clear()
+                self.get_cll_cached.cache_clear()
+                self.get_change_analysis_cached.cache_clear()
         elif self.base_path and target_type == os.path.basename(self.base_path):
-            if refresh_file_path.endswith('manifest.json'):
+            if refresh_file_path.endswith("manifest.json"):
                 self.base_manifest = load_manifest(path=refresh_file_path)
-            elif refresh_file_path.endswith('catalog.json'):
+                self.get_change_analysis_cached.cache_clear()
+            elif refresh_file_path.endswith("catalog.json"):
                 self.base_catalog = load_catalog(path=refresh_file_path)
+                self.get_change_analysis_cached.cache_clear()
 
     def create_relation(self, model, base=False):
         node = self.find_node_by_name(model, base)
@@ -1157,42 +1513,69 @@ class DbtAdapter(BaseAdapter):
         select: Optional[str] = None,
         exclude: Optional[str] = None,
         packages: Optional[list[str]] = None,
-        view_mode: Optional[Literal['all', 'changed_models']] = None,
+        view_mode: Optional[Literal["all", "changed_models"]] = None,
     ) -> Set[str]:
-        from dbt.graph import NodeSelector
-        from dbt.compilation import Compiler
-        from dbt.graph import parse_difference, SelectionIntersection, SelectionUnion
         import dbt.compilation
+        from dbt.compilation import Compiler
+        from dbt.graph import (
+            NodeSelector,
+            SelectionIntersection,
+            SelectionUnion,
+            parse_difference,
+        )
 
         select_list = [select] if select else None
         exclude_list = [exclude] if exclude else None
 
         def _parse_difference(include, exclude):
-            if dbt_version < 'v1.8':
+            if dbt_version < "v1.8":
                 return parse_difference(include, exclude, "eager")
             else:
                 return parse_difference(include, exclude)
 
         specs = [_parse_difference(select_list, exclude_list)]
 
+        # If packages is not provided, use the project name from manifest metadata as default
+        if packages is None:
+            if (
+                self.manifest.metadata
+                and hasattr(self.manifest.metadata, "project_name")
+                and self.manifest.metadata.project_name
+            ):
+                packages = [self.manifest.metadata.project_name]
+
         if packages is not None:
-            package_spec = SelectionUnion([_parse_difference([f'package:{p}'], None) for p in packages])
+            package_spec = SelectionUnion([_parse_difference([f"package:{p}"], None) for p in packages])
             specs.append(package_spec)
-        if view_mode and view_mode == 'changed_models':
-            specs.append(_parse_difference(['1+state:modified+'], None))
+        if view_mode and view_mode == "changed_models":
+            specs.append(_parse_difference(["1+state:modified+"], None))
         spec = SelectionIntersection(specs)
 
         manifest = Manifest()
+        manifest.metadata.adapter_type = self.adapter.type()
         manifest_prev = self.previous_state.manifest
         manifest_curr = self.manifest
 
-        manifest.nodes = {**manifest_prev.nodes, **manifest_curr.nodes}
+        manifest.nodes = {**manifest_curr.nodes}
+        # # mark a node is removed if the node id is no in the curr nodes
+        for node_id, node in manifest_prev.nodes.items():
+            if node_id not in manifest.nodes:
+                node_dict = node.to_dict()
+                if "raw_code" in node_dict:
+                    node_dict["raw_code"] = "__removed__"
+                node_class = type(node)
+                removed_node = node_class.from_dict(node_dict)
+                manifest.nodes[node_id] = removed_node
+
         manifest.macros = {**manifest_prev.macros, **manifest_curr.macros}
         manifest.sources = {**manifest_prev.sources, **manifest_curr.sources}
         manifest.exposures = {**manifest_prev.exposures, **manifest_curr.exposures}
         manifest.metrics = {**manifest_prev.metrics, **manifest_curr.metrics}
-        if hasattr(manifest_prev, 'semantic_models'):
-            manifest.semantic_models = {**manifest_prev.semantic_models, **manifest_curr.semantic_models}
+        if hasattr(manifest_prev, "semantic_models"):
+            manifest.semantic_models = {
+                **manifest_prev.semantic_models,
+                **manifest_curr.semantic_models,
+            }
 
         compiler = Compiler(self.runtime_config)
         # disable to print compile states
@@ -1207,28 +1590,28 @@ class DbtAdapter(BaseAdapter):
             return selector.get_selected(spec)
 
     def export_artifacts(self) -> ArtifactsRoot:
-        '''
+        """
         Export the artifacts from the current state
-        '''
+        """
         artifacts = ArtifactsRoot()
 
         def _load_artifact(artifact):
             return artifact.to_dict() if artifact else None
 
         artifacts.base = {
-            'manifest': _load_artifact(self.base_manifest),
-            'catalog': _load_artifact(self.base_catalog),
+            "manifest": _load_artifact(self.base_manifest),
+            "catalog": _load_artifact(self.base_catalog),
         }
         artifacts.current = {
-            'manifest': _load_artifact(self.curr_manifest),
-            'catalog': _load_artifact(self.curr_catalog),
+            "manifest": _load_artifact(self.curr_manifest),
+            "catalog": _load_artifact(self.curr_catalog),
         }
         return artifacts
 
     def export_artifacts_from_file(self) -> ArtifactsRoot:
-        '''
+        """
         Export the artifacts from the state file. This is the old implementation
-        '''
+        """
         artifacts = ArtifactsRoot()
         target_path = self.runtime_config.target_path
         target_base_path = self.base_path
@@ -1237,18 +1620,18 @@ class DbtAdapter(BaseAdapter):
             if not os.path.isfile(path):
                 return None
 
-            with open(path, 'r') as f:
+            with open(path, "r", encoding="utf-8") as f:
                 json_content = f.read()
                 return json.loads(json_content)
 
         project_root = self.runtime_config.project_root
         artifacts.base = {
-            'manifest': _load_artifact(os.path.join(project_root, target_base_path, 'manifest.json')),
-            'catalog': _load_artifact(os.path.join(project_root, target_base_path, 'catalog.json')),
+            "manifest": _load_artifact(os.path.join(project_root, target_base_path, "manifest.json")),
+            "catalog": _load_artifact(os.path.join(project_root, target_base_path, "catalog.json")),
         }
         artifacts.current = {
-            'manifest': _load_artifact(os.path.join(project_root, target_path, 'manifest.json')),
-            'catalog': _load_artifact(os.path.join(project_root, target_path, 'catalog.json')),
+            "manifest": _load_artifact(os.path.join(project_root, target_path, "manifest.json")),
+            "catalog": _load_artifact(os.path.join(project_root, target_path, "catalog.json")),
         }
         return artifacts
 
@@ -1256,7 +1639,7 @@ class DbtAdapter(BaseAdapter):
         # Merge the artifacts from the state file or cloud
         def _select_artifact(
             original: Union[WritableManifest, CatalogArtifact],
-            new: Union[WritableManifest, CatalogArtifact]
+            new: Union[WritableManifest, CatalogArtifact],
         ):
             if merge:
                 if not original:
@@ -1267,16 +1650,16 @@ class DbtAdapter(BaseAdapter):
             else:
                 return new
 
-        self.base_manifest = _select_artifact(self.base_manifest, load_manifest(data=artifacts.base.get('manifest')))
-        self.curr_manifest = _select_artifact(self.curr_manifest, load_manifest(data=artifacts.current.get('manifest')))
-        self.base_catalog = _select_artifact(self.base_catalog, load_catalog(data=artifacts.base.get('catalog')))
-        self.curr_catalog = _select_artifact(self.curr_catalog, load_catalog(data=artifacts.current.get('catalog')))
+        self.base_manifest = _select_artifact(self.base_manifest, load_manifest(data=artifacts.base.get("manifest")))
+        self.curr_manifest = _select_artifact(self.curr_manifest, load_manifest(data=artifacts.current.get("manifest")))
+        self.base_catalog = _select_artifact(self.base_catalog, load_catalog(data=artifacts.base.get("catalog")))
+        self.curr_catalog = _select_artifact(self.curr_catalog, load_catalog(data=artifacts.current.get("catalog")))
 
         self.manifest = as_manifest(self.curr_manifest)
         self.previous_state = previous_state(
             Path(self.base_path),
             Path(self.runtime_config.target_path),
-            Path(self.runtime_config.project_root)
+            Path(self.runtime_config.project_root),
         )
         self.previous_state.manifest = as_manifest(self.base_manifest)
 
@@ -1292,7 +1675,8 @@ class DbtAdapter(BaseAdapter):
 
         if not self.curr_manifest or not self.base_manifest:
             raise Exception(
-                'No enough dbt artifacts in the state file. Please use the latest recce to generate the recce state')
+                "No enough dbt artifacts in the state file. Please use the latest recce to generate the recce state"
+            )
 
     @contextmanager
     def connection_named(self, name: str) -> Iterator[None]: