recce-nightly 0.59.0.20250325__tar.gz → 0.60.0.20250327__tar.gz

{recce_nightly-0.59.0.20250325 → recce_nightly-0.60.0.20250327}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recce-nightly
-Version: 0.59.0.20250325
+Version: 0.60.0.20250327
 Summary: Environment diff tool for dbt
 Home-page: https://github.com/InfuseAI/recce
 Author: InfuseAI Dev Team

recce_nightly-0.60.0.20250327/recce/VERSION

@@ -0,0 +1 @@
+0.60.0.20250327

{recce_nightly-0.59.0.20250325 → recce_nightly-0.60.0.20250327}/recce/adapter/base.py

@@ -42,7 +42,7 @@ class BaseAdapter(ABC):
 
                 diff[key] = NodeDiff(change_status='modified', change_category='breaking')
             elif base_node:
-                diff[key] = NodeDiff(chnage_status='removed')
+                diff[key] = NodeDiff(change_status='removed')
             elif curr_node:
                 diff[key] = NodeDiff(change_status='added')
         return LineageDiff(

{recce_nightly-0.59.0.20250325 → recce_nightly-0.60.0.20250327}/recce/adapter/dbt_adapter/__init__.py

@@ -3,6 +3,7 @@ import logging
 import os
 import uuid
 from contextlib import contextmanager
+from copy import deepcopy
 from dataclasses import dataclass, fields
 from errno import ENOENT
 from functools import lru_cache
@@ -13,6 +14,7 @@ from recce.event import log_performance
 from recce.exceptions import RecceException
 from recce.util.cll import cll, CLLPerformanceTracking
 from ...tasks.profile import ProfileTask
+from recce.util.lineage import find_upstream, find_downstream
 
 try:
     import agate
@@ -571,22 +573,12 @@ class DbtAdapter(BaseAdapter):
 
         return parent_map
 
-    @lru_cache(maxsize=2)
-    def build_table_map(self, base: Optional[bool] = False) -> Dict[str, str]:
+    def build_parent_list_per_node(self, node_id: str, base: Optional[bool] = False) -> List[str]:
         manifest = self.curr_manifest if base is False else self.base_manifest
         manifest_dict = manifest.to_dict()
 
-        table_map = {}
-        for node in manifest_dict['nodes'].values():
-            resource_type = node['resource_type']
-            if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
-                continue
-            table_map[node['unique_id']] = node.get('alias')
-
-        for source in manifest_dict['sources'].values():
-            table_map[source['unique_id']] = source.get('identifier')
-
-        return table_map
+        if node_id in manifest_dict['parent_map']:
+            return manifest_dict['parent_map'][node_id]
 
     def get_lineage(self, base: Optional[bool] = False):
         manifest = self.curr_manifest if base is False else self.base_manifest
@@ -687,24 +679,16 @@ class DbtAdapter(BaseAdapter):
                 'resource_type': source['resource_type'],
                 'package_name': source['package_name'],
                 'config': source['config'],
-                'columns': {
-                    col.get('name'): {
-                        'name': col.get('name'),
-                        'type': col.get('data_type')
-                    }
-                    for col in source.get('columns', {}).values()
-                }
             }
 
             if catalog is not None and unique_id in catalog.sources:
-                columns = {
+                nodes[unique_id]['columns'] = {
                     col_name: {
                         'name': col_name,
                         'type': col_metadata.type
                     }
                     for col_name, col_metadata in catalog.sources[unique_id].columns.items()
                 }
-                nodes[unique_id]['columns'].update(columns)
 
         for exposure in manifest_dict['exposures'].values():
             nodes[exposure['unique_id']] = {
@@ -735,16 +719,10 @@ class DbtAdapter(BaseAdapter):
 
         parent_map = self.build_parent_map(nodes, base)
 
-        # Handle column-level lineage, only enable if env var is set to true
-        if os.getenv('RECCE_CLL_ENABLED') != 'false':
-            if base is False:
-                cll_tracker.start_column_lineage()
-                self.append_column_lineage(nodes, parent_map, base)
-                cll_tracker.end_column_lineage()
-
         if base is False:
             cll_tracker.end_lineage()
-            log_performance("column level lineage", cll_tracker.to_dict())
+            cll_tracker.set_total_nodes(len(nodes))
+            log_performance('model lineage', cll_tracker.to_dict())
             cll_tracker.reset()
 
         return dict(
@@ -754,111 +732,6 @@ class DbtAdapter(BaseAdapter):
             catalog_metadata=catalog_metadata,
         )
 
-    def append_column_lineage(self, nodes: Dict, parent_map: Dict, base: Optional[bool] = False):
-        def _apply_all_columns(node, trans_type, depends_on):
-            for col in node.get('columns', {}).values():
-                col['transformation_type'] = trans_type
-                col['depends_on'] = depends_on
-
-        def _depend_node_to_id(column_lineage, nodes):
-            for cl in column_lineage.values():
-                for depend_on in cl.depends_on:
-                    if depend_on.node.startswith('__'):
-                        for n in nodes.values():
-                            if n.get('resource_type') != 'source':
-                                continue
-                            # __source__table -> source.table
-                            source_table = depend_on.node.lstrip("_").replace("__", ".", 1).lower()
-                            if source_table in n.get('id'):
-                                depend_on.node = n.get('id')
-                                break
-                    else:
-                        for n in nodes.values():
-                            if n.get('name') == depend_on.node.lower():
-                                depend_on.node = n.get('id')
-                                break
-
-        cll_tracker = CLLPerformanceTracking()
-        cll_tracker.set_total_nodes(len(nodes))
-        manifest = as_manifest(self.get_manifest(base))
-        for node in nodes.values():
-            resource_type = node.get('resource_type')
-            if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
-                continue
-
-            if resource_type == 'source' or resource_type == 'seed':
-                _apply_all_columns(node, 'source', [])
-                continue
-
-            if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
-                _apply_all_columns(node, 'unknown', [])
-                continue
-
-            # dbt <= 1.8, MetricFlow expects the time spine table to be named metricflow_time_spine
-            if node.get('name') == 'metricflow_time_spine':
-                _apply_all_columns(node, 'source', [])
-                continue
-
-            if not node.get('columns', {}):
-                # no catalog
-                continue
-
-            def ref_func(*args):
-                if len(args) == 1:
-                    node = args[0]
-                elif len(args) > 1:
-                    node = args[1]
-                else:
-                    return None
-                return node
-
-            def source_func(source_name, table_name):
-                return f"__{source_name}__{table_name}"
-
-            raw_code = node.get('raw_code')
-            jinja_context = dict(
-                ref=ref_func,
-                source=source_func,
-            )
-
-            schema = {}
-            for parent_id in parent_map[node.get('id')]:
-                parent_node = nodes.get(parent_id)
-                if parent_node is None:
-                    continue
-                columns = parent_node.get('columns') or {}
-                name = parent_node.get('name')
-                if parent_node.get('resource_type') == 'source':
-                    parts = parent_id.split('.')
-                    source = parts[2]
-                    table = parts[3]
-                    name = f"__{source}__{table}"
-                schema[name] = {
-                    name: column.get('type') for name, column in columns.items()
-                }
-
-            try:
-                # provide a manifest to speedup and not pollute the manifest
-                compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
-                dialect = self.adapter.type()
-                column_lineage = cll(compiled_sql, schema=schema, dialect=dialect)
-            except RecceException:
-                # TODO: provide parsing error message if needed
-                _apply_all_columns(node, 'unknown', [])
-                cll_tracker.increment_sqlglot_error_nodes()
-                continue
-            except Exception:
-                _apply_all_columns(node, 'unknown', [])
-                cll_tracker.increment_other_error_nodes()
-                continue
-
-            _depend_node_to_id(column_lineage, nodes)
-
-            for name, column in node.get('columns', {}).items():
-                if name in column_lineage:
-                    column['depends_on'] = column_lineage[name].depends_on
-                    column['transformation_type'] = column_lineage[name].type
-
     @lru_cache(maxsize=1)
     def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
         base = self.get_lineage(base=True)
@@ -902,6 +775,198 @@ class DbtAdapter(BaseAdapter):
             diff=diff,
         )
 
+    def get_cll_by_node_id(self, node_id: str, base: Optional[bool] = False):
+        cll_tracker = CLLPerformanceTracking()
+        cll_tracker.start_column_lineage()
+
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        manifest_dict = manifest.to_dict()
+
+        parent_ids = find_upstream(node_id, manifest_dict.get('parent_map'))
+        child_ids = find_downstream(node_id, manifest_dict.get('child_map'))
+        cll_node_ids = parent_ids.union(child_ids)
+        cll_node_ids.add(node_id)
+
+        node_manifest = self.get_lineage_nodes_metadata(base=base)
+        nodes = {}
+        for node_id in cll_node_ids:
+            if node_id not in node_manifest:
+                continue
+            nodes[node_id] = self.get_cll_cached(node_id, base=base)
+
+        cll_tracker.end_column_lineage()
+        cll_tracker.set_total_nodes(len(nodes))
+        log_performance('column level lineage', cll_tracker.to_dict())
+        cll_tracker.reset()
+
+        return dict(nodes=nodes)
+
+    @lru_cache(maxsize=128)
+    def get_cll_cached(self, node_id: str, base: Optional[bool] = False):
+        nodes = self.get_lineage_nodes_metadata(base=base)
+
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        manifest_dict = manifest.to_dict()
+        parent_list = []
+        if node_id in manifest_dict['parent_map']:
+            parent_list = manifest_dict['parent_map'][node_id]
+
+        node = deepcopy(nodes[node_id])
+        self.append_column_lineage(node, parent_list, base)
+        return node
+
+    def append_column_lineage(self, node: Dict, parent_list: List, base: Optional[bool] = False):
+        def _apply_all_columns(node, trans_type, depends_on):
+            for col in node.get('columns', {}).values():
+                col['transformation_type'] = trans_type
+                col['depends_on'] = depends_on
+
+        def _depend_node_to_id(column_lineage, nodes):
+            for cl in column_lineage.values():
+                for depend_on in cl.depends_on:
+                    if depend_on.node.startswith('__'):
+                        for n in nodes.values():
+                            if n.get('resource_type') != 'source':
+                                continue
+                            # __source__table -> source.table
+                            source_table = depend_on.node.lstrip("_").replace("__", ".", 1).lower()
+                            if source_table in n.get('id'):
+                                depend_on.node = n.get('id')
+                                break
+                    else:
+                        for n in nodes.values():
+                            if n.get('name') == depend_on.node.lower():
+                                depend_on.node = n.get('id')
+                                break
+
+        cll_tracker = CLLPerformanceTracking()
+        nodes = self.get_lineage_nodes_metadata(base=base)
+        manifest = as_manifest(self.get_manifest(base))
+        resource_type = node.get('resource_type')
+        if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
+            return
+
+        if resource_type == 'source' or resource_type == 'seed':
+            _apply_all_columns(node, 'source', [])
+            return
+
+        if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
+            _apply_all_columns(node, 'unknown', [])
+            return
+
+        # dbt <= 1.8, MetricFlow expects the time spine table to be named metricflow_time_spine
+        if node.get('name') == 'metricflow_time_spine':
+            _apply_all_columns(node, 'source', [])
+            return
+
+        if not node.get('columns', {}):
+            # no catalog
+            return
+
+        def ref_func(*args):
+            if len(args) == 1:
+                node = args[0]
+            elif len(args) > 1:
+                node = args[1]
+            else:
+                return None
+            return node
+
+        def source_func(source_name, table_name):
+            return f"__{source_name}__{table_name}"
+
+        raw_code = node.get('raw_code')
+        jinja_context = dict(
+            ref=ref_func,
+            source=source_func,
+        )
+
+        schema = {}
+        for parent_id in parent_list:
+            parent_node = nodes.get(parent_id)
+            if parent_node is None:
+                continue
+            columns = parent_node.get('columns') or {}
+            name = parent_node.get('name')
+            if parent_node.get('resource_type') == 'source':
+                parts = parent_id.split('.')
+                source = parts[2]
+                table = parts[3]
+                name = f"__{source}__{table}"
+            schema[name] = {
+                name: column.get('type') for name, column in columns.items()
+            }
+
+        try:
+            # provide a manifest to speedup and not pollute the manifest
+            compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
+            dialect = self.adapter.type()
+            column_lineage = cll(compiled_sql, schema=schema, dialect=dialect)
+        except RecceException:
+            # TODO: provide parsing error message if needed
+            _apply_all_columns(node, 'unknown', [])
+            cll_tracker.increment_sqlglot_error_nodes()
+            return
+        except Exception:
+            _apply_all_columns(node, 'unknown', [])
+            cll_tracker.increment_other_error_nodes()
+            return
+
+        _depend_node_to_id(column_lineage, nodes)
+
+        for name, column in node.get('columns', {}).items():
+            if name in column_lineage:
+                column['depends_on'] = column_lineage[name].depends_on
+                column['transformation_type'] = column_lineage[name].type
+
+    @lru_cache(maxsize=2)
+    def get_lineage_nodes_metadata(self, base: Optional[bool] = False):
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        catalog = self.curr_catalog if base is False else self.base_catalog
+        manifest_dict = manifest.to_dict()
+
+        nodes = {}
+        for node in manifest_dict['nodes'].values():
+            unique_id = node['unique_id']
+            resource_type = node['resource_type']
+
+            if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
+                continue
+
+            nodes[unique_id] = {
+                'id': node['unique_id'],
+                'name': node['name'],
+                'resource_type': node['resource_type'],
+                'raw_code': node['raw_code'],
+            }
+
+            if catalog is not None and unique_id in catalog.nodes:
+                columns = {}
+                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
+                    col = dict(name=col_name, type=col_metadata.type)
+                    columns[col_name] = col
+                nodes[unique_id]['columns'] = columns
+
+        for source in manifest_dict['sources'].values():
+            unique_id = source['unique_id']
+
+            nodes[unique_id] = {
+                'id': source['unique_id'],
+                'name': source['name'],
+                'resource_type': source['resource_type'],
+            }
+
+            if catalog is not None and unique_id in catalog.sources:
+                nodes[unique_id]['columns'] = {
+                    col_name: {
+                        'name': col_name,
+                        'type': col_metadata.type
+                    }
+                    for col_name, col_metadata in catalog.sources[unique_id].columns.items()
+                }
+
+        return nodes
+
     def get_manifests_by_id(self, unique_id: str):
         curr_manifest = self.get_manifest(base=False)
         base_manifest = self.get_manifest(base=True)
@@ -997,8 +1062,11 @@ class DbtAdapter(BaseAdapter):
             if refresh_file_path.endswith('manifest.json'):
                 self.curr_manifest = load_manifest(path=refresh_file_path)
                 self.manifest = as_manifest(self.curr_manifest)
+                self.get_cll_cached.cache_clear()
+                self.get_lineage_nodes_metadata.cache_clear()
             elif refresh_file_path.endswith('catalog.json'):
                 self.curr_catalog = load_catalog(path=refresh_file_path)
+                self.get_lineage_nodes_metadata.cache_clear()
         elif self.base_path and target_type == os.path.basename(self.base_path):
             if refresh_file_path.endswith('manifest.json'):
                 self.base_manifest = load_manifest(path=refresh_file_path)

recce_nightly-0.60.0.20250327/recce/data/404.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-8a8709fe9d22c323.js"/><script src="/_next/static/chunks/38610ee6-24a85e2dfb424ddb.js" async=""></script><script src="/_next/static/chunks/783-784ce10322400a48.js" async=""></script><script src="/_next/static/chunks/main-app-cfd795a5fe58bf37.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-8a8709fe9d22c323.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[83798,[],\"\"]\n3:I[26375,[],\"\"]\n4:I[22587,[],\"\"]\na:I[114,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"toQz_Vr_ujCxxim6p2pBl\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>