recce-nightly 0.55.0.20250220__tar.gz → 0.55.0.20250224__tar.gz

{recce_nightly-0.55.0.20250220 → recce_nightly-0.55.0.20250224}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: recce-nightly
-Version: 0.55.0.20250220
+Version: 0.55.0.20250224
 Summary: Environment diff tool for dbt
 Home-page: https://github.com/InfuseAI/recce
 Author: InfuseAI Dev Team

recce_nightly-0.55.0.20250224/recce/VERSION

@@ -0,0 +1 @@
+0.55.0.20250224

{recce_nightly-0.55.0.20250220 → recce_nightly-0.55.0.20250224}/recce/adapter/dbt_adapter/__init__.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, Iterator, Any, Set, Union, Literal, Type
 
 from recce.util.cll import cll
+from recce.exceptions import RecceException
 
 try:
     import agate
@@ -470,6 +471,14 @@ class DbtAdapter(BaseAdapter):
             os.path.join(project_root, target_base_path, 'catalog.json'),
         ]
 
+    def is_python_model(self, node_id: str, base: Optional[bool] = False):
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        model = manifest.nodes.get(node_id)
+        if hasattr(model, 'language'):
+            return model.language == 'python'
+
+        return False
+
     def find_node_by_name(self, node_name, base=False) -> Optional[ManifestNode]:
 
         manifest = self.curr_manifest if base is False else self.base_manifest
@@ -545,6 +554,36 @@ class DbtAdapter(BaseAdapter):
 
         return self.adapter.execute(sql, auto_begin=auto_begin, fetch=fetch, limit=limit)
 
+    def build_parent_map(self, nodes: Dict, base: Optional[bool] = False) -> Dict[str, List[str]]:
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        manifest_dict = manifest.to_dict()
+
+        node_ids = nodes.keys()
+        parent_map = {}
+        for k, parents in manifest_dict['parent_map'].items():
+            if k not in node_ids:
+                continue
+            parent_map[k] = [parent for parent in parents if parent in node_ids]
+
+        return parent_map
+
+    @lru_cache(maxsize=2)
+    def build_table_map(self, base: Optional[bool] = False) -> Dict[str, str]:
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        manifest_dict = manifest.to_dict()
+
+        table_map = {}
+        for node in manifest_dict['nodes'].values():
+            resource_type = node['resource_type']
+            if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
+                continue
+            table_map[node['unique_id']] = node.get('alias')
+
+        for source in manifest_dict['sources'].values():
+            table_map[source['unique_id']] = source.get('identifier')
+
+        return table_map
+
     def get_lineage(self, base: Optional[bool] = False):
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
@@ -618,8 +657,8 @@ class DbtAdapter(BaseAdapter):
             if catalog is not None and unique_id in catalog.nodes:
                 columns = {}
                 primary_key = None
-                for col_name, col in catalog.nodes[unique_id].columns.items():
-                    col = dict(name=col_name, type=col.type)
+                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
+                    col = dict(name=col_name, type=col_metadata.type)
                     if col_name in cols_not_null:
                         col['not_null'] = True
                     if col_name in cols_unique:
@@ -643,7 +682,13 @@ class DbtAdapter(BaseAdapter):
             }
 
             if catalog is not None and unique_id in catalog.sources:
-                nodes[unique_id]['columns'] = catalog.sources[unique_id].columns
+                nodes[unique_id]['columns'] = {
+                    col_name: {
+                        'name': col_name,
+                        'type': col_metadata.type
+                    }
+                    for col_name, col_metadata in catalog.sources[unique_id].columns.items()
+                }
 
         for exposure in manifest_dict['exposures'].values():
             nodes[exposure['unique_id']] = {
@@ -672,45 +717,70 @@ class DbtAdapter(BaseAdapter):
                     'config': semantic_models['config'],
                 }
 
-        nodeIds = nodes.keys()
-        parent_map = {}
-        for k, parents in manifest_dict['parent_map'].items():
-            if k not in nodeIds:
-                continue
-            parent_map[k] = [parent for parent in parents if parent in nodeIds]
+        parent_map = self.build_parent_map(nodes, base)
+        table_map = {}
 
         # Handle column-level lineage, only enable if env var is set to true
         if os.getenv('RECCE_CLL_ENABLED') == 'true':
-            for node in nodes.values():
-                if node.get('resource_type') != 'model':
-                    continue
-
-                if node.get('raw_code') is None:
-                    continue
-
-                compiled_sql = self.generate_sql(node.get('raw_code'), base=base)
-                schema = {}
-                for parent_id in parent_map[node.get('id')]:
-                    parent_node = nodes.get(parent_id)
-                    if parent_node is None:
-                        continue
-                    columns = parent_node.get('columns') or {}
-                    schema[parent_node['name']] = {
-                        name: column.get('type') for name, column in columns.items()
-                    }
-                column_lineage = cll(compiled_sql, schema=schema)
-                for name, column in node['columns'].items():
-                    if name in column_lineage:
-                        column['depends_on'] = column_lineage[name].depends_on
-                        column['transformation_type'] = column_lineage[name].type
+            if base is False:
+                table_map = self.build_table_map(base)
+                self.append_column_lineage(nodes, parent_map, base)
 
         return dict(
             parent_map=parent_map,
             nodes=nodes,
+            table_map=table_map,
             manifest_metadata=manifest_metadata,
             catalog_metadata=catalog_metadata,
         )
 
+    def append_column_lineage(self, nodes: Dict, parent_map: Dict, base: Optional[bool] = False):
+        def _apply_all_columns(node, trans_type, depends_on):
+            for col in node.get('columns', {}).values():
+                col['transformation_type'] = trans_type
+                col['depends_on'] = depends_on
+
+        for node in nodes.values():
+            resource_type = node.get('resource_type')
+            if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
+                continue
+
+            if resource_type == 'source' or resource_type == 'seed':
+                _apply_all_columns(node, 'source', [])
+                continue
+
+            if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
+                _apply_all_columns(node, 'unknown', [])
+                continue
+
+            if not node.get('columns', {}):
+                # no catalog
+                continue
+
+            compiled_sql = self.generate_sql(node.get('raw_code'), base=base)
+
+            schema = {}
+            for parent_id in parent_map[node.get('id')]:
+                parent_node = nodes.get(parent_id)
+                if parent_node is None:
+                    continue
+                columns = parent_node.get('columns') or {}
+                schema[parent_node['name']] = {
+                    name: column.get('type') for name, column in columns.items()
+                }
+
+            try:
+                column_lineage = cll(compiled_sql, schema=schema)
+            except RecceException as exception:
+                print(exception)
+                _apply_all_columns(node, 'unknown', [])
+                continue
+
+            for name, column in node.get('columns', {}).items():
+                if name in column_lineage:
+                    column['depends_on'] = column_lineage[name].depends_on
+                    column['transformation_type'] = column_lineage[name].type
+
     @lru_cache(maxsize=1)
     def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
         base = self.get_lineage(base=True)

{recce_nightly-0.55.0.20250220 → recce_nightly-0.55.0.20250224}/recce/data/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-300961f547f48fe7.js"/><script src="/_next/static/chunks/157e448a-49151cad1ec51175.js" async=""></script><script src="/_next/static/chunks/312-3e9b49b718b98737.js" async=""></script><script src="/_next/static/chunks/main-app-19c0918a6bb54549.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-300961f547f48fe7.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[16892,[],\"\"]\n3:I[60299,[],\"\"]\n4:I[43276,[],\"\"]\na:I[28103,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"QkPW3VTSbTMwgQRBH2Ook\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-300961f547f48fe7.js"/><script src="/_next/static/chunks/157e448a-49151cad1ec51175.js" async=""></script><script src="/_next/static/chunks/312-3e9b49b718b98737.js" async=""></script><script src="/_next/static/chunks/main-app-19c0918a6bb54549.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-300961f547f48fe7.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[16892,[],\"\"]\n3:I[60299,[],\"\"]\n4:I[43276,[],\"\"]\na:I[28103,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"0XMZCpfFuF8AnVBxj1oJA\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>