recce-nightly 0.51.0.20250116__tar.gz → 0.51.0.20250120__tar.gz

{recce_nightly-0.51.0.20250116/recce_nightly.egg-info → recce_nightly-0.51.0.20250120}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: recce-nightly
-Version: 0.51.0.20250116
+Version: 0.51.0.20250120
 Summary: Environment diff tool for dbt
 Home-page: https://github.com/InfuseAI/recce
 Author: InfuseAI Dev Team
@@ -36,6 +36,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: python-multipart
 Requires-Dist: GitPython
 Requires-Dist: PyGithub
+Requires-Dist: sqlglot
 Provides-Extra: dev
 Requires-Dist: pytest>=4.6; extra == "dev"
 Requires-Dist: pytest-flake8; extra == "dev"

recce_nightly-0.51.0.20250120/recce/VERSION

@@ -0,0 +1 @@
+0.51.0.20250120

{recce_nightly-0.51.0.20250116 → recce_nightly-0.51.0.20250120}/recce/adapter/base.py

@@ -2,6 +2,7 @@ import logging
 from abc import ABC, abstractmethod
 from typing import Optional, Callable, Set, Literal
 
+from recce.models.types import LineageDiff, NodeDiff
 from recce.state import ArtifactsRoot
 
 # from dbt.contracts.graph.nodes import ManifestNode
@@ -19,6 +20,37 @@ class BaseAdapter(ABC):
     def get_lineage(self, base: Optional[bool] = False):
         raise NotImplementedError()
 
+    def get_lineage_diff(self) -> LineageDiff:
+        base = self.get_lineage(base=True)
+        current = self.get_lineage(base=False)
+        keys = {
+            *base.get('nodes', {}).keys(),
+            *current.get('nodes', {}).keys()
+        }
+
+        # for each node, compare the base and current lineage
+        diff = {}
+        for key in keys:
+            base_node = base.get('nodes', {}).get(key)
+            curr_node = current.get('nodes', {}).get(key)
+            if base_node and curr_node:
+                base_checksum = base_node.get('checksum', {}).get('checksum')
+                curr_checksum = curr_node.get('checksum', {}).get('checksum')
+
+                if base_checksum is not None and base_checksum == curr_checksum:
+                    continue
+
+                diff[key] = NodeDiff(change_status='modified', change_category='breaking')
+            elif base_node:
+                diff[key] = NodeDiff(chnage_status='removed')
+            elif curr_node:
+                diff[key] = NodeDiff(change_status='added')
+        return LineageDiff(
+            base=base,
+            current=current,
+            diff=diff,
+        )
+
     @abstractmethod
     def select_nodes(
         self,

{recce_nightly-0.51.0.20250116 → recce_nightly-0.51.0.20250120}/recce/adapter/dbt_adapter/__init__.py

@@ -5,6 +5,7 @@ import uuid
 from contextlib import contextmanager
 from dataclasses import dataclass, fields
 from errno import ENOENT
+from functools import lru_cache
 from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, Iterator, Any, Set, Union, Literal, Type
 
@@ -18,6 +19,7 @@ from recce.adapter.base import BaseAdapter
 from recce.state import ArtifactsRoot
 from .dbt_version import DbtVersion
 from ...models import RunType
+from ...models.types import LineageDiff, NodeDiff
 from ...tasks import Task, QueryTask, QueryBaseTask, QueryDiffTask, ValueDiffTask, ValueDiffDetailTask, ProfileDiffTask, \
     RowCountDiffTask, TopKDiffTask, HistogramDiffTask
 
@@ -537,6 +539,23 @@ class DbtAdapter(BaseAdapter):
     def get_lineage(self, base: Optional[bool] = False):
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
+        cache_key = hash((id(manifest), id(catalog)))
+        return self.get_lineage_cached(base, cache_key)
+
+    def get_lineage_diff(self) -> LineageDiff:
+        cache_key = hash((
+            id(self.base_manifest),
+            id(self.base_catalog),
+            id(self.curr_manifest),
+            id(self.curr_catalog),
+        ))
+        return self._get_lineage_diff_cached(cache_key)
+
+    @lru_cache(maxsize=2)
+    def get_lineage_cached(self, base: Optional[bool] = False, cache_key=0):
+        manifest = self.curr_manifest if base is False else self.base_manifest
+        catalog = self.curr_catalog if base is False else self.base_catalog
+
         manifest_metadata = manifest.metadata if manifest is not None else None
         catalog_metadata = catalog.metadata if catalog is not None else None
 
@@ -658,6 +677,48 @@ class DbtAdapter(BaseAdapter):
             catalog_metadata=catalog_metadata,
         )
 
+    @lru_cache(maxsize=1)
+    def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
+        base = self.get_lineage(base=True)
+        current = self.get_lineage(base=False)
+        keys = {
+            *base.get('nodes', {}).keys(),
+            *current.get('nodes', {}).keys()
+        }
+
+        # for each node, compare the base and current lineage
+        diff = {}
+        for key in keys:
+            base_node = base.get('nodes', {}).get(key)
+            curr_node = current.get('nodes', {}).get(key)
+            if base_node and curr_node:
+                base_checksum = base_node.get('checksum', {}).get('checksum')
+                curr_checksum = curr_node.get('checksum', {}).get('checksum')
+                if base_checksum is None or curr_checksum is None or base_checksum == curr_checksum:
+                    continue
+
+                change_category = 'breaking'
+                if curr_node.get('resource_type') == 'model':
+                    try:
+                        from recce.util.breaking import is_breaking_change
+                        base_sql = self.generate_sql(base_node.get('raw_code'))
+                        curr_sql = self.generate_sql(curr_node.get('raw_code'))
+                        if not is_breaking_change(base_sql, curr_sql):
+                            change_category = 'non-breaking'
+                    except Exception:
+                        pass
+
+                diff[key] = NodeDiff(change_status='modified', change_category=change_category)
+            elif base_node:
+                diff[key] = NodeDiff(chnage_status='removed')
+            elif curr_node:
+                diff[key] = NodeDiff(change_status='added')
+        return LineageDiff(
+            base=base,
+            current=current,
+            diff=diff,
+        )
+
     def get_manifests_by_id(self, unique_id: str):
         curr_manifest = self.get_manifest(base=False)
         base_manifest = self.get_manifest(base=True)
@@ -916,3 +977,9 @@ class DbtAdapter(BaseAdapter):
 
     def cancel(self, connection: Connection):
         self.adapter.connections.cancel(connection)
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        return self.__hash__() == other.__hash__()

{recce_nightly-0.51.0.20250116 → recce_nightly-0.51.0.20250120}/recce/core.py

@@ -7,6 +7,7 @@ from typing import Callable, Dict, Optional, List, Tuple
 
 from recce.adapter.base import BaseAdapter
 from recce.models import Check, Run
+from recce.models.types import LineageDiff
 from recce.state import RecceState, RecceStateMetadata, GitRepoInfo, PullRequestInfo, RecceStateLoader
 from recce.util.recce_cloud import set_recce_cloud_onboarding_state
 
@@ -67,6 +68,9 @@ class RecceContext:
     def get_lineage(self, base: Optional[bool] = False):
         return self.adapter.get_lineage(base=base)
 
+    def get_lineage_diff(self) -> LineageDiff:
+        return self.adapter.get_lineage_diff()
+
     def build_name_to_unique_id_index(self) -> Dict[str, str]:
         name_to_unique_id = {}
         curr = self.get_lineage(base=False)

{recce_nightly-0.51.0.20250116 → recce_nightly-0.51.0.20250120}/recce/data/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-300961f547f48fe7.js"/><script src="/_next/static/chunks/157e448a-49151cad1ec51175.js" async=""></script><script src="/_next/static/chunks/312-3e9b49b718b98737.js" async=""></script><script src="/_next/static/chunks/main-app-19c0918a6bb54549.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-300961f547f48fe7.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[16892,[],\"\"]\n3:I[60299,[],\"\"]\n4:I[43276,[],\"\"]\na:I[28103,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"6ANkv4M7ypHedA7qwoJqp\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-300961f547f48fe7.js"/><script src="/_next/static/chunks/157e448a-49151cad1ec51175.js" async=""></script><script src="/_next/static/chunks/312-3e9b49b718b98737.js" async=""></script><script src="/_next/static/chunks/main-app-19c0918a6bb54549.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-300961f547f48fe7.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[16892,[],\"\"]\n3:I[60299,[],\"\"]\n4:I[43276,[],\"\"]\na:I[28103,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"R7dmZds4xWYx3fGG1bDQE\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>