recce-nightly 1.10.0.20250629__py3-none-any.whl → 1.25.0.20251112a20664__py3-none-any.whl

recce/VERSION

@@ -1 +1 @@
-1.10.0.20250629
+1.25.0.20251112a20664

recce/__init__.py

@@ -35,6 +35,11 @@ def is_ci_env():
     return False
 
 
+def is_recce_cloud_instance():
+    """Check if running in Recce Cloud instance."""
+    return os.environ.get("RECCE_CLOUD_INSTANCE", "false").lower() == "true"
+
+
 def get_runner():
     # GitHub Action
     if os.environ.get("GITHUB_ACTIONS", "false") == "true":

recce/adapter/dbt_adapter/__init__.py

@@ -31,9 +31,10 @@ from recce.util.lineage import (
     find_downstream,
     find_upstream,
 )
+from recce.util.perf_tracking import LineagePerfTracker
 
 from ...tasks.profile import ProfileTask
-from ...util.breaking import parse_change_category
+from ...util.breaking import BreakingPerformanceTracking, parse_change_category
 
 try:
     import agate
@@ -278,7 +279,7 @@ class DbtArgs:
     target_path: Optional[str] = (None,)
     project_only_flags: Optional[Dict[str, Any]] = None
     which: Optional[str] = None
-    state_modified_compare_more_unrendered_values: Optional[bool] = False  # new flag added since dbt v1.9
+    state_modified_compare_more_unrendered_values: Optional[bool] = True  # new flag added since dbt v1.9
 
 
 @dataclass
@@ -407,7 +408,7 @@ class DbtAdapter(BaseAdapter):
 
         if self.adapter.connections.TYPE == "databricks":
             # reference: get_columns_in_relation (dbt/adapters/databricks/impl.py)
-            from dbt.adapters.databricks import DatabricksColumn
+            from dbt.adapters.databricks.column import DatabricksColumn
 
             rows = columns
             columns = []
@@ -599,7 +600,15 @@ class DbtAdapter(BaseAdapter):
             return node.compiled_code
         else:
             from dbt.clients import jinja
-            from dbt.context.providers import generate_runtime_model_context
+            from dbt.context.providers import (
+                generate_runtime_macro_context,
+                generate_runtime_model_context,
+            )
+
+            # Set up macro resolver for dbt >= 1.8
+            macro_manifest = MacroManifest(manifest.macros)
+            self.adapter.set_macro_resolver(macro_manifest)
+            self.adapter.set_macro_context_generator(generate_runtime_macro_context)
 
             jinja_ctx = generate_runtime_model_context(node, self.runtime_config, manifest)
             jinja_ctx.update(context)
@@ -658,8 +667,8 @@ class DbtAdapter(BaseAdapter):
     @lru_cache(maxsize=2)
     def get_lineage_cached(self, base: Optional[bool] = False, cache_key=0):
         if base is False:
-            cll_tracker = CLLPerformanceTracking()
-            cll_tracker.start_lineage()
+            perf_tracker = LineagePerfTracker()
+            perf_tracker.start_lineage()
 
         manifest = self.curr_manifest if base is False else self.base_manifest
         catalog = self.curr_catalog if base is False else self.base_catalog
@@ -736,6 +745,7 @@ class DbtAdapter(BaseAdapter):
             nodes[unique_id] = {
                 "id": source["unique_id"],
                 "name": source["name"],
+                "source_name": source["source_name"],
                 "resource_type": source["resource_type"],
                 "package_name": source["package_name"],
                 "config": source["config"],
@@ -777,10 +787,10 @@ class DbtAdapter(BaseAdapter):
         parent_map = self.build_parent_map(nodes, base)
 
         if base is False:
-            cll_tracker.end_lineage()
-            cll_tracker.set_total_nodes(len(nodes))
-            log_performance("model lineage", cll_tracker.to_dict())
-            cll_tracker.reset()
+            perf_tracker.end_lineage()
+            perf_tracker.set_total_nodes(len(nodes))
+            log_performance("model lineage", perf_tracker.to_dict())
+            perf_tracker.reset()
 
         return dict(
             parent_map=parent_map,
@@ -814,17 +824,22 @@ class DbtAdapter(BaseAdapter):
 
     @lru_cache(maxsize=128)
     def get_change_analysis_cached(self, node_id: str):
+        breaking_perf_tracker = BreakingPerformanceTracking()
         lineage_diff = self.get_lineage_diff()
         diff = lineage_diff.diff
 
         if node_id not in diff or diff[node_id].change_status != "modified":
             return diff.get(node_id)
 
+        breaking_perf_tracker.increment_modified_nodes()
+        breaking_perf_tracker.start_lineage_diff()
+
         base = lineage_diff.base
         current = lineage_diff.current
 
         base_manifest = as_manifest(self.get_manifest(True))
         curr_manifest = as_manifest(self.get_manifest(False))
+        breaking_perf_tracker.record_checkpoint("manifest")
 
         def ref_func(*args):
             if len(args) == 1:
@@ -895,6 +910,7 @@ class DbtAdapter(BaseAdapter):
                     old_schema=base_schema,
                     new_schema=curr_schema,
                     dialect=dialect,
+                    perf_tracking=breaking_perf_tracker,
                 )
 
                 # Make sure that the case of the column names are the same
@@ -917,6 +933,9 @@ class DbtAdapter(BaseAdapter):
                 # TODO: telemetry
                 pass
 
+        breaking_perf_tracker.end_lineage_diff()
+        log_performance("change analysis per node", breaking_perf_tracker.to_dict())
+        breaking_perf_tracker.reset()
         node_diff = diff.get(node_id)
         node_diff.change = change
         return node_diff
@@ -931,7 +950,15 @@ class DbtAdapter(BaseAdapter):
         no_downstream: Optional[bool] = False,
         no_filter: Optional[bool] = False,
     ) -> CllData:
-        cll_tracker = CLLPerformanceTracking()
+        cll_tracker = LineagePerfTracker()
+        cll_tracker.set_params(
+            has_node=node_id is not None,
+            has_column=column is not None,
+            change_analysis=change_analysis,
+            no_cll=no_cll,
+            no_upstream=no_upstream,
+            no_downstream=no_downstream,
+        )
         cll_tracker.start_column_lineage()
 
         manifest = self.curr_manifest
@@ -944,6 +971,8 @@ class DbtAdapter(BaseAdapter):
             lineage_diff = self.get_lineage_diff()
             cll_node_ids = set(lineage_diff.diff.keys())
 
+        cll_tracker.set_init_nodes(len(cll_node_ids))
+
         nodes = {}
         columns = {}
         parent_map = {}
@@ -955,29 +984,41 @@ class DbtAdapter(BaseAdapter):
             cll_node_ids = cll_node_ids.union(find_downstream(cll_node_ids, manifest_dict.get("child_map")))
 
         if not no_cll:
+            allowed_related_nodes = set()
+            for key in ["sources", "nodes", "exposures", "metrics"]:
+                attr = getattr(manifest, key)
+                allowed_related_nodes.update(set(attr.keys()))
+            if hasattr(manifest, "semantic_models"):
+                attr = getattr(manifest, "semantic_models")
+                allowed_related_nodes.update(set(attr.keys()))
             for cll_node_id in cll_node_ids:
-                if (
-                    cll_node_id not in manifest.sources
-                    and cll_node_id not in manifest.nodes
-                    and cll_node_id not in manifest.exposures
-                ):
+                if cll_node_id not in allowed_related_nodes:
                     continue
                 cll_data_one = deepcopy(self.get_cll_cached(cll_node_id, base=False))
+                cll_tracker.increment_cll_nodes()
                 if cll_data_one is None:
                     continue
 
                 nodes[cll_node_id] = cll_data_one.nodes.get(cll_node_id)
-                node_diff = self.get_change_analysis_cached(cll_node_id) if change_analysis else None
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
                 if node_diff is not None:
                     nodes[cll_node_id].change_status = node_diff.change_status
                     if node_diff.change is not None:
                         nodes[cll_node_id].change_category = node_diff.change.category
                 for c_id, c in cll_data_one.columns.items():
                     columns[c_id] = c
-                    if node_diff is not None and node_diff.change is not None and node_diff.change.columns is not None:
-                        column_diff = node_diff.change.columns.get(c.name)
-                        if column_diff:
-                            c.change_status = column_diff
+                    if node_diff is not None:
+                        if node_diff.change_status == "added":
+                            c.change_status = "added"
+                        elif node_diff.change_status == "removed":
+                            c.change_status = "removed"
+                        elif node_diff.change is not None and node_diff.change.columns is not None:
+                            column_diff = node_diff.change.columns.get(c.name)
+                            if column_diff:
+                                c.change_status = column_diff
 
                 for p_id, parents in cll_data_one.parent_map.items():
                     parent_map[p_id] = parents
@@ -987,13 +1028,7 @@ class DbtAdapter(BaseAdapter):
                 cll_node_columns: Dict[str, CllColumn] = {}
 
                 if cll_node_id in manifest.sources:
-                    n = manifest.sources[cll_node_id]
-                    cll_node = CllNode(
-                        id=n.unique_id,
-                        name=n.name,
-                        source_name=n.source_name,
-                        package_name=n.package_name,
-                    )
+                    cll_node = CllNode.build_cll_node(manifest, "sources", cll_node_id)
                     if self.curr_catalog and cll_node_id in self.curr_catalog.sources:
                         cll_node_columns = {
                             column.name: CllColumn(
@@ -1005,15 +1040,7 @@ class DbtAdapter(BaseAdapter):
                             for column in self.curr_catalog.sources[cll_node_id].columns.values()
                         }
                 elif cll_node_id in manifest.nodes:
-                    n = manifest.nodes[cll_node_id]
-                    if n.resource_type not in ["model", "seed", "snapshot"]:
-                        continue
-                    cll_node = CllNode(
-                        id=n.unique_id,
-                        name=n.name,
-                        package_name=n.package_name,
-                        resource_type=n.resource_type,
-                    )
+                    cll_node = CllNode.build_cll_node(manifest, "nodes", cll_node_id)
                     if self.curr_catalog and cll_node_id in self.curr_catalog.nodes:
                         cll_node_columns = {
                             column.name: CllColumn(
@@ -1025,19 +1052,20 @@ class DbtAdapter(BaseAdapter):
                             for column in self.curr_catalog.nodes[cll_node_id].columns.values()
                         }
                 elif cll_node_id in manifest.exposures:
-                    n = manifest.exposures[cll_node_id]
-                    cll_node = CllNode(
-                        id=n.unique_id,
-                        name=n.name,
-                        package_name=n.package_name,
-                        resource_type=n.resource_type,
-                    )
+                    cll_node = CllNode.build_cll_node(manifest, "exposures", cll_node_id)
+                elif hasattr(manifest, "semantic_models") and cll_node_id in manifest.semantic_models:
+                    cll_node = CllNode.build_cll_node(manifest, "semantic_models", cll_node_id)
+                elif cll_node_id in manifest.metrics:
+                    cll_node = CllNode.build_cll_node(manifest, "metrics", cll_node_id)
 
                 if not cll_node:
                     continue
                 nodes[cll_node_id] = cll_node
 
-                node_diff = self.get_change_analysis_cached(cll_node_id) if change_analysis else None
+                node_diff = None
+                if change_analysis:
+                    node_diff = self.get_change_analysis_cached(cll_node_id)
+                    cll_tracker.increment_change_analysis_nodes()
                 if node_diff is not None:
                     cll_node.change_status = node_diff.change_status
                     if node_diff.change is not None:
@@ -1063,7 +1091,19 @@ class DbtAdapter(BaseAdapter):
         if node_id is None and column is None:
             if change_analysis:
                 # If change analysis is requested, we need to find the nodes that have changes
-                for nid in self.get_lineage_diff().diff.keys():
+                lineage_diff = self.get_lineage_diff()
+                for nid, nd in lineage_diff.diff.items():
+                    if nd.change_status == "added":
+                        anchor_node_ids.add(nid)
+                        n = lineage_diff.current["nodes"].get(nid)
+                        n_columns = n.get("columns", {})
+                        for c in n_columns:
+                            anchor_node_ids.add(build_column_key(nid, c))
+                        continue
+                    if nd.change_status == "removed":
+                        extra_node_ids.add(nid)
+                        continue
+
                     node_diff = self.get_change_analysis_cached(nid)
                     if node_diff is not None and node_diff.change is not None:
                         extra_node_ids.add(nid)
@@ -1107,6 +1147,7 @@ class DbtAdapter(BaseAdapter):
         else:
             anchor_node_ids.add(f"{node_id}_{column}")
 
+        cll_tracker.set_anchor_nodes(len(anchor_node_ids))
         result_node_ids = set(anchor_node_ids)
         if not no_upstream:
             result_node_ids = result_node_ids.union(find_upstream(anchor_node_ids, parent_map))
@@ -1122,10 +1163,14 @@ class DbtAdapter(BaseAdapter):
                 node.columns = {
                     k: v for k, v in node.columns.items() if v.id in result_node_ids or v.id in extra_node_ids
                 }
+
+                if change_analysis:
+                    node.impacted = node.id in result_node_ids
+
             parent_map, child_map = filter_dependency_maps(parent_map, child_map, result_node_ids)
 
         cll_tracker.end_column_lineage()
-        cll_tracker.set_total_nodes(len(nodes))
+        cll_tracker.set_total_nodes(len(nodes) + len(columns))
         log_performance("column level lineage", cll_tracker.to_dict())
         cll_tracker.reset()
 
@@ -1144,6 +1189,9 @@ class DbtAdapter(BaseAdapter):
         if node is None:
             return None
 
+        cll_tracker.set_total_nodes(1)
+        cll_tracker.start_column_lineage()
+
         def _apply_all_columns(node: CllNode, transformation_type):
             cll_data = CllData()
             cll_data.nodes[node.id] = node
@@ -1268,6 +1316,10 @@ class DbtAdapter(BaseAdapter):
                     depends_on.add(parent_key)
                 column.transformation_type = c2c_map[name].transformation_type
             cll_data.parent_map[column_id] = set(depends_on)
+
+        cll_tracker.end_column_lineage()
+        log_performance("column level lineage per node", cll_tracker.to_dict())
+        cll_tracker.reset()
         return cll_data
 
     def get_cll_node(self, node_id: str, base: Optional[bool] = False) -> Tuple[Optional[CllNode], list[str]]:
@@ -1279,21 +1331,12 @@ class DbtAdapter(BaseAdapter):
         # model, seed, snapshot
         if node_id in manifest.nodes:
             found = manifest.nodes[node_id]
-            if found.resource_type not in ["model", "seed", "snapshot"]:
-                return None, []
-
             unique_id = found.unique_id
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-                raw_code=found.raw_code,
-            )
+            node = CllNode.build_cll_node(manifest, "nodes", node_id)
             if hasattr(found.depends_on, "nodes"):
                 parent_list = found.depends_on.nodes
 
-            if catalog is not None and unique_id in catalog.nodes:
+            if catalog is not None and node is not None and unique_id in catalog.nodes:
                 columns = {}
                 for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
                     column_id = f"{unique_id}_{col_name}"
@@ -1305,17 +1348,10 @@ class DbtAdapter(BaseAdapter):
         if node_id in manifest.sources:
             found = manifest.sources[node_id]
             unique_id = found.unique_id
-
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-                source_name=found.source_name,
-            )
+            node = CllNode.build_cll_node(manifest, "sources", node_id)
             parent_list = []
 
-            if catalog is not None and unique_id in catalog.sources:
+            if catalog is not None and node is not None and unique_id in catalog.sources:
                 columns = {}
                 for col_name, col_metadata in catalog.sources[unique_id].columns.items():
                     column_id = f"{unique_id}_{col_name}"
@@ -1326,13 +1362,19 @@ class DbtAdapter(BaseAdapter):
         # exposure
         if node_id in manifest.exposures:
             found = manifest.exposures[node_id]
+            node = CllNode.build_cll_node(manifest, "exposures", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
 
-            node = CllNode(
-                id=found.unique_id,
-                name=found.name,
-                package_name=found.package_name,
-                resource_type=found.resource_type,
-            )
+        if hasattr(manifest, "semantic_models") and node_id in manifest.semantic_models:
+            found = manifest.semantic_models[node_id]
+            node = CllNode.build_cll_node(manifest, "semantic_models", node_id)
+            if hasattr(found.depends_on, "nodes"):
+                parent_list = found.depends_on.nodes
+
+        if node_id in manifest.metrics:
+            found = manifest.metrics[node_id]
+            node = CllNode.build_cll_node(manifest, "metrics", node_id)
             if hasattr(found.depends_on, "nodes"):
                 parent_list = found.depends_on.nodes
 
@@ -1558,7 +1600,7 @@ class DbtAdapter(BaseAdapter):
             if not os.path.isfile(path):
                 return None
 
-            with open(path, "r") as f:
+            with open(path, "r", encoding="utf-8") as f:
                 json_content = f.read()
                 return json.loads(json_content)
 

recce/artifact.py

@@ -40,7 +40,7 @@ def verify_artifacts_path(target_path: str) -> bool:
 
 
 def parse_dbt_version(file_path: str) -> str:
-    with open(file_path, "r") as f:
+    with open(file_path, "r", encoding="utf-8") as f:
         data = json.load(f)
 
     dbt_version = data.get("metadata", {}).get("dbt_version", None)
@@ -80,6 +80,64 @@ def archive_artifacts(target_path: str) -> (str, str):
     return artifacts_tar_gz_path, dbt_version
 
 
+def upload_artifacts_to_session(target_path: str, session_id: str, token: str, debug: bool = False):
+    """Upload dbt artifacts to a specific session ID in Recce Cloud."""
+    console = Console()
+    if verify_artifacts_path(target_path) is False:
+        console.print(f"[[red]Error[/red]] Invalid target path: {target_path}")
+        console.print("Please provide a valid target path containing manifest.json and catalog.json.")
+        return 1
+
+    manifest_path = os.path.join(target_path, "manifest.json")
+    catalog_path = os.path.join(target_path, "catalog.json")
+
+    # get the adapter type from the manifest file
+    with open(manifest_path, "r", encoding="utf-8") as f:
+        manifest_data = json.load(f)
+        adapter_type = manifest_data.get("metadata", {}).get("adapter_type")
+        if adapter_type is None:
+            raise Exception("Failed to parse adapter type from manifest.json")
+
+    recce_cloud = RecceCloud(token)
+
+    session = recce_cloud.get_session(session_id)
+
+    org_id = session.get("org_id")
+    if org_id is None:
+        raise Exception(f"Session ID {session_id} does not belong to any organization.")
+
+    project_id = session.get("project_id")
+    if project_id is None:
+        raise Exception(f"Session ID {session_id} does not belong to any project.")
+
+    # Get the presigned URL for uploading the artifacts using session ID
+    console.print(f'Uploading artifacts for session ID "{session_id}"')
+    presigned_urls = recce_cloud.get_upload_urls_by_session_id(org_id, project_id, session_id)
+    if debug:
+        console.rule("Debug information", style="blue")
+        console.print(f"Org ID: {org_id}")
+        console.print(f"Project ID: {project_id}")
+        console.print(f"Session ID: {session_id}")
+        console.print(f"Manifest path: {presigned_urls['manifest_url']}")
+        console.print(f"Catalog path: {presigned_urls['catalog_url']}")
+        console.print(f"Adapter type: {adapter_type}")
+
+    # Upload the compressed artifacts (no password needed for session uploads)
+    console.print(f'Uploading manifest from path "{manifest_path}"')
+    response = requests.put(presigned_urls["manifest_url"], data=open(manifest_path, "rb").read())
+    if response.status_code != 200 and response.status_code != 204:
+        raise Exception(response.text)
+    console.print(f'Uploading catalog from path "{catalog_path}"')
+    response = requests.put(presigned_urls["catalog_url"], data=open(catalog_path, "rb").read())
+    if response.status_code != 200 and response.status_code != 204:
+        raise Exception(response.text)
+
+    # Update the session metadata
+    recce_cloud.update_session(org_id, project_id, session_id, adapter_type)
+
+    return 0
+
+
 def upload_dbt_artifacts(target_path: str, branch: str, token: str, password: str, debug: bool = False):
     console = Console()
     if verify_artifacts_path(target_path) is False:
@@ -100,7 +158,7 @@ def upload_dbt_artifacts(target_path: str, branch: str, token: str, password: st
     metadata = {"commit": sha, "dbt_version": dbt_version}
 
     # Get the presigned URL for uploading the artifacts
-    presigned_url = RecceCloud(token).get_presigned_url(
+    presigned_url = RecceCloud(token).get_presigned_url_by_github_repo(
         method=PresignedUrlMethod.UPLOAD,
         repository=repo,
         artifact_name="dbt_artifacts.tar.gz",
@@ -145,7 +203,7 @@ def download_dbt_artifacts(
     sha = None
     dbt_version = None
 
-    presigned_url, tags = RecceCloud(token).get_download_presigned_url_with_tags(
+    presigned_url, tags = RecceCloud(token).get_download_presigned_url_by_github_repo_with_tags(
         repository=repo,
         artifact_name="dbt_artifacts.tar.gz",
         branch=branch,
@@ -191,3 +249,18 @@ def download_dbt_artifacts(
     except FileNotFoundError:
         pass
     return 0
+
+
+def delete_dbt_artifacts(branch: str, token: str, debug: bool = False):
+    """Delete dbt artifacts from a specific branch in Recce Cloud."""
+    console = Console()
+    repo = hosting_repo()
+
+    if debug:
+        console.rule("Debug information", style="blue")
+        console.print(f"Git Branch: {branch}")
+        console.print(f"GitHub repository: {repo}")
+
+    console.print(f'Deleting dbt artifacts from branch: "{branch}"')
+
+    RecceCloud(token).purge_artifacts(repo, branch=branch)