recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/util/lineage.py

@@ -1,34 +1,83 @@
-def find_upstream(node, parent_map):
+from typing import Dict, Iterable, Set, Tuple
+
+from recce.models.types import CllColumn, CllNode
+
+
+def find_upstream(node_ids: Iterable, parent_map):
     visited = set()
     upstream = set()
 
-    def dfs(current):
+    stack = list(node_ids)
+    while stack:
+        current = stack.pop()
         if current in visited:
-            return
+            continue
         visited.add(current)
-
         parents = parent_map.get(current, [])
         for parent in parents:
-            upstream.add(parent)
-            dfs(parent)
-
-    dfs(node)
+            if parent not in upstream:
+                upstream.add(parent)
+                stack.append(parent)
     return upstream
 
 
-def find_downstream(node, child_map):
+def find_downstream(node_ids: Iterable, child_map):
     visited = set()
     downstream = set()
 
-    def dfs(current):
+    stack = list(node_ids)
+    while stack:
+        current = stack.pop()
         if current in visited:
-            return
+            continue
         visited.add(current)
-
         children = child_map.get(current, [])
         for child in children:
-            downstream.add(child)
-            dfs(child)
-
-    dfs(node)
+            if child not in downstream:
+                downstream.add(child)
+                stack.append(child)
     return downstream
+
+
+def find_column_dependencies(node_column_id: str, parent_map: Dict, child_map: Dict) -> Tuple[Set, Set]:
+    upstream_cols = find_upstream([node_column_id], parent_map)
+    downstream_cols = find_downstream([node_column_id], child_map)
+    return upstream_cols, downstream_cols
+
+
+def filter_lineage_vertices(
+    lineage_nodes: Dict[str, CllNode], lineage_columns: Dict[str, CllColumn], relevant_columns: Set[str]
+) -> Tuple[Dict[str, CllNode], Dict[str, CllColumn]]:
+    nodes = {}
+    columns = {}
+
+    for node_id, node in lineage_nodes.items():
+        if node_id in relevant_columns:
+            nodes[node_id] = node
+
+    for col_id, column in lineage_columns.items():
+        if col_id in relevant_columns:
+            columns[col_id] = column
+
+    return nodes, columns
+
+
+def filter_dependency_maps(
+    parent_map: Dict, child_map: Dict, relevant_ids: Set
+) -> Tuple[Dict[str, Set], Dict[str, Set]]:
+    p_map = {}
+    c_map = {}
+    for node_id, parents in parent_map.items():
+        if node_id in relevant_ids:
+            p_map[node_id] = {p for p in parents if p in relevant_ids}
+
+    for node_id, children in child_map.items():
+        if node_id in relevant_ids:
+            c_map[node_id] = {c for c in children if c in relevant_ids}
+
+    return p_map, c_map
+
+
+def build_column_key(node_id: str, column_name: str) -> str:
+    """Build a unique column key from node name and column name."""
+    return f"{node_id}_{column_name}"

recce/util/logger.py

@@ -16,7 +16,7 @@ class CustomFormatter(logging.Formatter):
         logging.INFO: green + format + reset,
         logging.WARNING: yellow + format + reset,
         logging.ERROR: red + format + reset,
-        logging.CRITICAL: bold_red + format + reset
+        logging.CRITICAL: bold_red + format + reset,
     }
 
     def format(self, record):

recce/util/onboarding_state.py

@@ -0,0 +1,45 @@
+from enum import Enum
+from typing import Union
+
+from recce.util.recce_cloud import (
+    get_recce_cloud_onboarding_state,
+    set_recce_cloud_onboarding_state,
+)
+
+
+class OnboardingState(Enum):
+    SIGNUP_SUCCESSFUL = "signup_successful"
+    LAUNCHED_WITH_TOKEN = "launched_with_token"
+    CONFIGURE_TWO_ENV = "configure_two_env"
+
+
+def update_onboarding_state(api_token: Union[str, bool, None], is_single_env: bool) -> OnboardingState:
+    if api_token:
+        # existing onboarding_state values -> new, launched, launched_with_two_envs
+        #   new -> launched
+        #   new -> launched_with_two_envs
+        #   launched -> launched_with_two_envs
+        cloud_onboarding_state = get_recce_cloud_onboarding_state(api_token)
+
+        if cloud_onboarding_state == OnboardingState.SIGNUP_SUCCESSFUL.value:
+            # User has an API Token and is a "new" user
+            if is_single_env:
+                # Mark the onboarding state as "launched" if the user is new
+                set_recce_cloud_onboarding_state(api_token, OnboardingState.LAUNCHED_WITH_TOKEN.value)
+                return OnboardingState.LAUNCHED_WITH_TOKEN
+            else:
+                set_recce_cloud_onboarding_state(api_token, OnboardingState.CONFIGURE_TWO_ENV.value)
+                return OnboardingState.CONFIGURE_TWO_ENV
+        elif cloud_onboarding_state == OnboardingState.LAUNCHED_WITH_TOKEN.value:
+            # User has an API Token and has Two Environments
+            if is_single_env:
+                # Just return the current state
+                return OnboardingState.LAUNCHED_WITH_TOKEN
+            else:
+                set_recce_cloud_onboarding_state(api_token, OnboardingState.CONFIGURE_TWO_ENV.value)
+                return OnboardingState.CONFIGURE_TWO_ENV
+        elif cloud_onboarding_state == OnboardingState.CONFIGURE_TWO_ENV.value:
+            # Just return the current state
+            return OnboardingState.CONFIGURE_TWO_ENV
+
+    return OnboardingState.SIGNUP_SUCCESSFUL

recce/util/perf_tracking.py

@@ -0,0 +1,85 @@
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class LineagePerfTracker:
+    lineage_start = None
+    lineage_elapsed = None
+    column_lineage_start = None
+    column_lineage_elapsed = None
+
+    total_nodes = None
+    init_nodes = None
+    cll_nodes = 0
+    change_analysis_nodes = 0
+    anchor_nodes = None
+
+    params = None
+
+    def start_lineage(self):
+        self.lineage_start = time.perf_counter_ns()
+
+    def end_lineage(self):
+        if self.lineage_start is None:
+            return
+        self.lineage_elapsed = (time.perf_counter_ns() - self.lineage_start) / 1000000
+
+    def start_column_lineage(self):
+        self.column_lineage_start = time.perf_counter_ns()
+
+    def end_column_lineage(self):
+        if self.column_lineage_start is None:
+            return
+        self.column_lineage_elapsed = (time.perf_counter_ns() - self.column_lineage_start) / 1000000
+
+    def set_total_nodes(self, total_nodes):
+        self.total_nodes = total_nodes
+
+    def set_init_nodes(self, init_nodes):
+        self.init_nodes = init_nodes
+
+    def set_anchor_nodes(self, anchor_nodes):
+        self.anchor_nodes = anchor_nodes
+
+    def increment_cll_nodes(self):
+        self.cll_nodes += 1
+
+    def increment_change_analysis_nodes(self):
+        self.change_analysis_nodes += 1
+
+    def set_params(self, has_node, has_column, change_analysis, no_cll, no_upstream, no_downstream):
+        self.params = {
+            "has_node": has_node,
+            "has_column": has_column,
+            "change_analysis": change_analysis,
+            "no_cll": no_cll,
+            "no_upstream": no_upstream,
+            "no_downstream": no_downstream,
+        }
+
+    def to_dict(self):
+        return {
+            "lineage_elapsed_ms": self.lineage_elapsed,
+            "column_lineage_elapsed_ms": self.column_lineage_elapsed,
+            "total_nodes": self.total_nodes,
+            "init_nodes": self.init_nodes,
+            "cll_nodes": self.cll_nodes,
+            "change_analysis_nodes": self.change_analysis_nodes,
+            "anchor_nodes": self.anchor_nodes,
+            "params": self.params,
+        }
+
+    def reset(self):
+        self.lineage_start = None
+        self.lineage_elapsed = None
+        self.column_lineage_start = None
+        self.column_lineage_elapsed = None
+
+        self.total_nodes = None
+        self.init_nodes = None
+        self.change_analysis_nodes = 0
+        self.cll_nodes = 0
+        self.anchor_nodes = 0
+
+        self.params = None