recce-nightly 1.3.0.20250507__py3-none-any.whl → 1.4.0.20250514__py3-none-any.whl

recce/util/breaking.py

@@ -3,15 +3,15 @@ from dataclasses import dataclass
 from typing import Optional
 
 import sqlglot.expressions as exp
-from sqlglot import parse_one, Dialect
+from sqlglot import Dialect, parse_one
 from sqlglot.errors import SqlglotError
-from sqlglot.optimizer import traverse_scope, Scope
+from sqlglot.optimizer import Scope, traverse_scope
 from sqlglot.optimizer.qualify import qualify
 
-from recce.models.types import NodeChange, ChangeStatus
+from recce.models.types import ChangeStatus, NodeChange
 
-CHANGE_CATEGORY_UNKNOWN = NodeChange(category='unknown')
-CHANGE_CATEGORY_BREAKING = NodeChange(category='breaking')
+CHANGE_CATEGORY_UNKNOWN = NodeChange(category="unknown")
+CHANGE_CATEGORY_BREAKING = NodeChange(category="breaking")
 
 
 @dataclass
@@ -48,11 +48,11 @@ class BreakingPerformanceTracking:
 
     def to_dict(self):
         return {
-            'lineage_diff_elapsed_ms': self.lineage_diff_elapsed,
-            'modified_nodes': self.modified_nodes,
-            'sqlglot_error_nodes': self.sqlglot_error_nodes,
-            'other_error_nodes': self.other_error_nodes,
-            'checkpoints': self.checkpoints,
+            "lineage_diff_elapsed_ms": self.lineage_diff_elapsed,
+            "modified_nodes": self.modified_nodes,
+            "sqlglot_error_nodes": self.sqlglot_error_nodes,
+            "other_error_nodes": self.other_error_nodes,
+            "checkpoints": self.checkpoints,
         }
 
     def reset(self):
@@ -64,33 +64,29 @@ class BreakingPerformanceTracking:
         self.checkpoints = {}
 
 
-def _diff_select_scope(
-    old_scope: Scope,
-    new_scope: Scope,
-    scope_changes_map: dict[Scope, NodeChange]
-) -> NodeChange:
-    assert old_scope.expression.key == 'select'
-    assert new_scope.expression.key == 'select'
+def _diff_select_scope(old_scope: Scope, new_scope: Scope, scope_changes_map: dict[Scope, NodeChange]) -> NodeChange:
+    assert old_scope.expression.key == "select"
+    assert new_scope.expression.key == "select"
 
-    change_category = 'non_breaking'
+    change_category = "non_breaking"
     changed_columns = {}
 
     # check if the upstream scopes is not breaking
     for source_name, source in new_scope.sources.items():
         if scope_changes_map.get(source) is not None:
             chanage = scope_changes_map[source]
-            if chanage.category == 'breaking':
-                change_category = 'breaking'
+            if chanage.category == "breaking":
+                change_category = "breaking"
 
     # check if non-select expressions are the same
     old_select = old_scope.expression  # type: exp.Select
     new_select = new_scope.expression  # type: exp.Select
     for arg_key in old_select.args.keys() | new_select.args.keys():
-        if arg_key in ['expressions', 'with', 'from']:
+        if arg_key in ["expressions", "with", "from"]:
             continue
 
         if old_select.args.get(arg_key) != new_select.args.get(arg_key):
-            change_category = 'breaking'
+            change_category = "breaking"
 
     def source_column_change_status(ref_column: exp.Column) -> Optional[ChangeStatus]:
         table_name = ref_column.table
@@ -108,9 +104,10 @@ def _diff_select_scope(
     # selects
     old_column_map = {projection.alias_or_name: projection for projection in old_select.selects}
     new_column_map = {projection.alias_or_name: projection for projection in new_select.selects}
-    is_distinct = new_select.args.get('distinct') is not None
+    is_distinct = new_select.args.get("distinct") is not None
+
+    for column_name in old_column_map.keys() | new_column_map.keys():
 
-    for column_name in (old_column_map.keys() | new_column_map.keys()):
         def _has_udtf(expr: exp.Expression) -> bool:
             return expr.find(exp.UDTF) is not None
 
@@ -124,116 +121,112 @@ def _diff_select_scope(
         new_column = new_column_map.get(column_name)
         if old_column is None:
             if is_distinct:
-                change_category = 'breaking'
+                change_category = "breaking"
             elif _has_udtf(new_column):
-                change_category = 'breaking'
+                change_category = "breaking"
 
-            changed_columns[column_name] = 'added'
+            changed_columns[column_name] = "added"
         elif new_column is None:
             if is_distinct:
-                change_category = 'breaking'
+                change_category = "breaking"
             elif _has_udtf(old_column):
-                change_category = 'breaking'
+                change_category = "breaking"
 
-            changed_columns[column_name] = 'removed'
-            if change_category != 'breaking':
-                change_category = 'partial_breaking'
+            changed_columns[column_name] = "removed"
+            if change_category != "breaking":
+                change_category = "partial_breaking"
         elif old_column != new_column:
             if is_distinct:
-                change_category = 'breaking'
+                change_category = "breaking"
             elif _has_udtf(old_column) and _has_udtf(new_column):
-                change_category = 'breaking'
+                change_category = "breaking"
             elif _has_aggregate(old_column) != _has_aggregate(new_column):
-                change_category = 'breaking'
+                change_category = "breaking"
 
-            changed_columns[column_name] = 'modified'
-            if change_category != 'breaking':
-                change_category = 'partial_breaking'
+            changed_columns[column_name] = "modified"
+            if change_category != "breaking":
+                change_category = "partial_breaking"
         else:
             if _has_star(new_column):
                 for source_name, (_, source) in new_scope.selected_sources.items():
                     change = scope_changes_map.get(source)
                     if change is not None:
-                        if change.category == 'breaking':
-                            change_category = 'breaking'
+                        if change.category == "breaking":
+                            change_category = "breaking"
                         for sub_column_name in change.columns.keys():
                             column_change_status = change.columns[sub_column_name]
                             changed_columns[sub_column_name] = column_change_status
-                            if change_category != 'breaking' and column_change_status in ['removed', 'modified']:
-                                change_category = 'partial_breaking'
+                            if change_category != "breaking" and column_change_status in ["removed", "modified"]:
+                                change_category = "partial_breaking"
                 continue
 
             ref_columns = new_column.find_all(exp.Column)
             for ref_column in ref_columns:
                 if source_column_change_status(ref_column) is not None:
                     if is_distinct:
-                        change_category = 'breaking'
+                        change_category = "breaking"
                     elif _has_udtf(new_column):
-                        change_category = 'breaking'
+                        change_category = "breaking"
 
-                    if change_category != 'breaking':
-                        change_category = 'partial_breaking'
-                    changed_columns[column_name] = 'modified'
+                    if change_category != "breaking":
+                        change_category = "partial_breaking"
+                    changed_columns[column_name] = "modified"
 
     def selected_column_change_status(ref_column: exp.Column) -> Optional[ChangeStatus]:
         column_name = ref_column.name
         return changed_columns.get(column_name)
 
     # joins clause: Reference the source columns
-    if new_select.args.get('joins'):
-        joins = new_select.args.get('joins')
+    if new_select.args.get("joins"):
+        joins = new_select.args.get("joins")
         for join in joins:
             if isinstance(join, exp.Join):
                 for ref_column in join.find_all(exp.Column):
                     if source_column_change_status(ref_column) is not None:
-                        change_category = 'breaking'
+                        change_category = "breaking"
 
     # where clauses: Reference the source columns
-    if new_select.args.get('where'):
-        where = new_select.args.get('where')
+    if new_select.args.get("where"):
+        where = new_select.args.get("where")
         if isinstance(where, exp.Where):
             for ref_column in where.find_all(exp.Column):
                 if source_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
 
     # group by clause: Reference the source columns, column index
-    if new_select.args.get('group'):
-        group = new_select.args.get('group')
+    if new_select.args.get("group"):
+        group = new_select.args.get("group")
         if isinstance(group, exp.Group):
             for ref_column in group.find_all(exp.Column):
                 if source_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
 
     # having clause: Reference the source columns, selected columns
-    if new_select.args.get('having'):
-        having = new_select.args.get('having')
+    if new_select.args.get("having"):
+        having = new_select.args.get("having")
         if isinstance(having, exp.Having):
             for ref_column in having.find_all(exp.Column):
                 if source_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
                 elif selected_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
 
     # order by clause: Reference the source columns, selected columns, column index
-    if new_select.args.get('order'):
-        order = new_select.args.get('order')
+    if new_select.args.get("order"):
+        order = new_select.args.get("order")
         if isinstance(order, exp.Order):
             for ref_column in order.find_all(exp.Column):
                 if source_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
                 elif selected_column_change_status(ref_column) is not None:
-                    change_category = 'breaking'
+                    change_category = "breaking"
 
     return NodeChange(category=change_category, columns=changed_columns)
 
 
-def _diff_union_scope(
-    old_scope: Scope,
-    new_scope: Scope,
-    scope_changes_map: dict[Scope, NodeChange]
-) -> NodeChange:
-    assert old_scope.expression.key == 'union'
-    assert new_scope.expression.key == 'union'
+def _diff_union_scope(old_scope: Scope, new_scope: Scope, scope_changes_map: dict[Scope, NodeChange]) -> NodeChange:
+    assert old_scope.expression.key == "union"
+    assert new_scope.expression.key == "union"
     assert len(old_scope.union_scopes) == len(new_scope.union_scopes)
     assert new_scope.union_scopes is not None
     assert len(new_scope.union_scopes) > 0
@@ -244,11 +237,11 @@ def _diff_union_scope(
 
     for sub_scope in new_scope.union_scopes[1:]:
         result_right = scope_changes_map.get(sub_scope)
-        if change_category == 'partial_breaking':
-            if result_right.category in ['breaking']:
+        if change_category == "partial_breaking":
+            if result_right.category in ["breaking"]:
                 change_category = result_right.category
-        elif change_category == 'non_breaking':
-            if result_right.category in ['breaking', 'partial_breaking']:
+        elif change_category == "non_breaking":
+            if result_right.category in ["breaking", "partial_breaking"]:
                 change_category = result_right.category
         for column_name, column_change_status in result_right.columns.items():
             changed_columns[column_name] = column_change_status
@@ -265,7 +258,7 @@ def parse_change_category(
     perf_tracking: BreakingPerformanceTracking = None,
 ) -> NodeChange:
     if old_sql == new_sql:
-        return NodeChange(category='non_breaking')
+        return NodeChange(category="non_breaking")
 
     try:
         dialect = Dialect.get(dialect)
@@ -294,31 +287,31 @@ def parse_change_category(
     old_scopes = traverse_scope(old_exp)
     new_scopes = traverse_scope(new_exp)
     if len(old_scopes) != len(new_scopes):
-        return NodeChange(category='breaking', columns={})
+        return NodeChange(category="breaking", columns={})
 
     scope_changes_map = {}
     for old_scope, new_scope in zip(old_scopes, new_scopes):
         if old_scope.expression.key != new_scope.expression.key:
-            scope_changes_map[new_scope] = NodeChange(category='breaking')
+            scope_changes_map[new_scope] = NodeChange(category="breaking")
             continue
         if old_scope == new_scope:
-            scope_changes_map[new_scope] = NodeChange(category='non_breaking')
+            scope_changes_map[new_scope] = NodeChange(category="non_breaking")
             continue
 
         scope_type = old_scope.expression.key
-        if scope_type == 'select':
+        if scope_type == "select":
             # CTE, Subquery, Root
             result = _diff_select_scope(old_scope, new_scope, scope_changes_map)
-        elif scope_type == 'union':
+        elif scope_type == "union":
             # Union
             result = _diff_union_scope(old_scope, new_scope, scope_changes_map)
         else:
             if old_scope.expression != new_scope.expression:
-                result = NodeChange(category='breaking', columns={})
+                result = NodeChange(category="breaking", columns={})
             else:
-                result = NodeChange(category='non_breaking', columns={})
+                result = NodeChange(category="non_breaking", columns={})
 
-        if result.category == 'unknown':
+        if result.category == "unknown":
             return result
 
         scope_changes_map[new_scope] = result

recce/util/cll.py

@@ -2,10 +2,21 @@ import time
 from dataclasses import dataclass
 from typing import Dict, List, Literal
 
-from sqlglot import parse_one, Dialect
-from sqlglot.errors import SqlglotError, OptimizeError
-from sqlglot.expressions import Column, Alias, Func, Binary, Paren, Case, Expression, If, Union, Intersect
-from sqlglot.optimizer import traverse_scope, Scope
+from sqlglot import Dialect, parse_one
+from sqlglot.errors import OptimizeError, SqlglotError
+from sqlglot.expressions import (
+    Alias,
+    Binary,
+    Case,
+    Column,
+    Expression,
+    Func,
+    If,
+    Intersect,
+    Paren,
+    Union,
+)
+from sqlglot.optimizer import Scope, traverse_scope
 from sqlglot.optimizer.qualify import qualify
 
 from recce.exceptions import RecceException
@@ -50,11 +61,11 @@ class CLLPerformanceTracking(metaclass=SingletonMeta):
 
     def to_dict(self):
         return {
-            'lineage_elapsed_ms': self.lineage_elapsed,
-            'column_lineage_elapsed_ms': self.column_lineage_elapsed,
-            'total_nodes': self.total_nodes,
-            'sqlglot_error_nodes': self.sqlglot_error_nodes,
-            'other_error_nodes': self.other_error_nodes
+            "lineage_elapsed_ms": self.lineage_elapsed,
+            "column_lineage_elapsed_ms": self.column_lineage_elapsed,
+            "total_nodes": self.total_nodes,
+            "sqlglot_error_nodes": self.sqlglot_error_nodes,
+            "other_error_nodes": self.other_error_nodes,
         }
 
     def reset(self):
@@ -76,7 +87,7 @@ class ColumnLevelDependsOn:
 
 @dataclass
 class ColumnLevelDependencyColumn:
-    type: Literal['source', 'passthrough', 'renamed', 'derived']
+    type: Literal["source", "passthrough", "renamed", "derived"]
     depends_on: List[ColumnLevelDependsOn]
 
 
@@ -93,10 +104,7 @@ def _cll_expression(expression, table_alias_map) -> ColumnLevelDependencyColumn:
         else:
             table = table_alias_map.get(alias, alias)
 
-        return ColumnLevelDependencyColumn(
-            type='passthrough',
-            depends_on=[ColumnLevelDependsOn(table, column.name)]
-        )
+        return ColumnLevelDependencyColumn(type="passthrough", depends_on=[ColumnLevelDependsOn(table, column.name)])
     elif isinstance(expression, Paren):
         return _cll_expression(expression.this, table_alias_map)
     elif isinstance(expression, Binary):
@@ -107,31 +115,31 @@ def _cll_expression(expression, table_alias_map) -> ColumnLevelDependencyColumn:
         if expression.right:
             depends_on_right = _cll_expression(expression.right, table_alias_map).depends_on
             depends_on.extend(depends_on_right)
-        type = 'derived' if depends_on else 'source'
+        type = "derived" if depends_on else "source"
         return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
     elif isinstance(expression, Case):
-        ifs = expression.args['ifs']
-        default = expression.args['default']
+        ifs = expression.args["ifs"]
+        default = expression.args["default"]
         depends_on = []
         for expr in ifs:
             depends_on_one = _cll_expression(expr, table_alias_map).depends_on
             depends_on.extend(depends_on_one)
         if default is not None:
             depends_on.extend(_cll_expression(default, table_alias_map).depends_on)
-        type = 'derived' if depends_on else 'source'
+        type = "derived" if depends_on else "source"
         return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
     elif isinstance(expression, If):
         depends_on = []
         if expression.this:
             depends_on_one = _cll_expression(expression.this, table_alias_map).depends_on
             depends_on.extend(depends_on_one)
-        if expression.args.get('true'):
-            depends_on_one = _cll_expression(expression.args.get('true'), table_alias_map).depends_on
+        if expression.args.get("true"):
+            depends_on_one = _cll_expression(expression.args.get("true"), table_alias_map).depends_on
             depends_on.extend(depends_on_one)
-        if expression.args.get('false'):
-            depends_on_one = _cll_expression(expression.args.get('false'), table_alias_map).depends_on
+        if expression.args.get("false"):
+            depends_on_one = _cll_expression(expression.args.get("false"), table_alias_map).depends_on
             depends_on.extend(depends_on_one)
-        type = 'derived' if depends_on else 'source'
+        type = "derived" if depends_on else "source"
         return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
     elif isinstance(expression, Func):
         if expression.expressions:
@@ -139,28 +147,28 @@ def _cll_expression(expression, table_alias_map) -> ColumnLevelDependencyColumn:
             for expr in expression.expressions:
                 depends_on_one = _cll_expression(expr, table_alias_map).depends_on
                 depends_on.extend(depends_on_one)
-            type = 'derived' if depends_on else 'source'
+            type = "derived" if depends_on else "source"
             return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
         if expression.this:
             depends_on = _cll_expression(expression.this, table_alias_map).depends_on
-            type = 'derived' if depends_on else 'source'
+            type = "derived" if depends_on else "source"
             return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
 
-        return ColumnLevelDependencyColumn(type='source', depends_on=[])
+        return ColumnLevelDependencyColumn(type="source", depends_on=[])
     elif expression.this and isinstance(expression.this, Expression):
         depends_on = _cll_expression(expression.this, table_alias_map).depends_on
-        type = 'derived' if depends_on else 'source'
+        type = "derived" if depends_on else "source"
         return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
     elif expression.expressions:
         depends_on = []
         for expr in expression.expressions:
             depends_on_one = _cll_expression(expr, table_alias_map).depends_on
             depends_on.extend(depends_on_one)
-        type = 'derived' if depends_on else 'source'
+        type = "derived" if depends_on else "source"
         return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
     else:
         depends_on = []
-        return ColumnLevelDependencyColumn(type='source', depends_on=depends_on)
+        return ColumnLevelDependencyColumn(type="source", depends_on=depends_on)
 
 
 def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn]:
@@ -177,24 +185,21 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
     try:
         expression = parse_one(sql, dialect=dialect)
     except SqlglotError as e:
-        raise RecceException(f'Failed to parse SQL: {str(e)}')
+        raise RecceException(f"Failed to parse SQL: {str(e)}")
 
     try:
         expression = qualify(expression, schema=schema, dialect=dialect)
     except OptimizeError as e:
-        raise RecceException(f'Failed to optimize SQL: {str(e)}')
+        raise RecceException(f"Failed to optimize SQL: {str(e)}")
     except SqlglotError as e:
-        raise RecceException(f'Failed to qualify SQL: {str(e)}')
+        raise RecceException(f"Failed to qualify SQL: {str(e)}")
 
     result = {}
     global_lineage = {}
     for scope in traverse_scope(expression):
         scope_lineage = {}
 
-        table_alias_map = {
-            t.alias_or_name: t.name
-            for t in scope.tables
-        }
+        table_alias_map = {t.alias_or_name: t.name for t in scope.tables}
 
         if isinstance(scope.expression, Union) or isinstance(scope.expression, Intersect):
             for union_scope in scope.union_scopes:
@@ -203,7 +208,7 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
                         scope_lineage[k] = v
                     else:
                         scope_lineage[k].depends_on.extend(v.depends_on)
-                        scope_lineage[k].type = 'derived'
+                        scope_lineage[k].type = "derived"
         else:
             for select in scope.expression.selects:
                 # instance of Column
@@ -218,14 +223,14 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
                     col_expression = alias.this
                     column_cll = _cll_expression(col_expression, table_alias_map)
                     if (
-                        column_cll and
-                        column_cll.type == 'passthrough' and
-                        column_cll.depends_on[0].column != alias.alias_or_name
+                        column_cll
+                        and column_cll.type == "passthrough"
+                        and column_cll.depends_on[0].column != alias.alias_or_name
                     ):
-                        column_cll.type = 'renamed'
+                        column_cll.type = "renamed"
                 else:
                     # 'select 1'
-                    column_cll = ColumnLevelDependencyColumn(type='source', depends_on=[])
+                    column_cll = ColumnLevelDependencyColumn(type="source", depends_on=[])
 
                 cte_type = None
                 flatten_col_depends_on = []
@@ -258,24 +263,24 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
                 dedup_col_depends_on = []
                 dedup_set = set()
                 for col_dep in flatten_col_depends_on:
-                    node_col = col_dep.node + '.' + col_dep.column
+                    node_col = col_dep.node + "." + col_dep.column
                     if node_col not in dedup_set:
                         dedup_col_depends_on.append(col_dep)
                         dedup_set.add(node_col)
 
                 # transformation type
                 type = column_cll.type
-                if type == 'derived':
+                if type == "derived":
                     if len(dedup_col_depends_on) == 0:
-                        type = 'source'
+                        type = "source"
                     else:
                         # keep current scope type
                         pass
                 elif cte_type is not None:
                     if len(dedup_col_depends_on) > 1:
-                        type = 'derived'
+                        type = "derived"
                     elif len(dedup_col_depends_on) == 0:
-                        type = 'source'
+                        type = "source"
                     else:
                         if isinstance(select, Column):
                             type = cte_type
@@ -284,13 +289,12 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
                             if column_cll.depends_on[0].column == alias.alias_or_name:
                                 type = cte_type
                             else:
-                                type = 'renamed' if cte_type == 'passthrough' else cte_type
+                                type = "renamed" if cte_type == "passthrough" else cte_type
                         else:
-                            type = 'source'
+                            type = "source"
 
                 scope_lineage[select.alias_or_name] = ColumnLevelDependencyColumn(
-                    type=type,
-                    depends_on=dedup_col_depends_on
+                    type=type, depends_on=dedup_col_depends_on
                 )
 
         global_lineage[scope] = scope_lineage