recce-nightly 1.4.0.20250519__py3-none-any.whl → 1.5.0.20250521__py3-none-any.whl

recce/util/cll.py

@@ -1,21 +1,10 @@
 import time
 from dataclasses import dataclass
-from typing import Dict, List, Literal
+from typing import Dict, List, Literal, Optional
 
+import sqlglot.expressions as exp
 from sqlglot import Dialect, parse_one
 from sqlglot.errors import OptimizeError, SqlglotError
-from sqlglot.expressions import (
-    Alias,
-    Binary,
-    Case,
-    Column,
-    Expression,
-    Func,
-    If,
-    Intersect,
-    Paren,
-    Union,
-)
 from sqlglot.optimizer import Scope, traverse_scope
 from sqlglot.optimizer.qualify import qualify
 
@@ -91,94 +80,231 @@ class ColumnLevelDependencyColumn:
     depends_on: List[ColumnLevelDependsOn]
 
 
-def _cll_expression(expression, table_alias_map) -> ColumnLevelDependencyColumn:
+@dataclass()
+class CllResult:
+    # Model to column dependencies
+    depends_on: List[ColumnLevelDependsOn]
+
+    # Column to column dependencies
+    columns: Dict[str, ColumnLevelDependencyColumn]
+
+
+def _cll_column(proj, table_alias_map) -> ColumnLevelDependencyColumn:
     # given an expression, return the columns depends on
     # [{node: table, column: column}, ...]
+    type = "source"
+    depends_on: List[ColumnLevelDependsOn] = []
+
+    # instance of Column
+    if isinstance(proj, exp.Alias):
+        # 'select a as b'
+        # 'select CURRENT_TIMESTAMP() as create_at'
+        root = proj.this
+
+    for expression in root.walk(bfs=False):
+        if isinstance(expression, exp.Column):
+            column = expression
+            alias = column.table
+
+            if alias is None:
+                table = next(iter(table_alias_map.values()))
+            else:
+                table = table_alias_map.get(alias, alias)
+            depends_on.append(ColumnLevelDependsOn(table, column.name))
+            if type == "source":
+                type = "passthrough"
+        elif isinstance(expression, (exp.Paren, exp.Identifier)):
+            pass
+        else:
+            type = "derived"
+
+    depends_on = _dedeup_depends_on(depends_on)
+
+    if len(depends_on) == 0:
+        type = "source"
+
+    if isinstance(proj, exp.Alias):
+        alias = proj
+        if type == "passthrough" and depends_on[0].column != alias.alias_or_name:
+            type = "renamed"
+
+    return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
+
+
+def cll_old(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn]:
+    result = cll(sql, schema=schema, dialect=dialect)
+    return result.columns
+
+
+def _dedeup_depends_on(depends_on: List[ColumnLevelDependsOn]) -> List[ColumnLevelDependsOn]:
+    # deduplicate the depends_on list
+    dedup_set = set()
+    dedup_list = []
+    for col_dep in depends_on:
+        node_col = col_dep.node + "." + col_dep.column
+        if node_col not in dedup_set:
+            dedup_list.append(col_dep)
+            dedup_set.add(node_col)
+    return dedup_list
 
-    if isinstance(expression, Column):
-        column = expression
-        alias = column.table
 
-        if alias is None:
-            table = next(iter(table_alias_map.values()))
+def _dedeup_cll_result(cll_result: CllResult):
+    cll_result.depends_on = _dedeup_depends_on(cll_result.depends_on)
+    for column in cll_result.columns.values():
+        column.depends_on = _dedeup_depends_on(column.depends_on)
+
+
+def _cll_set_scope(scope: Scope, scope_cll_map: dict[Scope, CllResult]) -> CllResult:
+    result = CllResult(depends_on=[], columns={})
+    scope_lineage = result.columns
+
+    for union_scope in scope.union_scopes:
+        sub_scope_result = scope_cll_map.get(union_scope)
+
+        for k, v in sub_scope_result.columns.items():
+            if k not in result.columns:
+                scope_lineage[k] = v
+            else:
+                scope_lineage[k].depends_on.extend(v.depends_on)
+                scope_lineage[k].type = "derived"
+
+        result.depends_on.extend(sub_scope_result.depends_on)
+    return result
+
+
+def _cll_select_scope(scope: Scope, scope_cll_map: dict[Scope, CllResult]) -> CllResult:
+    assert scope.expression.key == "select"
+
+    column_dep_map = {}
+    model_depends_on = []
+    table_alias_map = {t.alias_or_name: t.name for t in scope.tables}
+    select = scope.expression
+
+    def source_column_dependency(ref_column: exp.Column) -> Optional[ColumnLevelDependencyColumn]:
+        column_name = ref_column.name
+        table_name = ref_column.table if ref_column.table != "" else next(iter(table_alias_map.values()))
+        source = scope.sources.get(table_name, None)  # type: exp.Table | Scope
+        if isinstance(source, Scope):
+            ref_cll = scope_cll_map.get(source)
+            if ref_cll is None:
+                return None
+            return ref_cll.columns.get(column_name)
+        elif isinstance(source, exp.Table):
+            return ColumnLevelDependencyColumn(
+                type="passthrough", depends_on=[ColumnLevelDependsOn(source.name, column_name)]
+            )
         else:
-            table = table_alias_map.get(alias, alias)
-
-        return ColumnLevelDependencyColumn(type="passthrough", depends_on=[ColumnLevelDependsOn(table, column.name)])
-    elif isinstance(expression, Paren):
-        return _cll_expression(expression.this, table_alias_map)
-    elif isinstance(expression, Binary):
-        depends_on = []
-        if expression.left:
-            depends_on_left = _cll_expression(expression.left, table_alias_map).depends_on
-            depends_on.extend(depends_on_left)
-        if expression.right:
-            depends_on_right = _cll_expression(expression.right, table_alias_map).depends_on
-            depends_on.extend(depends_on_right)
-        type = "derived" if depends_on else "source"
-        return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-    elif isinstance(expression, Case):
-        ifs = expression.args["ifs"]
-        default = expression.args["default"]
-        depends_on = []
-        for expr in ifs:
-            depends_on_one = _cll_expression(expr, table_alias_map).depends_on
-            depends_on.extend(depends_on_one)
-        if default is not None:
-            depends_on.extend(_cll_expression(default, table_alias_map).depends_on)
-        type = "derived" if depends_on else "source"
-        return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-    elif isinstance(expression, If):
-        depends_on = []
-        if expression.this:
-            depends_on_one = _cll_expression(expression.this, table_alias_map).depends_on
-            depends_on.extend(depends_on_one)
-        if expression.args.get("true"):
-            depends_on_one = _cll_expression(expression.args.get("true"), table_alias_map).depends_on
-            depends_on.extend(depends_on_one)
-        if expression.args.get("false"):
-            depends_on_one = _cll_expression(expression.args.get("false"), table_alias_map).depends_on
-            depends_on.extend(depends_on_one)
-        type = "derived" if depends_on else "source"
-        return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-    elif isinstance(expression, Func):
-        if expression.expressions:
-            depends_on = []
-            for expr in expression.expressions:
-                depends_on_one = _cll_expression(expr, table_alias_map).depends_on
-                depends_on.extend(depends_on_one)
-            type = "derived" if depends_on else "source"
-            return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-        if expression.this:
-            depends_on = _cll_expression(expression.this, table_alias_map).depends_on
-            type = "derived" if depends_on else "source"
-            return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-
-        return ColumnLevelDependencyColumn(type="source", depends_on=[])
-    elif expression.this and isinstance(expression.this, Expression):
-        depends_on = _cll_expression(expression.this, table_alias_map).depends_on
-        type = "derived" if depends_on else "source"
-        return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-    elif expression.expressions:
-        depends_on = []
-        for expr in expression.expressions:
-            depends_on_one = _cll_expression(expr, table_alias_map).depends_on
-            depends_on.extend(depends_on_one)
-        type = "derived" if depends_on else "source"
-        return ColumnLevelDependencyColumn(type=type, depends_on=depends_on)
-    else:
-        depends_on = []
-        return ColumnLevelDependencyColumn(type="source", depends_on=depends_on)
-
-
-def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn]:
-    # given a sql, return the columns depends on
+            return None
+
+    for proj in scope.expression.selects:
+        type = "source"
+        column_depends_on: List[ColumnLevelDependsOn] = []
+        root = proj.this if isinstance(proj, exp.Alias) else proj
+        for expression in root.walk(bfs=False):
+            if isinstance(expression, exp.Column):
+                ref_column_dependency = source_column_dependency(expression)
+                if ref_column_dependency is not None:
+                    column_depends_on.extend(ref_column_dependency.depends_on)
+                    if ref_column_dependency.type == "derived":
+                        type = "derived"
+                    elif ref_column_dependency.type == "renamed":
+                        if type == "source" or type == "passthrough":
+                            type = "renamed"
+                    elif ref_column_dependency.type == "passthrough":
+                        if type == "source":
+                            type = "passthrough"
+                else:
+                    column_depends_on.append(ColumnLevelDependsOn(expression.table, expression.name))
+                    if type == "source":
+                        type = "passthrough"
+
+            elif isinstance(expression, (exp.Paren, exp.Identifier)):
+                pass
+            else:
+                type = "derived"
+
+        column_depends_on = _dedeup_depends_on(column_depends_on)
+
+        if len(column_depends_on) == 0 and type != "source":
+            type = "source"
+
+        if isinstance(proj, exp.Alias):
+            alias = proj
+            if type == "passthrough" and column_depends_on[0].column != alias.alias_or_name:
+                type = "renamed"
+
+        column_dep_map[proj.alias_or_name] = ColumnLevelDependencyColumn(type=type, depends_on=column_depends_on)
+
+    def selected_column_dependency(ref_column: exp.Column) -> Optional[ColumnLevelDependencyColumn]:
+        column_name = ref_column.name
+        return column_dep_map.get(column_name)
+
+    # joins clause: Reference the source columns
+    if select.args.get("joins"):
+        joins = select.args.get("joins")
+        for join in joins:
+            if isinstance(join, exp.Join):
+                for ref_column in join.find_all(exp.Column):
+                    if source_column_dependency(ref_column) is not None:
+                        model_depends_on.extend(source_column_dependency(ref_column).depends_on)
+
+    # where clauses: Reference the source columns
+    if select.args.get("where"):
+        where = select.args.get("where")
+        if isinstance(where, exp.Where):
+            for ref_column in where.find_all(exp.Column):
+                if source_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(source_column_dependency(ref_column).depends_on)
+
+    # group by clause: Reference the source columns, column index
+    if select.args.get("group"):
+        group = select.args.get("group")
+        if isinstance(group, exp.Group):
+            for ref_column in group.find_all(exp.Column):
+                if source_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(source_column_dependency(ref_column).depends_on)
+
+    # having clause: Reference the source columns, selected columns
+    if select.args.get("having"):
+        having = select.args.get("having")
+        if isinstance(having, exp.Having):
+            for ref_column in having.find_all(exp.Column):
+                if source_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(source_column_dependency(ref_column).depends_on)
+                elif selected_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(selected_column_dependency(ref_column).depends_on)
+
+    # order by clause: Reference the source columns, selected columns, column index
+    if select.args.get("order"):
+        order = select.args.get("order")
+        if isinstance(order, exp.Order):
+            for ref_column in order.find_all(exp.Column):
+                if source_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(source_column_dependency(ref_column).depends_on)
+                elif selected_column_dependency(ref_column) is not None:
+                    model_depends_on.extend(selected_column_dependency(ref_column).depends_on)
+
+    for source in scope.sources.values():
+        scope_result = scope_cll_map.get(source)
+        if scope_result is not None:
+            model_depends_on.extend(scope_result.depends_on)
+
+    model_depends_on = _dedeup_depends_on(model_depends_on)
+
+    return CllResult(columns=column_dep_map, depends_on=model_depends_on)
+
+
+def cll(sql, schema=None, dialect=None) -> CllResult:
+    # given a sql, return the cll for the sql
     # {
-    #   'column1': {
-    #      'transformation_type': 'transform' or 'original',
-    #      'depends_on': {
-    #           {node: model_id, column: column}, ...]
-    #      }
+    #     'depends_on': [{'node': 'model_id', 'column': 'column'}],
+    #     'columns': {
+    #         'column1': {
+    #             'type': 'derived',
+    #             'depends_on': [{'node': 'model_id', 'column': 'column'}],
+    #         }
+    #     }
+    # }
 
     dialect = Dialect.get(dialect) if dialect is not None else None
 
@@ -194,111 +320,17 @@ def cll(sql, schema=None, dialect=None) -> Dict[str, ColumnLevelDependencyColumn
     except SqlglotError as e:
         raise RecceException(f"Failed to qualify SQL: {str(e)}")
 
-    result = {}
-    global_lineage = {}
+    result = CllResult(depends_on=[], columns={})
+    scope_cll_map = {}
     for scope in traverse_scope(expression):
-        scope_lineage = {}
-
-        table_alias_map = {t.alias_or_name: t.name for t in scope.tables}
-
-        if isinstance(scope.expression, Union) or isinstance(scope.expression, Intersect):
-            for union_scope in scope.union_scopes:
-                for k, v in global_lineage[union_scope].items():
-                    if k not in scope_lineage:
-                        scope_lineage[k] = v
-                    else:
-                        scope_lineage[k].depends_on.extend(v.depends_on)
-                        scope_lineage[k].type = "derived"
+        scope_type = scope.expression.key
+        if scope_type == "union" or scope_type == "intersect" or scope_type == "except":
+            result = _cll_set_scope(scope, scope_cll_map)
+        elif scope_type == "select":
+            result = _cll_select_scope(scope, scope_cll_map)
         else:
-            for select in scope.expression.selects:
-                # instance of Column
-                if isinstance(select, Column):
-                    # 'select a'
-                    column = select
-                    column_cll = _cll_expression(column, table_alias_map)
-                elif isinstance(select, Alias):
-                    # 'select a as b'
-                    # 'select CURRENT_TIMESTAMP() as create_at'
-                    alias = select
-                    col_expression = alias.this
-                    column_cll = _cll_expression(col_expression, table_alias_map)
-                    if (
-                        column_cll
-                        and column_cll.type == "passthrough"
-                        and column_cll.depends_on[0].column != alias.alias_or_name
-                    ):
-                        column_cll.type = "renamed"
-                else:
-                    # 'select 1'
-                    column_cll = ColumnLevelDependencyColumn(type="source", depends_on=[])
-
-                cte_type = None
-                flatten_col_depends_on = []
-                for col_dep in column_cll.depends_on:
-                    col_dep_node = col_dep.node
-                    col_dep_column = col_dep.column
-                    # cte
-                    cte_scope = scope.cte_sources.get(col_dep_node)
-                    # inline derived table
-                    source_scope = None
-                    if isinstance(scope.sources.get(col_dep_node), Scope):
-                        source_scope = scope.sources.get(col_dep_node)
-
-                    if cte_scope is not None:
-                        cte_cll = global_lineage[cte_scope]
-                        if cte_cll is None or cte_cll.get(col_dep_column) is None:
-                            # In dbt-duckdb, the external source is compiled as `read_csv('..') rather than a table.
-                            continue
-                        cte_type = cte_cll.get(col_dep_column).type
-                        flatten_col_depends_on.extend(cte_cll.get(col_dep_column).depends_on)
-                    elif source_scope is not None:
-                        source_cll = global_lineage[source_scope]
-                        if source_cll is None or source_cll.get(col_dep_column) is None:
-                            continue
-                        flatten_col_depends_on.extend(source_cll.get(col_dep_column).depends_on)
-                    else:
-                        flatten_col_depends_on.append(col_dep)
-
-                # deduplicate
-                dedup_col_depends_on = []
-                dedup_set = set()
-                for col_dep in flatten_col_depends_on:
-                    node_col = col_dep.node + "." + col_dep.column
-                    if node_col not in dedup_set:
-                        dedup_col_depends_on.append(col_dep)
-                        dedup_set.add(node_col)
-
-                # transformation type
-                type = column_cll.type
-                if type == "derived":
-                    if len(dedup_col_depends_on) == 0:
-                        type = "source"
-                    else:
-                        # keep current scope type
-                        pass
-                elif cte_type is not None:
-                    if len(dedup_col_depends_on) > 1:
-                        type = "derived"
-                    elif len(dedup_col_depends_on) == 0:
-                        type = "source"
-                    else:
-                        if isinstance(select, Column):
-                            type = cte_type
-                        elif isinstance(select, Alias):
-                            alias = select
-                            if column_cll.depends_on[0].column == alias.alias_or_name:
-                                type = cte_type
-                            else:
-                                type = "renamed" if cte_type == "passthrough" else cte_type
-                        else:
-                            type = "source"
-
-                scope_lineage[select.alias_or_name] = ColumnLevelDependencyColumn(
-                    type=type, depends_on=dedup_col_depends_on
-                )
-
-        global_lineage[scope] = scope_lineage
-        if not scope.is_cte:
-            result = scope_lineage
+            continue
+
+        scope_cll_map[scope] = result
 
     return result

recce/util/io.py

@@ -68,6 +68,9 @@ class ZipFileIO(AbstractFileIO, ABC):
     def _is_pyminizip_installed():
         try:
             import pyminizip
+
+            # Use the module to avoid F401
+            return pyminizip is not None
         except ImportError:
             raise ImportError("pyminizip is not installed. Please install it using `pip install pyminizip`")
 

recce/util/recce_cloud.py

@@ -8,6 +8,7 @@ import requests
 from recce.pull_request import PullRequestInfo
 
 RECCE_CLOUD_API_HOST = os.environ.get("RECCE_CLOUD_API_HOST", "https://cloud.datarecce.io")
+RECCE_CLOUD_BASE_URL = os.environ.get("RECCE_CLOUD_BASE_URL", RECCE_CLOUD_API_HOST)
 
 logger = logging.getLogger("uvicorn")
 
@@ -31,13 +32,30 @@ class RecceCloudException(Exception):
 
 class RecceCloud:
     def __init__(self, token: str):
+        if token is None:
+            raise ValueError("Token cannot be None.")
         self.token = token
+        self.token_type = "github_token" if token.startswith(
+            ("ghp_", "gho_", "ghu_", "ghs_", "ghr_")) else "api_token"
         self.base_url = f"{RECCE_CLOUD_API_HOST}/api/v1"
 
     def _request(self, method, url, **kwargs):
         headers = {"Authorization": f"Bearer {self.token}"}
         return requests.request(method, url, headers=headers, **kwargs)
 
+    def verify_token(self) -> bool:
+        if self.token_type == "github_token":
+            return True
+        # Verify the Recce Cloud API token
+        api_url = f"{self.base_url}/verify-token"
+        try:
+            response = self._request("GET", api_url)
+            if response.status_code == 200:
+                return True
+        except Exception:
+            pass
+        return False
+
     def get_presigned_url(
         self,
         method: PresignedUrlMethod,

{recce_nightly-1.4.0.20250519.dist-info → recce_nightly-1.5.0.20250521.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recce-nightly
-Version: 1.4.0.20250519
+Version: 1.5.0.20250521
 Summary: Environment diff tool for dbt
 Home-page: https://github.com/InfuseAI/recce
 Author: InfuseAI Dev Team