sql-glider 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

{sql_glider-0.1.3.dist-info → sql_glider-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.3
+Version: 0.1.5
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

{sql_glider-0.1.3.dist-info → sql_glider-0.1.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlglider/__init__.py,sha256=gDf7s52dMcX7JuCZ1SLawcB1vb3U0yJCohu9RQAATBY,125
-sqlglider/_version.py,sha256=q5nF98G8SoVeJqaknL0xdyxtv0egsqb0fK06_84Izu8,704
+sqlglider/_version.py,sha256=rdxBMYpwzYxiWk08QbPLHSAxHoDfeKWwyaJIAM0lSic,704
 sqlglider/cli.py,sha256=9sweHRVLk2iBSzCzT2Gcj8y1g1XKzq26iApQsMaFbx4,51786
 sqlglider/global_models.py,sha256=2vyJXAuXOsXQpE-D3F0ejj7eR9z0nDWFjTkielhzM8k,356
 sqlglider/catalog/__init__.py,sha256=2PqFPyzFXJ14FpSUcBmVK2L-a_ypWQHAbHFHxLDk_LE,814
@@ -17,7 +17,7 @@ sqlglider/graph/models.py,sha256=EYmjv_WzDSNp_WfhJ6H-qBIOkAcoNKS7GRUryfKrHuY,933
 sqlglider/graph/query.py,sha256=LHU8Cvn7ZPPSEnqdDn2pF8f1_LQjIvNIrZqs8cFlb6U,9433
 sqlglider/graph/serialization.py,sha256=7JJo31rwSlxnDhdqdTJdK4Dr_ZcSYetXfx3_CmndSac,2662
 sqlglider/lineage/__init__.py,sha256=llXMeI5_PIZaiBo8tKk3-wOubF4m_6QBHbn1FtWxT7k,256
-sqlglider/lineage/analyzer.py,sha256=HyyjGMP7VvEmvt-V-qT48C-41Usj2OmT5FPYYKdJsSs,48218
+sqlglider/lineage/analyzer.py,sha256=9Ly93TBcpc2sRP9NuwxB0jCL3YK-yCV1rih93MFM-S0,64313
 sqlglider/lineage/formatters.py,sha256=_Y9wcTX4JXn1vVnZ1xI656g1FF2rMjcAVc-GHjbd9QA,10389
 sqlglider/templating/__init__.py,sha256=g3_wb6rSDI0usq2UUMDpn-J5kVwlAw3NtLdwbxL6UHs,1435
 sqlglider/templating/base.py,sha256=y5bWAW7qXl_4pPyo5KycfHwNVvt1-7slZ63DAsvTE1s,2902
@@ -27,8 +27,8 @@ sqlglider/templating/variables.py,sha256=5593PtLBcOxsnMCSRm2pGAD5I0Y9f__VV3_J_Hf
 sqlglider/utils/__init__.py,sha256=KGp9-UzKz_OFBOTFoSy-g-NXDZsvyWXG_9-1zcC6ePE,276
 sqlglider/utils/config.py,sha256=iNJgSXFw3pmL2MCdvW3SJp4X2T3AQP2QyQuXIXT-6H0,4761
 sqlglider/utils/file_utils.py,sha256=5_ff28E0r1R7emZzsOnRuHd-7zIX6873eyr1SuPEr4E,1093
-sql_glider-0.1.3.dist-info/METADATA,sha256=D83HzMc1l3AHbnR3Y9aOzwTbp6yMKR9tQMXWTWkH1Sw,28445
-sql_glider-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-sql_glider-0.1.3.dist-info/entry_points.txt,sha256=HDuakHqHS5C0HFKsMIxMYmDU7-BLBGrnIJcYaVRu-s0,251
-sql_glider-0.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sql_glider-0.1.3.dist-info/RECORD,,
+sql_glider-0.1.5.dist-info/METADATA,sha256=uLFCtpVoKp5F8ORuqJfJQtNZFjAJ3BQLavtRl-Bh0JA,28445
+sql_glider-0.1.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+sql_glider-0.1.5.dist-info/entry_points.txt,sha256=HDuakHqHS5C0HFKsMIxMYmDU7-BLBGrnIJcYaVRu-s0,251
+sql_glider-0.1.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sql_glider-0.1.5.dist-info/RECORD,,

sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.3'
-__version_tuple__ = version_tuple = (0, 1, 3)
+__version__ = version = '0.1.5'
+__version_tuple__ = version_tuple = (0, 1, 5)
 
 __commit_id__ = commit_id = None

sqlglider/lineage/analyzer.py

@@ -1,7 +1,7 @@
 """Core lineage analysis using SQLGlot."""
 
 from enum import Enum
-from typing import Callable, Iterator, List, Optional, Set, Tuple, Union
+from typing import Callable, Dict, Iterator, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel, Field
 from sqlglot import exp, parse
@@ -99,6 +99,9 @@ class LineageAnalyzer:
         self.sql = sql
         self.dialect = dialect
         self._skipped_queries: List[SkippedQuery] = []
+        # File-scoped schema context for cross-statement lineage
+        # Maps table/view names to their column definitions
+        self._file_schema: Dict[str, Dict[str, str]] = {}
 
         try:
             # Parse all statements in the SQL string
@@ -156,26 +159,66 @@ class LineageAnalyzer:
             # DML/DDL: Use target table for output column qualification
             # The columns are from the SELECT, but qualified with the target table
             projections = self._get_select_projections(select_node)
+            first_select = self._get_first_select(select_node)
+
             for projection in projections:
+                # Handle SELECT * by resolving from file schema
+                if isinstance(projection, exp.Star):
+                    if first_select:
+                        star_columns = self._resolve_star_columns(first_select)
+                        for star_col in star_columns:
+                            qualified_name = f"{target_table}.{star_col}"
+                            columns.append(qualified_name)
+                            self._column_mapping[qualified_name] = star_col
+                    if not columns:
+                        # Fallback: can't resolve *, use * as column name
+                        qualified_name = f"{target_table}.*"
+                        columns.append(qualified_name)
+                        self._column_mapping[qualified_name] = "*"
+                    continue
+
                 # Get the underlying expression (unwrap alias if present)
                 if isinstance(projection, exp.Alias):
                     # For aliased columns, use the alias as the column name
                     column_name = projection.alias
                     lineage_name = column_name  # SQLGlot lineage uses the alias
-                else:
-                    source_expr = projection
-                    if isinstance(source_expr, exp.Column):
-                        column_name = source_expr.name
-                        lineage_name = column_name
+                    # Qualify with target table
+                    qualified_name = f"{target_table}.{column_name}"
+                    columns.append(qualified_name)
+                    self._column_mapping[qualified_name] = lineage_name
+                elif isinstance(projection, exp.Column):
+                    # Check if this is a table-qualified star (e.g., t.*)
+                    if isinstance(projection.this, exp.Star):
+                        source_table = projection.table
+                        qualified_star_cols: List[str] = []
+                        if source_table and first_select:
+                            qualified_star_cols = self._resolve_qualified_star(
+                                source_table, first_select
+                            )
+                            for col in qualified_star_cols:
+                                qualified_name = f"{target_table}.{col}"
+                                columns.append(qualified_name)
+                                self._column_mapping[qualified_name] = col
+                        if not qualified_star_cols:
+                            # Fallback: can't resolve t.*, use * as column name
+                            qualified_name = f"{target_table}.*"
+                            columns.append(qualified_name)
+                            self._column_mapping[qualified_name] = "*"
                     else:
-                        # For expressions, use the SQL representation
-                        column_name = source_expr.sql(dialect=self.dialect)
+                        column_name = projection.name
                         lineage_name = column_name
-
-                # Qualify with target table
-                qualified_name = f"{target_table}.{column_name}"
-                columns.append(qualified_name)
-                self._column_mapping[qualified_name] = lineage_name
+                        # Qualify with target table
+                        qualified_name = f"{target_table}.{column_name}"
+                        columns.append(qualified_name)
+                        self._column_mapping[qualified_name] = lineage_name
+                else:
+                    # For expressions, use the SQL representation
+                    column_name = projection.sql(dialect=self.dialect)
+                    lineage_name = column_name
+                    # Qualify with target table
+                    qualified_name = f"{target_table}.{column_name}"
+                    columns.append(qualified_name)
+                    self._column_mapping[qualified_name] = lineage_name
 
         else:
             # DQL (pure SELECT): Use the SELECT columns as output
@@ -324,6 +367,7 @@ class LineageAnalyzer:
         """
         results = []
         self._skipped_queries = []  # Reset skipped queries for this analysis
+        self._file_schema = {}  # Reset file schema for this analysis run
 
         for query_index, expr, preview in self._iterate_queries(table_filter):
             # Temporarily swap self.expr to analyze this query
@@ -375,6 +419,9 @@ class LineageAnalyzer:
                     )
                 )
             finally:
+                # Extract schema from this statement AFTER analysis
+                # This builds up context for subsequent statements to use
+                self._extract_schema_from_statement(expr)
                 # Restore original expression
                 self.expr = original_expr
 
@@ -702,7 +749,13 @@ class LineageAnalyzer:
                 lineage_col = self._column_mapping.get(col, col)
 
                 # Get lineage tree for this column using current query SQL only
-                node = lineage(lineage_col, current_query_sql, dialect=self.dialect)
+                # Pass file schema to enable SELECT * expansion for known tables/views
+                node = lineage(
+                    lineage_col,
+                    current_query_sql,
+                    dialect=self.dialect,
+                    schema=self._file_schema if self._file_schema else None,
+                )
 
                 # Collect all source columns
                 sources: Set[str] = set()
@@ -1235,3 +1288,338 @@ class LineageAnalyzer:
             preview = self._generate_query_preview(expr)
 
             yield idx, expr, preview
+
+    # -------------------------------------------------------------------------
+    # File-scoped schema context methods
+    # -------------------------------------------------------------------------
+
+    def _extract_schema_from_statement(self, expr: exp.Expression) -> None:
+        """
+        Extract column definitions from CREATE VIEW/TABLE AS SELECT statements.
+
+        This method builds up file-scoped schema context as statements are processed,
+        enabling SQLGlot to correctly expand SELECT * and trace cross-statement references.
+
+        Args:
+            expr: The SQL expression to extract schema from
+        """
+        # Only handle CREATE VIEW or CREATE TABLE (AS SELECT)
+        if not isinstance(expr, exp.Create):
+            return
+        if expr.kind not in ("VIEW", "TABLE"):
+            return
+
+        # Get target table/view name
+        target = expr.this
+        if isinstance(target, exp.Schema):
+            target = target.this
+        if not isinstance(target, exp.Table):
+            return
+
+        target_name = self._get_qualified_table_name(target)
+
+        # Get the SELECT node from the CREATE statement
+        select_node = expr.expression
+        if select_node is None:
+            return
+
+        # Handle Subquery wrapper (e.g., CREATE VIEW AS (SELECT ...))
+        if isinstance(select_node, exp.Subquery):
+            select_node = select_node.this
+
+        if not isinstance(
+            select_node, (exp.Select, exp.Union, exp.Intersect, exp.Except)
+        ):
+            return
+
+        # Extract column names from the SELECT
+        columns = self._extract_columns_from_select(select_node)
+
+        if columns:
+            # Store with UNKNOWN type - SQLGlot only needs column names for expansion
+            self._file_schema[target_name] = {col: "UNKNOWN" for col in columns}
+
+    def _extract_columns_from_select(
+        self, select_node: Union[exp.Select, exp.Union, exp.Intersect, exp.Except]
+    ) -> List[str]:
+        """
+        Extract column names from a SELECT statement.
+
+        Handles aliases, direct column references, and SELECT * by resolving
+        against the known file schema.
+
+        Args:
+            select_node: The SELECT or set operation expression
+
+        Returns:
+            List of column names
+        """
+        columns: List[str] = []
+
+        # Get projections (for UNION, use first branch)
+        projections = self._get_select_projections(select_node)
+        first_select = self._get_first_select(select_node)
+
+        for projection in projections:
+            if isinstance(projection, exp.Alias):
+                # Use the alias name as the column name
+                columns.append(projection.alias)
+            elif isinstance(projection, exp.Column):
+                # Check if this is a table-qualified star (e.g., t.*)
+                if isinstance(projection.this, exp.Star):
+                    # Resolve table-qualified star from known schema
+                    table_name = projection.table
+                    if table_name and first_select:
+                        qualified_star_cols = self._resolve_qualified_star(
+                            table_name, first_select
+                        )
+                        columns.extend(qualified_star_cols)
+                else:
+                    # Use the column name
+                    columns.append(projection.name)
+            elif isinstance(projection, exp.Star):
+                # Resolve SELECT * from known schema
+                if first_select:
+                    star_columns = self._resolve_star_columns(first_select)
+                    columns.extend(star_columns)
+            else:
+                # For expressions without alias, use SQL representation
+                col_sql = projection.sql(dialect=self.dialect)
+                columns.append(col_sql)
+
+        return columns
+
+    def _resolve_star_columns(self, select_node: exp.Select) -> List[str]:
+        """
+        Resolve SELECT * to actual column names from known file schema or CTEs.
+
+        Args:
+            select_node: The SELECT node containing the * reference
+
+        Returns:
+            List of column names if source is known, empty list otherwise
+        """
+        columns: List[str] = []
+
+        # Get the source table(s) from FROM clause
+        from_clause = select_node.args.get("from")
+        if not from_clause or not isinstance(from_clause, exp.From):
+            return columns
+
+        source = from_clause.this
+
+        # Handle table reference from FROM clause
+        columns.extend(self._resolve_source_columns(source, select_node))
+
+        # Handle JOIN clauses - collect columns from all joined tables
+        joins = select_node.args.get("joins")
+        if joins:
+            for join in joins:
+                if isinstance(join, exp.Join):
+                    join_source = join.this
+                    columns.extend(
+                        self._resolve_source_columns(join_source, select_node)
+                    )
+
+        # Handle LATERAL VIEW clauses - collect generated columns
+        laterals = select_node.args.get("laterals")
+        if laterals:
+            for lateral in laterals:
+                if isinstance(lateral, exp.Lateral):
+                    lateral_cols = self._resolve_lateral_columns(lateral)
+                    columns.extend(lateral_cols)
+
+        return columns
+
+    def _resolve_lateral_columns(self, lateral: exp.Lateral) -> List[str]:
+        """
+        Extract generated column names from a LATERAL VIEW clause.
+
+        Args:
+            lateral: The Lateral expression node
+
+        Returns:
+            List of generated column names (e.g., ['elem'] for explode,
+            ['pos', 'elem'] for posexplode)
+        """
+        # Use SQLGlot's built-in property to get alias column names
+        return lateral.alias_column_names or []
+
+    def _resolve_source_columns(
+        self, source: exp.Expression, select_node: exp.Select
+    ) -> List[str]:
+        """
+        Resolve columns from a single source (table, subquery, etc.).
+
+        Args:
+            source: The source expression (Table, Subquery, etc.)
+            select_node: The containing SELECT node for CTE resolution
+
+        Returns:
+            List of column names from the source
+        """
+        columns: List[str] = []
+
+        # Handle table reference
+        if isinstance(source, exp.Table):
+            source_name = self._get_qualified_table_name(source)
+
+            # First check file schema (views/tables from previous statements)
+            if source_name in self._file_schema:
+                columns.extend(self._file_schema[source_name].keys())
+            else:
+                # Check if this is a CTE reference within the same statement
+                cte_columns = self._resolve_cte_columns(source_name, select_node)
+                columns.extend(cte_columns)
+
+        # Handle subquery with alias
+        elif isinstance(source, exp.Subquery):
+            # First check if this subquery alias is in file schema
+            if source.alias and source.alias in self._file_schema:
+                columns.extend(self._file_schema[source.alias].keys())
+            else:
+                # Extract columns from the subquery's SELECT
+                inner_select = source.this
+                if isinstance(inner_select, exp.Select):
+                    subquery_cols = self._extract_subquery_columns(inner_select)
+                    columns.extend(subquery_cols)
+
+        return columns
+
+    def _resolve_qualified_star(
+        self, table_name: str, select_node: exp.Select
+    ) -> List[str]:
+        """
+        Resolve a table-qualified star (e.g., t.*) to actual column names.
+
+        Args:
+            table_name: The table/alias name qualifying the star
+            select_node: The SELECT node for context
+
+        Returns:
+            List of column names from the specified table
+        """
+        # First check file schema
+        if table_name in self._file_schema:
+            return list(self._file_schema[table_name].keys())
+
+        # Check if it's a CTE reference
+        cte_columns = self._resolve_cte_columns(table_name, select_node)
+        if cte_columns:
+            return cte_columns
+
+        # Check if the table name is an alias - need to resolve the actual table
+        from_clause = select_node.args.get("from")
+        if from_clause and isinstance(from_clause, exp.From):
+            source = from_clause.this
+            if isinstance(source, exp.Table) and source.alias == table_name:
+                actual_name = self._get_qualified_table_name(source)
+                if actual_name in self._file_schema:
+                    return list(self._file_schema[actual_name].keys())
+
+            # Check JOIN clauses for aliased tables
+            joins = select_node.args.get("joins")
+            if joins:
+                for join in joins:
+                    if isinstance(join, exp.Join):
+                        join_source = join.this
+                        if (
+                            isinstance(join_source, exp.Table)
+                            and join_source.alias == table_name
+                        ):
+                            actual_name = self._get_qualified_table_name(join_source)
+                            if actual_name in self._file_schema:
+                                return list(self._file_schema[actual_name].keys())
+
+        return []
+
+    def _extract_subquery_columns(self, subquery_select: exp.Select) -> List[str]:
+        """
+        Extract column names from a subquery's SELECT statement.
+
+        Args:
+            subquery_select: The SELECT expression within the subquery
+
+        Returns:
+            List of column names
+        """
+        columns: List[str] = []
+
+        for projection in subquery_select.expressions:
+            if isinstance(projection, exp.Alias):
+                columns.append(projection.alias)
+            elif isinstance(projection, exp.Column):
+                # Check for table-qualified star (t.*)
+                if isinstance(projection.this, exp.Star):
+                    table_name = projection.table
+                    if table_name:
+                        qualified_cols = self._resolve_qualified_star(
+                            table_name, subquery_select
+                        )
+                        columns.extend(qualified_cols)
+                else:
+                    columns.append(projection.name)
+            elif isinstance(projection, exp.Star):
+                # Resolve SELECT * in subquery
+                star_columns = self._resolve_star_columns(subquery_select)
+                columns.extend(star_columns)
+            else:
+                col_sql = projection.sql(dialect=self.dialect)
+                columns.append(col_sql)
+
+        return columns
+
+    def _resolve_cte_columns(self, cte_name: str, select_node: exp.Select) -> List[str]:
+        """
+        Resolve columns from a CTE definition within the same statement.
+
+        Args:
+            cte_name: Name of the CTE to resolve
+            select_node: The SELECT node that references the CTE
+
+        Returns:
+            List of column names from the CTE, empty if CTE not found
+        """
+        # Walk up the tree to find the WITH clause containing this CTE
+        parent = select_node
+        while parent:
+            if hasattr(parent, "args") and parent.args.get("with"):
+                with_clause = parent.args["with"]
+                for cte in with_clause.expressions:
+                    if isinstance(cte, exp.CTE) and cte.alias == cte_name:
+                        # Found the CTE - extract its columns
+                        cte_select = cte.this
+                        if isinstance(cte_select, exp.Select):
+                            return self._extract_cte_select_columns(cte_select)
+            parent = parent.parent if hasattr(parent, "parent") else None
+
+        return []
+
+    def _extract_cte_select_columns(self, cte_select: exp.Select) -> List[str]:
+        """
+        Extract column names from a CTE's SELECT statement.
+
+        This handles SELECT * within the CTE by resolving against file schema.
+
+        Args:
+            cte_select: The SELECT expression within the CTE
+
+        Returns:
+            List of column names
+        """
+        columns: List[str] = []
+
+        for projection in cte_select.expressions:
+            if isinstance(projection, exp.Alias):
+                columns.append(projection.alias)
+            elif isinstance(projection, exp.Column):
+                columns.append(projection.name)
+            elif isinstance(projection, exp.Star):
+                # Resolve SELECT * in CTE from file schema
+                star_columns = self._resolve_star_columns(cte_select)
+                columns.extend(star_columns)
+            else:
+                col_sql = projection.sql(dialect=self.dialect)
+                columns.append(col_sql)
+
+        return columns