sql-glider 0.1.10__tar.gz → 0.1.11__tar.gz

{sql_glider-0.1.10 → sql_glider-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.10
+Version: 0.1.11
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

sql_glider-0.1.11/plans/2026-01-28-sparksql-table-extraction.md

@@ -0,0 +1,58 @@
+# SparkSQL Table Extraction for Unsupported Statement Types
+
+**Status:** Planned
+**Date:** 2026-01-28
+
+## Overview
+
+The `analyze_tables()` method in `LineageAnalyzer` currently extracts tables from a subset of statement types (SELECT, INSERT, CREATE, DELETE, DROP, TRUNCATE, CACHE). Several SparkSQL-specific statement types reference tables but are not captured during table extraction. This plan adds table extraction support for these missing types.
+
+Column lineage is **not affected** — these statements contain no SELECT and cannot produce column-level lineage. The goal is to ensure `sqlglider tables overview` reports all tables referenced in a SQL file.
+
+## Statements to Add
+
+| Statement | SQLGlot Expression | Table Location | Proposed Usage |
+|-----------|-------------------|----------------|----------------|
+| `UNCACHE TABLE t` | `exp.Uncache` | `expr.this` | `INPUT` |
+| `REFRESH TABLE t` | `exp.Refresh` | `expr.this` | `INPUT` |
+| `LOAD DATA INPATH '...' INTO TABLE t` | `exp.LoadData` | `expr.this` | `OUTPUT` |
+| `ALTER TABLE t ...` | `exp.Alter` | `expr.this` | `OUTPUT` |
+| `ANALYZE TABLE t COMPUTE STATISTICS` | `exp.Analyze` | `expr.this` | `INPUT` |
+
+### Usage Rationale
+
+- **UNCACHE / REFRESH / ANALYZE**: Read-oriented metadata operations on an existing table → `INPUT`
+- **LOAD DATA**: Writes data into a table → `OUTPUT`
+- **ALTER TABLE**: Modifies table structure → `OUTPUT`
+
+## Implementation Steps
+
+- [ ] Add extraction logic to `_get_target_table_info()` in [analyzer.py](src/sqlglider/lineage/analyzer.py) for each new expression type
+- [ ] Add each type to the `_get_statement_type()` type_map for readable skip messages
+- [ ] Add entries to `_is_target_table()` where applicable (LOAD DATA, ALTER)
+- [ ] Ensure `_get_target_and_select()` returns `None` gracefully for these types (they have no SELECT)
+- [ ] Add unit tests in [test_analyzer.py](tests/sqlglider/lineage/test_analyzer.py):
+  - Table extraction returns correct table name and usage for each type
+  - Column lineage correctly skips these with appropriate message
+  - Parameterized test covering all five statement types
+- [ ] Verify graph build handles these gracefully (skipped queries warning)
+- [ ] Run full test suite and coverage check
+
+## Files to Modify
+
+- `src/sqlglider/lineage/analyzer.py` — extraction logic
+- `tests/sqlglider/lineage/test_analyzer.py` — unit tests
+
+## Testing Strategy
+
+- Parameterized tests with SparkSQL syntax for each statement type
+- Verify `analyze_tables()` returns correct table name, usage, and object type
+- Verify `analyze_queries()` adds these to `skipped_queries` with clear reason
+- Ensure no regressions in existing tests
+- Coverage threshold (80%) maintained
+
+## Notes
+
+- These are all parsed by sqlglot's Spark dialect parser, so no custom parsing is needed
+- Some of these (SHOW, DESCRIBE, EXPLAIN) parse as `exp.Command` — those are intentionally excluded since they don't reference tables in a structured way
+- INSERT OVERWRITE and multi-INSERT patterns may warrant separate investigation

{sql_glider-0.1.10 → sql_glider-0.1.11}/src/sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.10'
-__version_tuple__ = version_tuple = (0, 1, 10)
+__version__ = version = '0.1.11'
+__version_tuple__ = version_tuple = (0, 1, 11)
 
 __commit_id__ = commit_id = None

{sql_glider-0.1.10 → sql_glider-0.1.11}/src/sqlglider/cli.py

@@ -166,6 +166,11 @@ def lineage(
         exists=True,
         help="Path to variables file (JSON or YAML)",
     ),
+    no_star: bool = typer.Option(
+        False,
+        "--no-star",
+        help="Fail if SELECT * cannot be resolved to actual columns",
+    ),
 ) -> None:
     """
     Analyze column or table lineage for a SQL file.
@@ -207,6 +212,7 @@ def lineage(
     level_str = level or config.level or "column"
     output_format = output_format or config.output_format or "text"
     templater = templater or config.templater  # None means no templating
+    no_star = no_star or config.no_star or False
     # Validate and convert level to enum
     try:
         analysis_level = AnalysisLevel(level_str)
@@ -261,7 +267,7 @@ def lineage(
         )
 
         # Create analyzer
-        analyzer = LineageAnalyzer(sql, dialect=dialect)
+        analyzer = LineageAnalyzer(sql, dialect=dialect, no_star=no_star)
 
         # Unified lineage analysis (handles both single and multi-query files)
         results = analyzer.analyze_queries(
@@ -990,6 +996,11 @@ def graph_build(
         exists=True,
         help="Path to variables file (JSON or YAML)",
     ),
+    no_star: bool = typer.Option(
+        False,
+        "--no-star",
+        help="Fail if SELECT * cannot be resolved to actual columns",
+    ),
 ) -> None:
     """
     Build a lineage graph from SQL files.
@@ -1024,6 +1035,7 @@ def graph_build(
     config = load_config()
     dialect = dialect or config.dialect or "spark"
     templater = templater or config.templater  # None means no templating
+    no_star = no_star or config.no_star or False
 
     # Validate and convert node format to enum
     try:
@@ -1080,6 +1092,7 @@ def graph_build(
             node_format=node_format_enum,
             dialect=dialect,
             sql_preprocessor=sql_preprocessor,
+            no_star=no_star,
         )
 
         # Process manifest if provided

{sql_glider-0.1.10 → sql_glider-0.1.11}/src/sqlglider/graph/builder.py

@@ -33,6 +33,7 @@ class GraphBuilder:
         node_format: NodeFormat = NodeFormat.QUALIFIED,
         dialect: str = "spark",
         sql_preprocessor: Optional[SqlPreprocessor] = None,
+        no_star: bool = False,
     ):
         """
         Initialize the graph builder.
@@ -43,10 +44,12 @@ class GraphBuilder:
             sql_preprocessor: Optional function to preprocess SQL before analysis.
                              Takes (sql: str, file_path: Path) and returns processed SQL.
                              Useful for templating (e.g., Jinja2 rendering).
+            no_star: If True, fail when SELECT * cannot be resolved to columns
         """
         self.node_format = node_format
         self.dialect = dialect
         self.sql_preprocessor = sql_preprocessor
+        self.no_star = no_star
         self.graph: rx.PyDiGraph = rx.PyDiGraph()
         self._node_index_map: Dict[str, int] = {}  # identifier -> rustworkx node index
         self._source_files: Set[str] = set()
@@ -82,7 +85,9 @@ class GraphBuilder:
             if self.sql_preprocessor:
                 sql_content = self.sql_preprocessor(sql_content, file_path)
 
-            analyzer = LineageAnalyzer(sql_content, dialect=file_dialect)
+            analyzer = LineageAnalyzer(
+                sql_content, dialect=file_dialect, no_star=self.no_star
+            )
             results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
 
             # Print warnings for any skipped queries within the file

{sql_glider-0.1.10 → sql_glider-0.1.11}/src/sqlglider/lineage/analyzer.py

@@ -11,6 +11,10 @@ from sqlglot.lineage import Node, lineage
 from sqlglider.global_models import AnalysisLevel
 
 
+class StarResolutionError(Exception):
+    """Raised when SELECT * cannot be resolved and no_star mode is enabled."""
+
+
 class TableUsage(str, Enum):
     """How a table is used in a query."""
 
@@ -85,19 +89,21 @@ WarningCallback = Callable[[str], None]
 class LineageAnalyzer:
     """Analyze column and table lineage for SQL queries."""
 
-    def __init__(self, sql: str, dialect: str = "spark"):
+    def __init__(self, sql: str, dialect: str = "spark", no_star: bool = False):
         """
         Initialize the lineage analyzer.
 
         Args:
             sql: SQL query string to analyze (can contain multiple statements)
             dialect: SQL dialect (default: spark)
+            no_star: If True, fail when SELECT * cannot be resolved to columns
 
         Raises:
             ParseError: If the SQL cannot be parsed
         """
         self.sql = sql
         self.dialect = dialect
+        self._no_star = no_star
         self._skipped_queries: List[SkippedQuery] = []
         # File-scoped schema context for cross-statement lineage
         # Maps table/view names to their column definitions
@@ -171,6 +177,12 @@ class LineageAnalyzer:
                             columns.append(qualified_name)
                             self._column_mapping[qualified_name] = star_col
                     if not columns:
+                        if self._no_star:
+                            raise StarResolutionError(
+                                f"SELECT * could not be resolved to columns "
+                                f"for target table '{target_table}'. "
+                                f"Provide schema context or avoid using SELECT *."
+                            )
                         # Fallback: can't resolve *, use * as column name
                         qualified_name = f"{target_table}.*"
                         columns.append(qualified_name)
@@ -200,6 +212,12 @@ class LineageAnalyzer:
                                 columns.append(qualified_name)
                                 self._column_mapping[qualified_name] = col
                         if not qualified_star_cols:
+                            if self._no_star:
+                                raise StarResolutionError(
+                                    f"SELECT {source_table}.* could not be resolved "
+                                    f"to columns for target table '{target_table}'. "
+                                    f"Provide schema context or avoid using SELECT *."
+                                )
                             # Fallback: can't resolve t.*, use * as column name
                             qualified_name = f"{target_table}.*"
                             columns.append(qualified_name)
@@ -226,6 +244,23 @@ class LineageAnalyzer:
             # Get the first SELECT for table resolution (handles UNION case)
             first_select = self._get_first_select(select_node)
             for projection in projections:
+                # Handle SELECT * in DQL context
+                if isinstance(projection, exp.Star):
+                    if first_select:
+                        star_columns = self._resolve_star_columns(first_select)
+                        for star_col in star_columns:
+                            columns.append(star_col)
+                            self._column_mapping[star_col] = star_col
+                    if not columns:
+                        if self._no_star:
+                            raise StarResolutionError(
+                                "SELECT * could not be resolved to columns. "
+                                "Provide schema context or avoid using SELECT *."
+                            )
+                        columns.append("*")
+                        self._column_mapping["*"] = "*"
+                    continue
+
                 # Get the underlying expression (unwrap alias if present)
                 if isinstance(projection, exp.Alias):
                     source_expr = projection.this
@@ -236,6 +271,30 @@ class LineageAnalyzer:
                     column_name = None
                     lineage_name = None
 
+                # Handle table-qualified star in DQL context (e.g., t.*)
+                if isinstance(source_expr, exp.Column) and isinstance(
+                    source_expr.this, exp.Star
+                ):
+                    source_table = source_expr.table
+                    dql_star_cols: List[str] = []
+                    if source_table and first_select:
+                        dql_star_cols = self._resolve_qualified_star(
+                            source_table, first_select
+                        )
+                        for col in dql_star_cols:
+                            columns.append(col)
+                            self._column_mapping[col] = col
+                    if not dql_star_cols:
+                        if self._no_star:
+                            raise StarResolutionError(
+                                f"SELECT {source_table}.* could not be resolved "
+                                f"to columns. "
+                                f"Provide schema context or avoid using SELECT *."
+                            )
+                        columns.append("*")
+                        self._column_mapping["*"] = "*"
+                    continue
+
                 # Try to extract fully qualified name
                 if isinstance(source_expr, exp.Column):
                     # Get table and column parts
@@ -407,6 +466,8 @@ class LineageAnalyzer:
                         level=level,
                     )
                 )
+            except StarResolutionError:
+                raise
             except ValueError as e:
                 # Unsupported statement type - track it and continue
                 stmt_type = self._get_statement_type(expr)

{sql_glider-0.1.10 → sql_glider-0.1.11}/src/sqlglider/utils/config.py

@@ -60,6 +60,7 @@ class ConfigSettings(BaseModel):
     catalog_type: Optional[str] = None
     ddl_folder: Optional[str] = None
     catalog: Optional[CatalogConfig] = None
+    no_star: Optional[bool] = None
 
 
 def find_config_file(start_path: Optional[Path] = None) -> Optional[Path]:

{sql_glider-0.1.10 → sql_glider-0.1.11}/tests/sqlglider/lineage/test_analyzer.py

@@ -3,7 +3,7 @@
 import pytest
 
 from sqlglider.global_models import AnalysisLevel
-from sqlglider.lineage.analyzer import LineageAnalyzer
+from sqlglider.lineage.analyzer import LineageAnalyzer, StarResolutionError
 
 
 class TestCaseInsensitiveForwardLineage:
@@ -2897,6 +2897,27 @@ class TestCacheTableStatements:
         assert "orders" in tables_by_name
         assert tables_by_name["orders"].usage.value == "INPUT"
 
+    def test_cache_table_with_inline_subquery_alias(self):
+        """CACHE TABLE with an aliased inline subquery should trace through to sources."""
+        sql = """
+        CACHE TABLE cached_result AS
+        SELECT s.customer_id, s.order_total
+        FROM (
+            SELECT c.id as customer_id, o.total as order_total
+            FROM customers c
+            JOIN orders o ON c.id = o.customer_id
+        ) s
+        """
+        analyzer = LineageAnalyzer(sql, dialect="spark")
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+
+        assert len(results) == 1
+        items = {
+            item.output_name: item.source_name for item in results[0].lineage_items
+        }
+        assert items["cached_result.customer_id"] == "customers.id"
+        assert items["cached_result.order_total"] == "orders.total"
+
     def test_bare_cache_table_is_skipped(self):
         """CACHE TABLE t (without AS SELECT) should be skipped."""
         sql = "CACHE TABLE my_table"
@@ -2923,3 +2944,58 @@ class TestCacheTableStatements:
         skipped = analyzer.skipped_queries
         assert len(skipped) == 1
         assert "DELETE" in skipped[0].statement_type
+
+
+class TestNoStar:
+    """Tests for the --no-star flag that fails on unresolvable SELECT *."""
+
+    def test_bare_star_no_star_raises(self):
+        """SELECT * from unknown table should raise with no_star=True."""
+        sql = "SELECT * FROM some_table"
+        analyzer = LineageAnalyzer(sql, dialect="spark", no_star=True)
+        with pytest.raises(
+            StarResolutionError, match="SELECT \\* could not be resolved"
+        ):
+            analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+
+    def test_qualified_star_no_star_raises(self):
+        """SELECT t.* from unknown table should raise with no_star=True."""
+        sql = "SELECT t.* FROM some_table t"
+        analyzer = LineageAnalyzer(sql, dialect="spark", no_star=True)
+        with pytest.raises(
+            StarResolutionError, match="SELECT t\\.\\* could not be resolved"
+        ):
+            analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+
+    def test_bare_star_default_falls_back(self):
+        """SELECT * without no_star should fall back to table.*."""
+        sql = "SELECT * FROM some_table"
+        analyzer = LineageAnalyzer(sql, dialect="spark")
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        assert len(results) == 1
+        output_names = [item.output_name for item in results[0].lineage_items]
+        assert any("*" in name for name in output_names)
+
+    def test_resolvable_star_with_no_star_succeeds(self):
+        """SELECT * from CTE should work with no_star=True since columns are known."""
+        sql = """
+        WITH cte AS (SELECT 1 AS id, 'alice' AS name)
+        SELECT * FROM cte
+        """
+        analyzer = LineageAnalyzer(sql, dialect="spark", no_star=True)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        assert len(results) == 1
+        output_names = [item.output_name for item in results[0].lineage_items]
+        assert not any("*" in name for name in output_names)
+
+    def test_resolvable_qualified_star_with_no_star_succeeds(self):
+        """SELECT t.* from CTE should work with no_star=True since columns are known."""
+        sql = """
+        WITH cte AS (SELECT 1 AS id, 'alice' AS name)
+        SELECT cte.* FROM cte
+        """
+        analyzer = LineageAnalyzer(sql, dialect="spark", no_star=True)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        assert len(results) == 1
+        output_names = [item.output_name for item in results[0].lineage_items]
+        assert not any("*" in name for name in output_names)