sql-glider 0.1.15__tar.gz → 0.1.18__tar.gz

{sql_glider-0.1.15 → sql_glider-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.15
+Version: 0.1.18
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

{sql_glider-0.1.15 → sql_glider-0.1.18}/src/sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.15'
-__version_tuple__ = version_tuple = (0, 1, 15)
+__version__ = version = '0.1.18'
+__version_tuple__ = version_tuple = (0, 1, 18)
 
 __commit_id__ = commit_id = None

{sql_glider-0.1.15 → sql_glider-0.1.18}/src/sqlglider/lineage/analyzer.py

@@ -11,6 +11,48 @@ from sqlglot.lineage import Node, lineage
 from sqlglider.global_models import AnalysisLevel
 
 
+def _flat_schema_to_nested(
+    schema: Dict[str, Dict[str, str]],
+) -> Dict[str, object]:
+    """Convert flat dot-notation schema keys to the nested dict structure sqlglot expects.
+
+    sqlglot's MappingSchema requires consistent nesting depth across all tables.
+    Flat keys like ``"db.table"`` are split on dots and nested accordingly.
+    Shorter keys are padded with empty-string prefixes to match the max depth.
+
+    Examples::
+
+        {"users": {"id": "UNKNOWN"}}
+        → {"users": {"id": "UNKNOWN"}}  (depth 1, no change)
+
+        {"db.users": {"id": "UNKNOWN"}, "my_view": {"x": "UNKNOWN"}}
+        → {"db": {"users": {"id": "UNKNOWN"}}, "": {"my_view": {"x": "UNKNOWN"}}}
+    """
+    if not schema:
+        return {}
+
+    # Split all keys into parts
+    entries = [(key.split("."), cols) for key, cols in schema.items()]
+    max_depth = max(len(parts) for parts, _ in entries)
+
+    # If all keys are single-part (unqualified), return as-is
+    if max_depth == 1:
+        return schema  # type: ignore[return-value]
+
+    # Pad shorter keys with empty-string prefixes to match max depth
+    nested: Dict[str, object] = {}
+    for parts, cols in entries:
+        while len(parts) < max_depth:
+            parts.insert(0, "")
+        d: Dict[str, object] = nested
+        for part in parts[:-1]:
+            if part not in d:
+                d[part] = {}
+            d = d[part]  # type: ignore[assignment]
+        d[parts[-1]] = cols
+    return nested
+
+
 class StarResolutionError(Exception):
     """Raised when SELECT * cannot be resolved and no_star mode is enabled."""
 
@@ -860,8 +902,10 @@ class LineageAnalyzer:
             current_query_sql = self.expr.sql(dialect=self.dialect)
 
         # Prune schema to only tables referenced in this query to avoid
-        # sqlglot.lineage() performance degradation with large schema dicts
-        pruned_schema: Optional[Dict[str, Dict[str, str]]] = None
+        # sqlglot.lineage() performance degradation with large schema dicts.
+        # Then convert from flat dot-notation keys to the nested dict structure
+        # that sqlglot's MappingSchema expects.
+        lineage_schema: Optional[Dict[str, object]] = None
         if self._file_schema:
             referenced = {t.lower() for t in self._get_query_tables()}
             pruned_schema = {
@@ -869,8 +913,8 @@ class LineageAnalyzer:
                 for table, cols in self._file_schema.items()
                 if table.lower() in referenced
             }
-            if not pruned_schema:
-                pruned_schema = None
+            if pruned_schema:
+                lineage_schema = _flat_schema_to_nested(pruned_schema)
 
         for col in columns_to_analyze:
             try:
@@ -883,7 +927,7 @@ class LineageAnalyzer:
                     lineage_col,
                     current_query_sql,
                     dialect=self.dialect,
-                    schema=pruned_schema,
+                    schema=lineage_schema,
                 )
 
                 # Collect all source columns
@@ -1133,7 +1177,7 @@ class LineageAnalyzer:
         if table.db:
             parts.append(table.db)
         parts.append(table.name)
-        return ".".join(parts)
+        return ".".join(parts).lower()
 
     def _resolve_table_reference(self, ref: str, select_node: exp.Select) -> str:
         """
@@ -1478,7 +1522,7 @@ class LineageAnalyzer:
 
         if columns:
             # Store with UNKNOWN type - SQLGlot only needs column names for expansion
-            self._file_schema[target_name] = {col: "UNKNOWN" for col in columns}
+            self._file_schema[target_name] = {col.lower(): "UNKNOWN" for col in columns}
 
     def _extract_schema_from_dql(self, expr: exp.Expression) -> None:
         """Infer table schemas from column references in DQL.
@@ -1590,8 +1634,9 @@ class LineageAnalyzer:
 
                 if actual_table not in self._file_schema:
                     self._file_schema[actual_table] = {}
-                if col_name not in self._file_schema[actual_table]:
-                    self._file_schema[actual_table][col_name] = "UNKNOWN"
+                col_lower = col_name.lower()
+                if col_lower not in self._file_schema[actual_table]:
+                    self._file_schema[actual_table][col_lower] = "UNKNOWN"
 
     def _extract_columns_from_select(
         self, select_node: Union[exp.Select, exp.Union, exp.Intersect, exp.Except]

{sql_glider-0.1.15 → sql_glider-0.1.18}/src/sqlglider/schema/extractor.py

@@ -41,7 +41,14 @@ def extract_schemas_from_files(
     if console is None:
         console = Console(stderr=True)
 
-    schema: SchemaDict = dict(initial_schema) if initial_schema else {}
+    schema: SchemaDict = (
+        {
+            k.lower(): {c.lower(): v for c, v in cols.items()}
+            for k, cols in initial_schema.items()
+        }
+        if initial_schema
+        else {}
+    )
     total = len(file_paths)
 
     with Progress(
@@ -65,7 +72,11 @@ def extract_schemas_from_files(
                     strict_schema=strict_schema,
                 )
                 file_schema = analyzer.extract_schema_only()
-                schema.update(file_schema)
+                for table_name, columns in file_schema.items():
+                    if table_name in schema:
+                        schema[table_name].update(columns)
+                    else:
+                        schema[table_name] = columns
             except SchemaResolutionError:
                 raise
             except Exception:

{sql_glider-0.1.15 → sql_glider-0.1.18}/src/sqlglider/utils/schema.py

@@ -46,7 +46,7 @@ def parse_ddl_to_schema(ddl: str, dialect: str = "spark") -> Dict[str, Dict[str,
         table_name = _get_qualified_name(target)
 
         if columns:
-            schema[table_name] = {col: "UNKNOWN" for col in columns}
+            schema[table_name] = {col.lower(): "UNKNOWN" for col in columns}
 
     return schema
 
@@ -59,4 +59,4 @@ def _get_qualified_name(table: exp.Table) -> str:
     if table.db:
         parts.append(table.db)
     parts.append(table.name)
-    return ".".join(parts)
+    return ".".join(parts).lower()

{sql_glider-0.1.15 → sql_glider-0.1.18}/tests/sqlglider/lineage/test_analyzer.py

@@ -3,7 +3,11 @@
 import pytest
 
 from sqlglider.global_models import AnalysisLevel
-from sqlglider.lineage.analyzer import LineageAnalyzer, StarResolutionError
+from sqlglider.lineage.analyzer import (
+    LineageAnalyzer,
+    StarResolutionError,
+    _flat_schema_to_nested,
+)
 
 
 class TestCaseInsensitiveForwardLineage:
@@ -115,39 +119,39 @@ class TestCaseInsensitiveForwardLineage:
             # Lowercase
             (
                 "target_table.customer_name",
-                "TARGET_TABLE.customer_name",
+                "target_table.customer_name",
                 ["customers.customer_name"],
             ),
             (
                 "target_table.region",
-                "TARGET_TABLE.region",
+                "target_table.region",
                 ["customers.region"],
             ),
             (
                 "target_table.total_amount",
-                "TARGET_TABLE.total_amount",
+                "target_table.total_amount",
                 ["orders.order_amount"],
             ),
             # Uppercase
             (
-                "TARGET_TABLE.CUSTOMER_NAME",
-                "TARGET_TABLE.customer_name",
+                "target_table.CUSTOMER_NAME",
+                "target_table.customer_name",
                 ["customers.customer_name"],
             ),
             (
-                "TARGET_TABLE.REGION",
-                "TARGET_TABLE.region",
+                "target_table.REGION",
+                "target_table.region",
                 ["customers.region"],
             ),
             # Mixed case
             (
                 "TaRgEt_TaBlE.CuStOmEr_NaMe",
-                "TARGET_TABLE.customer_name",
+                "target_table.customer_name",
                 ["customers.customer_name"],
             ),
             (
                 "target_TABLE.REGION",
-                "TARGET_TABLE.region",
+                "target_table.region",
                 ["customers.region"],
             ),
         ],
@@ -342,39 +346,39 @@ class TestCaseInsensitiveReverseLineage:
             (
                 "customers.customer_name",
                 "customers.customer_name",
-                ["TARGET_TABLE.customer_name"],
+                ["target_table.customer_name"],
             ),
             (
                 "customers.region",
                 "customers.region",
-                ["TARGET_TABLE.region"],
+                ["target_table.region"],
             ),
             (
                 "orders.order_amount",
                 "orders.order_amount",
-                ["TARGET_TABLE.segment", "TARGET_TABLE.total_amount"],
+                ["target_table.segment", "target_table.total_amount"],
             ),
             # Uppercase
             (
                 "CUSTOMERS.CUSTOMER_NAME",
                 "customers.customer_name",
-                ["TARGET_TABLE.customer_name"],
+                ["target_table.customer_name"],
             ),
             (
                 "CUSTOMERS.REGION",
                 "customers.region",
-                ["TARGET_TABLE.region"],
+                ["target_table.region"],
             ),
             # Mixed case
             (
                 "CuStOmErS.CuStOmEr_NaMe",
                 "customers.customer_name",
-                ["TARGET_TABLE.customer_name"],
+                ["target_table.customer_name"],
             ),
             (
                 "cUsToMeRs.ReGiOn",
                 "customers.region",
-                ["TARGET_TABLE.region"],
+                ["target_table.region"],
             ),
         ],
     )
@@ -3181,3 +3185,97 @@ class TestSchemaPruning:
         output_names = {item.output_name for r in results for item in r.lineage_items}
         assert "id" in output_names
         assert "email" in output_names
+
+
+class TestFlatSchemaToNested:
+    """Tests for _flat_schema_to_nested conversion utility."""
+
+    def test_empty(self):
+        assert _flat_schema_to_nested({}) == {}
+
+    def test_unqualified_passthrough(self):
+        schema = {"users": {"id": "UNKNOWN"}}
+        assert _flat_schema_to_nested(schema) == schema
+
+    def test_two_part_keys(self):
+        schema = {"db.users": {"id": "UNKNOWN"}}
+        result = _flat_schema_to_nested(schema)
+        assert result == {"db": {"users": {"id": "UNKNOWN"}}}
+
+    def test_three_part_keys(self):
+        schema = {"cat.db.users": {"id": "UNKNOWN"}}
+        result = _flat_schema_to_nested(schema)
+        assert result == {"cat": {"db": {"users": {"id": "UNKNOWN"}}}}
+
+    def test_mixed_depth_pads_shorter_keys(self):
+        schema = {
+            "my_view": {"x": "UNKNOWN"},
+            "db.users": {"id": "UNKNOWN"},
+        }
+        result = _flat_schema_to_nested(schema)
+        assert result == {
+            "": {"my_view": {"x": "UNKNOWN"}},
+            "db": {"users": {"id": "UNKNOWN"}},
+        }
+
+
+class TestQualifiedSchemaKeys:
+    """Tests for schema with qualified (dotted) table names."""
+
+    def test_qualified_star_expansion(self):
+        """SELECT * resolves correctly with qualified schema keys."""
+        sql = "SELECT * FROM mydb.users"
+        schema = {"mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("mydb.users.id", "id") in items
+        assert ("mydb.users.name", "name") in items
+
+    def test_qualified_explicit_columns(self):
+        """Explicit columns trace sources correctly with qualified schema keys."""
+        sql = "SELECT id, name FROM mydb.users"
+        schema = {"mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("mydb.users.id", "mydb.users.id") in items
+        assert ("mydb.users.name", "mydb.users.name") in items
+
+    def test_three_part_qualified(self):
+        """3-part qualified names (catalog.db.table) work correctly."""
+        sql = "SELECT id FROM catalog.mydb.users"
+        schema = {"catalog.mydb.users": {"id": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = [
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        ]
+        assert len(items) == 1
+        assert items[0] == ("catalog.mydb.users.id", "catalog.mydb.users.id")
+
+    def test_mixed_qualified_and_unqualified(self):
+        """Mix of qualified and unqualified table names in schema."""
+        sql = "SELECT * FROM my_view"
+        schema = {
+            "my_view": {"id": "UNKNOWN"},
+            "mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"},
+        }
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("my_view.id", "id") in items

{sql_glider-0.1.15 → sql_glider-0.1.18}/tests/sqlglider/schema/test_extractor.py

@@ -81,6 +81,58 @@ class TestExtractSchemasFromFiles:
         assert "id" in schema["customers"]
         assert "order_id" in schema["orders"]
 
+    def test_merges_columns_for_same_table(self, tmp_path, console):
+        """Test that columns are merged when the same table appears in multiple files."""
+        file1 = tmp_path / "a.sql"
+        file1.write_text("SELECT c.id, c.name FROM customers c;")
+
+        file2 = tmp_path / "b.sql"
+        file2.write_text("SELECT c.id, c.age FROM customers c;")
+
+        schema = extract_schemas_from_files(
+            [file1, file2], dialect="spark", console=console
+        )
+
+        assert "customers" in schema
+        assert "id" in schema["customers"]
+        assert "name" in schema["customers"]
+        assert "age" in schema["customers"]
+
+    def test_merges_columns_case_insensitive(self, tmp_path, console):
+        """Test that tables with different casing are merged into one entry."""
+        file1 = tmp_path / "a.sql"
+        file1.write_text("SELECT c.id, c.name FROM Customers c;")
+
+        file2 = tmp_path / "b.sql"
+        file2.write_text("SELECT c.id, c.AGE FROM customers c;")
+
+        schema = extract_schemas_from_files(
+            [file1, file2], dialect="spark", console=console
+        )
+
+        assert "customers" in schema
+        assert len([k for k in schema if k.lower() == "customers"]) == 1
+        assert "id" in schema["customers"]
+        assert "name" in schema["customers"]
+        assert "age" in schema["customers"]
+
+    def test_initial_schema_normalized(self, tmp_path, console):
+        """Test that initial schema keys are normalized to lowercase."""
+        sql_file = tmp_path / "query.sql"
+        sql_file.write_text("SELECT o.id FROM orders o;")
+
+        initial = {"Existing_Table": {"Col1": "UNKNOWN"}}
+        schema = extract_schemas_from_files(
+            [sql_file],
+            dialect="spark",
+            initial_schema=initial,
+            console=console,
+        )
+
+        assert "existing_table" in schema
+        assert "col1" in schema["existing_table"]
+        assert "orders" in schema
+
     def test_initial_schema_preserved(self, tmp_path, console):
         """Test that initial schema is included in result."""
         sql_file = tmp_path / "query.sql"