sql-glider 0.1.12__tar.gz → 0.1.13__tar.gz

{sql_glider-0.1.12 → sql_glider-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.12
+Version: 0.1.13
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

{sql_glider-0.1.12 → sql_glider-0.1.13}/src/sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.12'
-__version_tuple__ = version_tuple = (0, 1, 12)
+__version__ = version = '0.1.13'
+__version_tuple__ = version_tuple = (0, 1, 13)
 
 __commit_id__ = commit_id = None

{sql_glider-0.1.12 → sql_glider-0.1.13}/src/sqlglider/cli.py

@@ -1166,36 +1166,57 @@ def graph_build(
             strict_schema=strict_schema,
         )
 
-        # Process manifest if provided
-        if manifest:
-            builder.add_manifest(manifest, dialect=dialect)
-
-        # Process paths - collect all files first for progress tracking
+        # Collect file paths for schema extraction
+        path_files: list[Path] = []
         if paths:
-            all_files: list[Path] = []
             for path in paths:
                 if path.is_dir():
                     pattern = f"**/{glob_pattern}" if recursive else glob_pattern
-                    all_files.extend(
+                    path_files.extend(
                         f for f in sorted(path.glob(pattern)) if f.is_file()
                     )
                 elif path.is_file():
-                    all_files.append(path)
+                    path_files.append(path)
                 else:
                     err_console.print(f"[red]Error:[/red] Path not found: {path}")
                     raise typer.Exit(1)
-            builder.add_files(all_files, dialect=dialect)
 
-        # Dump resolved schema if requested
-        if dump_schema:
-            from sqlglider.graph.formatters import format_schema
+        manifest_files: list[Path] = []
+        if manifest:
+            from sqlglider.graph.models import Manifest
+
+            manifest_data = Manifest.from_csv(manifest)
+            base_dir = manifest.parent
+            for entry in manifest_data.entries:
+                file_path = Path(entry.file_path)
+                if not file_path.is_absolute():
+                    file_path = (base_dir / entry.file_path).resolve()
+                manifest_files.append(file_path)
+
+        # Extract schema upfront if requested, then dump before graph building
+        all_files = manifest_files + path_files
+        if resolve_schema and all_files:
+            builder.extract_schemas(all_files, dialect=dialect)
+
+            if dump_schema:
+                from sqlglider.graph.formatters import format_schema
+
+                schema_content = format_schema(
+                    builder.resolved_schema, dump_schema_format
+                )
+                dump_schema.write_text(schema_content, encoding="utf-8")
+                console.print(
+                    f"[green]Schema dumped to {dump_schema} "
+                    f"({len(builder.resolved_schema)} table(s))[/green]"
+                )
+
+        # Process manifest if provided
+        if manifest:
+            builder.add_manifest(manifest, dialect=dialect)
 
-            schema_content = format_schema(builder.resolved_schema, dump_schema_format)
-            dump_schema.write_text(schema_content, encoding="utf-8")
-            console.print(
-                f"[green]Schema dumped to {dump_schema} "
-                f"({len(builder.resolved_schema)} table(s))[/green]"
-            )
+        # Process path-based files
+        if path_files:
+            builder.add_files(path_files, dialect=dialect)
 
         # Build and save graph
         graph = builder.build()

{sql_glider-0.1.12 → sql_glider-0.1.13}/src/sqlglider/graph/builder.py

@@ -235,19 +235,10 @@ class GraphBuilder:
         if not files_with_dialects:
             return self
 
-        # Two-pass schema resolution
-        if self.resolve_schema:
-            console.print("[blue]Pass 1: Extracting schema from files[/blue]")
+        # Two-pass schema resolution (skip if already resolved)
+        if self.resolve_schema and not self._resolved_schema:
             file_paths_only = [fp for fp, _ in files_with_dialects]
-            self._resolved_schema = self._extract_schemas(file_paths_only, dialect)
-            if self.catalog_type:
-                self._resolved_schema = self._fill_schema_from_catalog(
-                    self._resolved_schema, file_paths_only, dialect
-                )
-            console.print(
-                f"[blue]Schema resolved for "
-                f"{len(self._resolved_schema)} table(s)[/blue]"
-            )
+            self.extract_schemas(file_paths_only, dialect)
 
         total = len(files_with_dialects)
         description = "Pass 2: Analyzing lineage" if self.resolve_schema else "Parsing"
@@ -286,18 +277,9 @@ class GraphBuilder:
         if not file_paths:
             return self
 
-        # Two-pass schema resolution: extract schema from all files first
-        if self.resolve_schema:
-            console.print("[blue]Pass 1: Extracting schema from files[/blue]")
-            self._resolved_schema = self._extract_schemas(file_paths, dialect)
-            if self.catalog_type:
-                self._resolved_schema = self._fill_schema_from_catalog(
-                    self._resolved_schema, file_paths, dialect
-                )
-            console.print(
-                f"[blue]Schema resolved for "
-                f"{len(self._resolved_schema)} table(s)[/blue]"
-            )
+        # Two-pass schema resolution (skip if already resolved)
+        if self.resolve_schema and not self._resolved_schema:
+            self.extract_schemas(file_paths, dialect)
 
         if show_progress:
             total = len(file_paths)
@@ -321,6 +303,34 @@ class GraphBuilder:
                 self.add_file(file_path, dialect)
         return self
 
+    def extract_schemas(
+        self,
+        file_paths: List[Path],
+        dialect: Optional[str] = None,
+    ) -> Dict[str, Dict[str, str]]:
+        """Run schema extraction pass and optionally fill from catalog.
+
+        Call this before add_files/add_manifest to resolve schema upfront.
+        The resolved schema is stored internally and also returned.
+
+        Args:
+            file_paths: SQL files to extract schema from
+            dialect: SQL dialect override
+
+        Returns:
+            Resolved schema dict
+        """
+        console.print("[blue]Pass 1: Extracting schema from files[/blue]")
+        self._resolved_schema = self._extract_schemas(file_paths, dialect)
+        if self.catalog_type:
+            self._resolved_schema = self._fill_schema_from_catalog(
+                self._resolved_schema, file_paths, dialect
+            )
+        console.print(
+            f"[blue]Schema resolved for {len(self._resolved_schema)} table(s)[/blue]"
+        )
+        return self._resolved_schema.copy()
+
     def _extract_schemas(
         self,
         file_paths: List[Path],

{sql_glider-0.1.12 → sql_glider-0.1.13}/src/sqlglider/lineage/analyzer.py

@@ -859,18 +859,31 @@ class LineageAnalyzer:
         else:
             current_query_sql = self.expr.sql(dialect=self.dialect)
 
+        # Prune schema to only tables referenced in this query to avoid
+        # sqlglot.lineage() performance degradation with large schema dicts
+        pruned_schema: Optional[Dict[str, Dict[str, str]]] = None
+        if self._file_schema:
+            referenced = {t.lower() for t in self._get_query_tables()}
+            pruned_schema = {
+                table: cols
+                for table, cols in self._file_schema.items()
+                if table.lower() in referenced
+            }
+            if not pruned_schema:
+                pruned_schema = None
+
         for col in columns_to_analyze:
             try:
                 # Get the column name that lineage expects
                 lineage_col = self._column_mapping.get(col, col)
 
                 # Get lineage tree for this column using current query SQL only
-                # Pass file schema to enable SELECT * expansion for known tables/views
+                # Pass pruned schema to enable SELECT * expansion for known tables/views
                 node = lineage(
                     lineage_col,
                     current_query_sql,
                     dialect=self.dialect,
-                    schema=self._file_schema if self._file_schema else None,
+                    schema=pruned_schema,
                 )
 
                 # Collect all source columns

{sql_glider-0.1.12 → sql_glider-0.1.13}/tests/sqlglider/graph/test_builder.py

@@ -842,3 +842,45 @@ class TestResolvedSchemaProperty:
         schema = builder.resolved_schema
         schema["injected"] = {"col": "UNKNOWN"}
         assert "injected" not in builder.resolved_schema
+
+
+class TestExtractSchemas:
+    """Tests for the public extract_schemas method."""
+
+    def test_extract_schemas_returns_schema(self, tmp_path):
+        """extract_schemas returns inferred schema from files."""
+        sql_file = tmp_path / "query.sql"
+        sql_file.write_text("SELECT c.id, c.name FROM customers c;")
+        builder = GraphBuilder(resolve_schema=True)
+        schema = builder.extract_schemas([sql_file])
+        assert "customers" in schema
+        assert "id" in schema["customers"]
+        assert "name" in schema["customers"]
+
+    def test_extract_schemas_before_add_files(self, tmp_path):
+        """Calling extract_schemas before add_files avoids duplicate extraction."""
+        schema_file = tmp_path / "schema.sql"
+        schema_file.write_text("SELECT c.id, c.name FROM customers c;")
+        query_file = tmp_path / "query.sql"
+        query_file.write_text("SELECT * FROM customers;")
+
+        builder = GraphBuilder(resolve_schema=True)
+        schema = builder.extract_schemas([schema_file, query_file])
+        assert "customers" in schema
+
+        # add_files should skip Pass 1 since schema is already resolved
+        builder.add_files([query_file])
+        graph = builder.build()
+        assert graph is not None
+
+        # Schema should still be the same
+        assert builder.resolved_schema == schema
+
+    def test_extract_schemas_populates_resolved_schema(self, tmp_path):
+        """extract_schemas populates the resolved_schema property."""
+        sql_file = tmp_path / "query.sql"
+        sql_file.write_text("SELECT u.id, u.email FROM users u;")
+        builder = GraphBuilder(resolve_schema=True)
+        assert builder.resolved_schema == {}
+        builder.extract_schemas([sql_file])
+        assert "users" in builder.resolved_schema

{sql_glider-0.1.12 → sql_glider-0.1.13}/tests/sqlglider/lineage/test_analyzer.py

@@ -3133,3 +3133,51 @@ class TestSchemaParam:
         schema = analyzer.get_extracted_schema()
         assert "v1" in schema
         assert set(schema["v1"].keys()) == {"id", "name"}
+
+
+class TestSchemaPruning:
+    """Tests that schema pruning doesn't affect lineage correctness."""
+
+    def test_large_schema_same_results_as_small(self):
+        """Lineage results are identical with a large unreferenced schema."""
+        sql = "SELECT c.id, c.name FROM customers c"
+
+        small_schema = {
+            "customers": {"id": "UNKNOWN", "name": "UNKNOWN"},
+        }
+        big_schema = dict(small_schema)
+        for i in range(200):
+            big_schema[f"unrelated_table_{i}"] = {
+                f"col_{j}": "UNKNOWN" for j in range(20)
+            }
+
+        analyzer_small = LineageAnalyzer(sql, dialect="spark", schema=small_schema)
+        results_small = analyzer_small.analyze_queries(level=AnalysisLevel.COLUMN)
+
+        analyzer_big = LineageAnalyzer(sql, dialect="spark", schema=big_schema)
+        results_big = analyzer_big.analyze_queries(level=AnalysisLevel.COLUMN)
+
+        items_small = [
+            (item.output_name, item.source_name)
+            for r in results_small
+            for item in r.lineage_items
+        ]
+        items_big = [
+            (item.output_name, item.source_name)
+            for r in results_big
+            for item in r.lineage_items
+        ]
+        assert items_small == items_big
+
+    def test_star_expansion_works_with_pruned_schema(self):
+        """SELECT * expansion still works when schema is pruned."""
+        sql = "SELECT * FROM users"
+        schema = {
+            "users": {"id": "UNKNOWN", "email": "UNKNOWN"},
+            "unrelated": {"col": "UNKNOWN"},
+        }
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        output_names = {item.output_name for r in results for item in r.lineage_items}
+        assert "id" in output_names
+        assert "email" in output_names