sql-glider 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

{sql_glider-0.1.12.dist-info → sql_glider-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.12
+Version: 0.1.14
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

{sql_glider-0.1.12.dist-info → sql_glider-0.1.14.dist-info}/RECORD

@@ -1,6 +1,6 @@
 sqlglider/__init__.py,sha256=gDf7s52dMcX7JuCZ1SLawcB1vb3U0yJCohu9RQAATBY,125
-sqlglider/_version.py,sha256=cEPXLUpTV7EzqolnyXW8nf8Hr6IVyBji9CzB6Cq_Ar0,706
-sqlglider/cli.py,sha256=qEDLZ1a6yr-BzrtkBsJEHPByMmRERsGKZsYFTn9kaMY,55624
+sqlglider/_version.py,sha256=1asLxKIxr0ym19WewGl0URtkfnEuN7mK8ZckB9dZw6Q,706
+sqlglider/cli.py,sha256=UvDaeDhQRu98M1PaUtWsIL_F_LtulOf58kWGn6SxUzE,64175
 sqlglider/global_models.py,sha256=2vyJXAuXOsXQpE-D3F0ejj7eR9z0nDWFjTkielhzM8k,356
 sqlglider/catalog/__init__.py,sha256=2PqFPyzFXJ14FpSUcBmVK2L-a_ypWQHAbHFHxLDk_LE,814
 sqlglider/catalog/base.py,sha256=R7htHC43InpH4uRjYk33dMYYji6oylHns7Ye_mgfjJE,3116
@@ -11,15 +11,17 @@ sqlglider/dissection/analyzer.py,sha256=-GD3-lTbfBthq1BW6HiDjvJx2y4LDmnUVHIVIb0H
 sqlglider/dissection/formatters.py,sha256=M7gsmTNljRIeLIRv4D0vHvqJVrTqWSpsg7vem83zSzY,7302
 sqlglider/dissection/models.py,sha256=RRD3RIteqbUBY6e-74skKDvMH3qeAUaqA2sFcrjP5GQ,3618
 sqlglider/graph/__init__.py,sha256=4DDdrPM75CmeQWt7wHdBsjCm1s70BHGLYdijIbaUEKY,871
-sqlglider/graph/builder.py,sha256=o0SnH5eWUUPpzRSdsdCXEva3QTlhLDagJulJ2hRFQqA,19895
+sqlglider/graph/builder.py,sha256=fS6p-73zyjuYIHRzM3uXFTFZ8zyal0s7oBdyO2Fv8vQ,15224
 sqlglider/graph/formatters.py,sha256=EGgdxTr9Mctz9tTN54XIjoX0KGNcpiSKsW3o27dhMxo,2549
 sqlglider/graph/merge.py,sha256=uUZlm4BN3S9gRL66Cc2mzhbtuh4SVAv2n4cN4eUEQBU,4077
 sqlglider/graph/models.py,sha256=EYmjv_WzDSNp_WfhJ6H-qBIOkAcoNKS7GRUryfKrHuY,9330
 sqlglider/graph/query.py,sha256=LHU8Cvn7ZPPSEnqdDn2pF8f1_LQjIvNIrZqs8cFlb6U,9433
 sqlglider/graph/serialization.py,sha256=vMXn7s35jA499e7l90vNVaJE_3QR_VHf3rEfQ9ZlgTQ,2781
 sqlglider/lineage/__init__.py,sha256=llXMeI5_PIZaiBo8tKk3-wOubF4m_6QBHbn1FtWxT7k,256
-sqlglider/lineage/analyzer.py,sha256=-LUeVNEsjfEWoKAJ2qVIiJO1noqwae4jQkwkkkVbAT8,75950
+sqlglider/lineage/analyzer.py,sha256=08pFR5aGFFPhSbRW6EqiX2d3mp91v-orcs6dm_T1FJg,76484
 sqlglider/lineage/formatters.py,sha256=_Y9wcTX4JXn1vVnZ1xI656g1FF2rMjcAVc-GHjbd9QA,10389
+sqlglider/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sqlglider/schema/extractor.py,sha256=iOi13ZStR4ngC2GkZGXjB0lsgmDqJ-OYwTRgH72hy1w,7082
 sqlglider/templating/__init__.py,sha256=g3_wb6rSDI0usq2UUMDpn-J5kVwlAw3NtLdwbxL6UHs,1435
 sqlglider/templating/base.py,sha256=y5bWAW7qXl_4pPyo5KycfHwNVvt1-7slZ63DAsvTE1s,2902
 sqlglider/templating/jinja.py,sha256=o01UG72N4G1-tOT5LKK1Wkccv4nJH2VN4VFaMi5c1-g,5220
@@ -29,8 +31,8 @@ sqlglider/utils/__init__.py,sha256=KGp9-UzKz_OFBOTFoSy-g-NXDZsvyWXG_9-1zcC6ePE,2
 sqlglider/utils/config.py,sha256=qx5zE9pjLCCzHQDFVPLVd7LgJ-lghxUa2x-aZOAHByY,4962
 sqlglider/utils/file_utils.py,sha256=5_ff28E0r1R7emZzsOnRuHd-7zIX6873eyr1SuPEr4E,1093
 sqlglider/utils/schema.py,sha256=-0Vd1A3EggBH3reXTiabO0zFeTENROgmDg861X1D7Qs,1867
-sql_glider-0.1.12.dist-info/METADATA,sha256=73yuoWaAE5DKE9wobDXxbERSP2Pq-WpdqCnaswAa9fQ,28446
-sql_glider-0.1.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-sql_glider-0.1.12.dist-info/entry_points.txt,sha256=HDuakHqHS5C0HFKsMIxMYmDU7-BLBGrnIJcYaVRu-s0,251
-sql_glider-0.1.12.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sql_glider-0.1.12.dist-info/RECORD,,
+sql_glider-0.1.14.dist-info/METADATA,sha256=SdedCDEPwWR2Kqrg8_mMgb0PNmNZj0OExHiapyKZ63A,28446
+sql_glider-0.1.14.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+sql_glider-0.1.14.dist-info/entry_points.txt,sha256=HDuakHqHS5C0HFKsMIxMYmDU7-BLBGrnIJcYaVRu-s0,251
+sql_glider-0.1.14.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sql_glider-0.1.14.dist-info/RECORD,,

sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.12'
-__version_tuple__ = version_tuple = (0, 1, 12)
+__version__ = version = '0.1.14'
+__version_tuple__ = version_tuple = (0, 1, 14)
 
 __commit_id__ = commit_id = None

sqlglider/cli.py

@@ -788,6 +788,274 @@ def tables_pull(
         raise typer.Exit(1)
 
 
+def _collect_sql_files(
+    paths: Optional[List[Path]],
+    manifest: Optional[Path],
+    recursive: bool,
+    glob_pattern: str,
+) -> tuple[list[Path], list[Path]]:
+    """Collect SQL files from paths and/or manifest.
+
+    Args:
+        paths: File or directory paths to scan.
+        manifest: Optional manifest CSV path.
+        recursive: Whether to recurse into directories.
+        glob_pattern: Glob pattern for directory scanning.
+
+    Returns:
+        Tuple of (manifest_files, path_files).
+    """
+    path_files: list[Path] = []
+    if paths:
+        for path in paths:
+            if path.is_dir():
+                pattern = f"**/{glob_pattern}" if recursive else glob_pattern
+                path_files.extend(f for f in sorted(path.glob(pattern)) if f.is_file())
+            elif path.is_file():
+                path_files.append(path)
+            else:
+                err_console.print(f"[red]Error:[/red] Path not found: {path}")
+                raise typer.Exit(1)
+
+    manifest_files: list[Path] = []
+    if manifest:
+        from sqlglider.graph.models import Manifest
+
+        manifest_data = Manifest.from_csv(manifest)
+        base_dir = manifest.parent
+        for entry in manifest_data.entries:
+            file_path = Path(entry.file_path)
+            if not file_path.is_absolute():
+                file_path = (base_dir / entry.file_path).resolve()
+            manifest_files.append(file_path)
+
+    return manifest_files, path_files
+
+
+@tables_app.command("scrape")
+def tables_scrape(
+    paths: List[Path] = typer.Argument(
+        None,
+        help="SQL file(s) or directory path to process",
+    ),
+    recursive: bool = typer.Option(
+        False,
+        "--recursive",
+        "-r",
+        help="Recursively search directories for SQL files",
+    ),
+    glob_pattern: str = typer.Option(
+        "*.sql",
+        "--glob",
+        "-g",
+        help="Glob pattern for matching SQL files in directories",
+    ),
+    manifest: Optional[Path] = typer.Option(
+        None,
+        "--manifest",
+        "-m",
+        exists=True,
+        help="Path to manifest CSV file with file_path and optional dialect columns",
+    ),
+    dialect: Optional[str] = typer.Option(
+        None,
+        "--dialect",
+        "-d",
+        help="SQL dialect (default: spark)",
+    ),
+    templater: Optional[str] = typer.Option(
+        None,
+        "--templater",
+        "-t",
+        help="Templater for SQL preprocessing (e.g., 'jinja', 'none')",
+    ),
+    var: Optional[List[str]] = typer.Option(
+        None,
+        "--var",
+        "-v",
+        help="Template variable in key=value format (repeatable)",
+    ),
+    vars_file: Optional[Path] = typer.Option(
+        None,
+        "--vars-file",
+        exists=True,
+        help="Path to variables file (JSON or YAML)",
+    ),
+    strict_schema: bool = typer.Option(
+        False,
+        "--strict-schema",
+        help="Fail if any column's table cannot be identified during schema extraction",
+    ),
+    catalog_type: Optional[str] = typer.Option(
+        None,
+        "--catalog-type",
+        "-c",
+        help="Catalog provider for pulling DDL of tables not found in files "
+        "(e.g. 'databricks')",
+    ),
+    output_format: Optional[str] = typer.Option(
+        None,
+        "--output-format",
+        "-f",
+        help="Output format: 'text' (default), 'json', or 'csv'",
+    ),
+    output_file: Optional[Path] = typer.Option(
+        None,
+        "--output-file",
+        "-o",
+        help="Output file path (prints to stdout if not provided)",
+    ),
+) -> None:
+    """
+    Scrape schema information from SQL files.
+
+    Infers table and column schemas from DDL statements and DQL column
+    references across one or more SQL files. Supports the same file input
+    modes as `graph build` (paths, directories, manifests).
+
+    Examples:
+
+        # Scrape schema from a directory
+        sqlglider tables scrape ./queries/ -r
+
+        # Output as JSON
+        sqlglider tables scrape ./queries/ -r -f json
+
+        # Save to file
+        sqlglider tables scrape ./queries/ -r -f csv -o schema.csv
+
+        # With Jinja2 templating
+        sqlglider tables scrape ./queries/ -r --templater jinja --var schema=prod
+
+        # With catalog fallback
+        sqlglider tables scrape ./queries/ -r -c databricks
+    """
+    from sqlglider.graph.formatters import format_schema
+    from sqlglider.lineage.analyzer import SchemaResolutionError
+    from sqlglider.schema.extractor import extract_and_resolve_schema
+
+    # Load config for defaults
+    config = load_config()
+    dialect = dialect or config.dialect or "spark"
+    templater = templater or config.templater
+    strict_schema = strict_schema or config.strict_schema or False
+    output_format = output_format or config.output_format or "text"
+
+    if output_format not in ("text", "json", "csv"):
+        err_console.print(
+            f"[red]Error:[/red] Invalid --output-format '{output_format}'. "
+            "Use 'text', 'json', or 'csv'."
+        )
+        raise typer.Exit(1)
+
+    # Only inherit catalog_type from config when not provided via CLI
+    if not catalog_type:
+        catalog_type = config.catalog_type
+
+    # Validate inputs
+    if not paths and not manifest:
+        err_console.print(
+            "[red]Error:[/red] Must provide either file/directory paths or --manifest option."
+        )
+        raise typer.Exit(1)
+
+    # Create SQL preprocessor if templating is enabled
+    sql_preprocessor: Optional[Callable[[str, Path], str]] = None
+    if templater:
+        config_vars_file = None
+        config_vars = None
+        if config.templating:
+            if config.templating.variables_file and not vars_file:
+                config_vars_file = Path(config.templating.variables_file)
+                if not config_vars_file.exists():
+                    err_console.print(
+                        f"[yellow]Warning:[/yellow] Variables file from config "
+                        f"not found: {config_vars_file}"
+                    )
+                    config_vars_file = None
+            config_vars = config.templating.variables
+
+        variables = load_all_variables(
+            cli_vars=var,
+            vars_file=vars_file or config_vars_file,
+            config_vars=config_vars,
+            use_env=True,
+        )
+
+        templater_instance = get_templater(templater)
+
+        def _preprocess(sql: str, file_path: Path) -> str:
+            return templater_instance.render(
+                sql, variables=variables, source_path=file_path
+            )
+
+        sql_preprocessor = _preprocess
+
+    try:
+        # Build catalog config from config file if available
+        catalog_config_dict = None
+        if catalog_type and config.catalog:
+            provider_config = getattr(config.catalog, catalog_type, None)
+            if provider_config:
+                catalog_config_dict = provider_config.model_dump(exclude_none=True)
+
+        # Collect files
+        manifest_files, path_files = _collect_sql_files(
+            paths, manifest, recursive, glob_pattern
+        )
+        all_files = manifest_files + path_files
+
+        if not all_files:
+            err_console.print("[yellow]Warning:[/yellow] No SQL files found.")
+            raise typer.Exit(0)
+
+        # Extract schema
+        schema = extract_and_resolve_schema(
+            all_files,
+            dialect=dialect,
+            sql_preprocessor=sql_preprocessor,
+            strict_schema=strict_schema,
+            catalog_type=catalog_type,
+            catalog_config=catalog_config_dict,
+            console=err_console,
+        )
+
+        if not schema:
+            err_console.print("[yellow]No schema information found.[/yellow]")
+            raise typer.Exit(0)
+
+        # Format and output
+        formatted = format_schema(schema, output_format)
+        if output_file:
+            OutputWriter.write(formatted, output_file)
+            err_console.print(
+                f"[green]Schema written to {output_file} "
+                f"({len(schema)} table(s))[/green]"
+            )
+        else:
+            console.print(formatted, end="")
+
+    except SchemaResolutionError as e:
+        err_console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+    except FileNotFoundError as e:
+        err_console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+    except TemplaterError as e:
+        err_console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+    except ValueError as e:
+        err_console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+    except Exception as e:
+        err_console.print(f"[red]Error:[/red] Unexpected error: {e}")
+        raise typer.Exit(1)
+
+
 @app.command()
 def template(
     sql_file: Annotated[
@@ -1166,36 +1434,35 @@ def graph_build(
             strict_schema=strict_schema,
         )
 
+        # Collect file paths for schema extraction
+        manifest_files, path_files = _collect_sql_files(
+            paths, manifest, recursive, glob_pattern
+        )
+
+        # Extract schema upfront if requested, then dump before graph building
+        all_files = manifest_files + path_files
+        if resolve_schema and all_files:
+            builder.extract_schemas(all_files, dialect=dialect)
+
+            if dump_schema:
+                from sqlglider.graph.formatters import format_schema
+
+                schema_content = format_schema(
+                    builder.resolved_schema, dump_schema_format
+                )
+                dump_schema.write_text(schema_content, encoding="utf-8")
+                console.print(
+                    f"[green]Schema dumped to {dump_schema} "
+                    f"({len(builder.resolved_schema)} table(s))[/green]"
+                )
+
         # Process manifest if provided
         if manifest:
             builder.add_manifest(manifest, dialect=dialect)
 
-        # Process paths - collect all files first for progress tracking
-        if paths:
-            all_files: list[Path] = []
-            for path in paths:
-                if path.is_dir():
-                    pattern = f"**/{glob_pattern}" if recursive else glob_pattern
-                    all_files.extend(
-                        f for f in sorted(path.glob(pattern)) if f.is_file()
-                    )
-                elif path.is_file():
-                    all_files.append(path)
-                else:
-                    err_console.print(f"[red]Error:[/red] Path not found: {path}")
-                    raise typer.Exit(1)
-            builder.add_files(all_files, dialect=dialect)
-
-        # Dump resolved schema if requested
-        if dump_schema:
-            from sqlglider.graph.formatters import format_schema
-
-            schema_content = format_schema(builder.resolved_schema, dump_schema_format)
-            dump_schema.write_text(schema_content, encoding="utf-8")
-            console.print(
-                f"[green]Schema dumped to {dump_schema} "
-                f"({len(builder.resolved_schema)} table(s))[/green]"
-            )
+        # Process path-based files
+        if path_files:
+            builder.add_files(path_files, dialect=dialect)
 
         # Build and save graph
         graph = builder.build()

sqlglider/graph/builder.py

@@ -16,9 +16,9 @@ from sqlglider.graph.models import (
     LineageGraph,
     Manifest,
 )
-from sqlglider.lineage.analyzer import LineageAnalyzer, SchemaResolutionError
+from sqlglider.lineage.analyzer import LineageAnalyzer
+from sqlglider.schema.extractor import extract_and_resolve_schema
 from sqlglider.utils.file_utils import read_sql_file
-from sqlglider.utils.schema import parse_ddl_to_schema
 
 console = Console(stderr=True)
 
@@ -235,19 +235,10 @@ class GraphBuilder:
         if not files_with_dialects:
             return self
 
-        # Two-pass schema resolution
-        if self.resolve_schema:
-            console.print("[blue]Pass 1: Extracting schema from files[/blue]")
+        # Two-pass schema resolution (skip if already resolved)
+        if self.resolve_schema and not self._resolved_schema:
             file_paths_only = [fp for fp, _ in files_with_dialects]
-            self._resolved_schema = self._extract_schemas(file_paths_only, dialect)
-            if self.catalog_type:
-                self._resolved_schema = self._fill_schema_from_catalog(
-                    self._resolved_schema, file_paths_only, dialect
-                )
-            console.print(
-                f"[blue]Schema resolved for "
-                f"{len(self._resolved_schema)} table(s)[/blue]"
-            )
+            self.extract_schemas(file_paths_only, dialect)
 
         total = len(files_with_dialects)
         description = "Pass 2: Analyzing lineage" if self.resolve_schema else "Parsing"
@@ -286,18 +277,9 @@ class GraphBuilder:
         if not file_paths:
             return self
 
-        # Two-pass schema resolution: extract schema from all files first
-        if self.resolve_schema:
-            console.print("[blue]Pass 1: Extracting schema from files[/blue]")
-            self._resolved_schema = self._extract_schemas(file_paths, dialect)
-            if self.catalog_type:
-                self._resolved_schema = self._fill_schema_from_catalog(
-                    self._resolved_schema, file_paths, dialect
-                )
-            console.print(
-                f"[blue]Schema resolved for "
-                f"{len(self._resolved_schema)} table(s)[/blue]"
-            )
+        # Two-pass schema resolution (skip if already resolved)
+        if self.resolve_schema and not self._resolved_schema:
+            self.extract_schemas(file_paths, dialect)
 
         if show_progress:
             total = len(file_paths)
@@ -321,128 +303,34 @@ class GraphBuilder:
                 self.add_file(file_path, dialect)
         return self
 
-    def _extract_schemas(
+    def extract_schemas(
         self,
         file_paths: List[Path],
         dialect: Optional[str] = None,
     ) -> Dict[str, Dict[str, str]]:
-        """Run schema extraction pass across all files.
+        """Run schema extraction pass and optionally fill from catalog.
 
-        Parses each file and extracts schema from CREATE TABLE/VIEW
-        statements without performing lineage analysis.
+        Call this before add_files/add_manifest to resolve schema upfront.
+        The resolved schema is stored internally and also returned.
 
         Args:
             file_paths: SQL files to extract schema from
             dialect: SQL dialect override
 
         Returns:
-            Accumulated schema dict from all files
+            Resolved schema dict
         """
-        schema: Dict[str, Dict[str, str]] = {}
-        total = len(file_paths)
-        with Progress(
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TaskProgressColumn(),
+        file_dialect = dialect or self.dialect
+        self._resolved_schema = extract_and_resolve_schema(
+            file_paths,
+            dialect=file_dialect,
+            sql_preprocessor=self.sql_preprocessor,
+            strict_schema=self.strict_schema,
+            catalog_type=self.catalog_type,
+            catalog_config=self.catalog_config,
             console=console,
-            transient=False,
-        ) as progress:
-            task = progress.add_task("Pass 1: Extracting schema", total=total)
-            for i, file_path in enumerate(file_paths, start=1):
-                console.print(f"Extracting schema {i}/{total}: {file_path.name}")
-                file_dialect = dialect or self.dialect
-                try:
-                    sql_content = read_sql_file(file_path)
-                    if self.sql_preprocessor:
-                        sql_content = self.sql_preprocessor(sql_content, file_path)
-                    analyzer = LineageAnalyzer(
-                        sql_content,
-                        dialect=file_dialect,
-                        schema=schema,
-                        strict_schema=self.strict_schema,
-                    )
-                    file_schema = analyzer.extract_schema_only()
-                    schema.update(file_schema)
-                except SchemaResolutionError:
-                    raise
-                except Exception:
-                    # Schema extraction failures are non-fatal; the file
-                    # will be reported during the lineage pass if it also fails.
-                    pass
-                progress.advance(task)
-        return schema
-
-    def _fill_schema_from_catalog(
-        self,
-        schema: Dict[str, Dict[str, str]],
-        file_paths: List[Path],
-        dialect: Optional[str] = None,
-    ) -> Dict[str, Dict[str, str]]:
-        """Pull DDL from catalog for tables not yet in schema.
-
-        Extracts all table names referenced across the files, identifies
-        those missing from the schema, and fetches their DDL from the
-        configured catalog provider.
-
-        Args:
-            schema: Schema dict already populated from file extraction
-            file_paths: SQL files to scan for table references
-            dialect: SQL dialect override
-
-        Returns:
-            Updated schema dict with catalog-sourced entries added
-        """
-        from sqlglider.catalog import get_catalog
-
-        catalog = get_catalog(self.catalog_type)  # type: ignore[arg-type]
-        if self.catalog_config:
-            catalog.configure(self.catalog_config)
-
-        # Collect all referenced table names across files
-        all_tables: Set[str] = set()
-        for file_path in file_paths:
-            file_dialect = dialect or self.dialect
-            try:
-                sql_content = read_sql_file(file_path)
-                if self.sql_preprocessor:
-                    sql_content = self.sql_preprocessor(sql_content, file_path)
-                analyzer = LineageAnalyzer(sql_content, dialect=file_dialect)
-                tables_results = analyzer.analyze_tables()
-                for result in tables_results:
-                    for table_info in result.tables:
-                        # Skip CTEs — they don't exist in catalogs
-                        from sqlglider.lineage.analyzer import ObjectType
-
-                        if table_info.object_type != ObjectType.CTE:
-                            all_tables.add(table_info.name)
-            except Exception:
-                pass
-
-        # Find tables missing from schema
-        missing = [t for t in all_tables if t not in schema]
-        if not missing:
-            return schema
-
-        console.print(
-            f"[blue]Pulling DDL from {self.catalog_type} "
-            f"for {len(missing)} table(s)...[/blue]"
         )
-
-        ddl_results = catalog.get_ddl_batch(missing)
-        file_dialect = dialect or self.dialect
-        for table_name, ddl in ddl_results.items():
-            if ddl.startswith("ERROR:"):
-                console.print(
-                    f"[yellow]Warning:[/yellow] Could not pull DDL "
-                    f"for {table_name}: {ddl}"
-                )
-                continue
-            parsed_schema = parse_ddl_to_schema(ddl, dialect=file_dialect)
-            for name, cols in parsed_schema.items():
-                if name not in schema:
-                    schema[name] = cols
-
-        return schema
+        return self._resolved_schema.copy()
 
     def _ensure_node(
         self,

sqlglider/lineage/analyzer.py

@@ -859,18 +859,31 @@ class LineageAnalyzer:
         else:
             current_query_sql = self.expr.sql(dialect=self.dialect)
 
+        # Prune schema to only tables referenced in this query to avoid
+        # sqlglot.lineage() performance degradation with large schema dicts
+        pruned_schema: Optional[Dict[str, Dict[str, str]]] = None
+        if self._file_schema:
+            referenced = {t.lower() for t in self._get_query_tables()}
+            pruned_schema = {
+                table: cols
+                for table, cols in self._file_schema.items()
+                if table.lower() in referenced
+            }
+            if not pruned_schema:
+                pruned_schema = None
+
         for col in columns_to_analyze:
             try:
                 # Get the column name that lineage expects
                 lineage_col = self._column_mapping.get(col, col)
 
                 # Get lineage tree for this column using current query SQL only
-                # Pass file schema to enable SELECT * expansion for known tables/views
+                # Pass pruned schema to enable SELECT * expansion for known tables/views
                 node = lineage(
                     lineage_col,
                     current_query_sql,
                     dialect=self.dialect,
-                    schema=self._file_schema if self._file_schema else None,
+                    schema=pruned_schema,
                 )
 
                 # Collect all source columns

sqlglider/schema/extractor.py

@@ -0,0 +1,202 @@
+"""Shared schema extraction logic for inferring table schemas from SQL files."""
+
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+from rich.console import Console
+from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
+
+from sqlglider.lineage.analyzer import LineageAnalyzer, SchemaResolutionError
+from sqlglider.utils.file_utils import read_sql_file
+from sqlglider.utils.schema import parse_ddl_to_schema
+
+SchemaDict = Dict[str, Dict[str, str]]
+SqlPreprocessor = Callable[[str, Path], str]
+
+
+def extract_schemas_from_files(
+    file_paths: List[Path],
+    dialect: str = "spark",
+    sql_preprocessor: Optional[SqlPreprocessor] = None,
+    initial_schema: Optional[SchemaDict] = None,
+    strict_schema: bool = False,
+    console: Optional[Console] = None,
+) -> SchemaDict:
+    """Extract schema from SQL files by parsing DDL and inferring from DQL.
+
+    Iterates through files, accumulating schema knowledge. Each file's
+    inferred schema is available when parsing subsequent files.
+
+    Args:
+        file_paths: SQL files to extract schema from.
+        dialect: SQL dialect.
+        sql_preprocessor: Optional function to preprocess SQL (e.g., templating).
+        initial_schema: Optional starting schema to build upon.
+        strict_schema: If True, fail on ambiguous column attribution.
+        console: Rich console for output. Uses stderr if not provided.
+
+    Returns:
+        Accumulated schema dict mapping table names to column dicts.
+    """
+    if console is None:
+        console = Console(stderr=True)
+
+    schema: SchemaDict = dict(initial_schema) if initial_schema else {}
+    total = len(file_paths)
+
+    with Progress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        console=console,
+        transient=False,
+    ) as progress:
+        task = progress.add_task("Extracting schema", total=total)
+        for i, file_path in enumerate(file_paths, start=1):
+            console.print(f"Extracting schema {i}/{total}: {file_path.name}")
+            try:
+                sql_content = read_sql_file(file_path)
+                if sql_preprocessor:
+                    sql_content = sql_preprocessor(sql_content, file_path)
+                analyzer = LineageAnalyzer(
+                    sql_content,
+                    dialect=dialect,
+                    schema=schema,
+                    strict_schema=strict_schema,
+                )
+                file_schema = analyzer.extract_schema_only()
+                schema.update(file_schema)
+            except SchemaResolutionError:
+                raise
+            except Exception:
+                # Schema extraction failures are non-fatal; the file
+                # will be reported during the lineage pass if it also fails.
+                pass
+            progress.advance(task)
+    return schema
+
+
+def fill_schema_from_catalog(
+    schema: SchemaDict,
+    file_paths: List[Path],
+    dialect: str = "spark",
+    sql_preprocessor: Optional[SqlPreprocessor] = None,
+    catalog_type: str = "databricks",
+    catalog_config: Optional[Dict[str, object]] = None,
+    console: Optional[Console] = None,
+) -> SchemaDict:
+    """Pull DDL from catalog for tables not yet in schema.
+
+    Args:
+        schema: Schema dict already populated from file extraction.
+        file_paths: SQL files to scan for table references.
+        dialect: SQL dialect.
+        sql_preprocessor: Optional SQL preprocessor.
+        catalog_type: Catalog provider name.
+        catalog_config: Provider-specific configuration dict.
+        console: Rich console for output.
+
+    Returns:
+        Updated schema dict with catalog-sourced entries added.
+    """
+    from sqlglider.catalog import get_catalog
+    from sqlglider.lineage.analyzer import ObjectType
+
+    if console is None:
+        console = Console(stderr=True)
+
+    catalog = get_catalog(catalog_type)
+    if catalog_config:
+        catalog.configure(catalog_config)
+
+    # Collect all referenced table names across files
+    all_tables: set[str] = set()
+    for file_path in file_paths:
+        try:
+            sql_content = read_sql_file(file_path)
+            if sql_preprocessor:
+                sql_content = sql_preprocessor(sql_content, file_path)
+            analyzer = LineageAnalyzer(sql_content, dialect=dialect)
+            tables_results = analyzer.analyze_tables()
+            for result in tables_results:
+                for table_info in result.tables:
+                    if table_info.object_type != ObjectType.CTE:
+                        all_tables.add(table_info.name)
+        except Exception:
+            pass
+
+    # Find tables missing from schema
+    missing = [t for t in all_tables if t not in schema]
+    if not missing:
+        return schema
+
+    console.print(
+        f"[blue]Pulling DDL from {catalog_type} for {len(missing)} table(s)...[/blue]"
+    )
+
+    ddl_results = catalog.get_ddl_batch(missing)
+    for table_name, ddl in ddl_results.items():
+        if ddl.startswith("ERROR:"):
+            console.print(
+                f"[yellow]Warning:[/yellow] Could not pull DDL for {table_name}: {ddl}"
+            )
+            continue
+        parsed_schema = parse_ddl_to_schema(ddl, dialect=dialect)
+        for name, cols in parsed_schema.items():
+            if name not in schema:
+                schema[name] = cols
+
+    return schema
+
+
+def extract_and_resolve_schema(
+    file_paths: List[Path],
+    dialect: str = "spark",
+    sql_preprocessor: Optional[SqlPreprocessor] = None,
+    strict_schema: bool = False,
+    catalog_type: Optional[str] = None,
+    catalog_config: Optional[Dict[str, object]] = None,
+    console: Optional[Console] = None,
+) -> SchemaDict:
+    """Extract schema from files and optionally fill from catalog.
+
+    High-level orchestrator that runs file-based extraction followed
+    by optional catalog resolution.
+
+    Args:
+        file_paths: SQL files to extract schema from.
+        dialect: SQL dialect.
+        sql_preprocessor: Optional SQL preprocessor.
+        strict_schema: If True, fail on ambiguous column attribution.
+        catalog_type: Optional catalog provider name.
+        catalog_config: Optional provider-specific configuration dict.
+        console: Rich console for output.
+
+    Returns:
+        Resolved schema dict.
+    """
+    if console is None:
+        console = Console(stderr=True)
+
+    console.print("[blue]Extracting schema from files[/blue]")
+    schema = extract_schemas_from_files(
+        file_paths,
+        dialect=dialect,
+        sql_preprocessor=sql_preprocessor,
+        strict_schema=strict_schema,
+        console=console,
+    )
+
+    if catalog_type:
+        schema = fill_schema_from_catalog(
+            schema,
+            file_paths,
+            dialect=dialect,
+            sql_preprocessor=sql_preprocessor,
+            catalog_type=catalog_type,
+            catalog_config=catalog_config,
+            console=console,
+        )
+
+    console.print(f"[blue]Schema resolved for {len(schema)} table(s)[/blue]")
+    return schema