sql-glider 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

sqlglider/dissection/formatters.py

@@ -0,0 +1,222 @@
+"""Output formatters for dissection results."""
+
+import csv
+import json
+from io import StringIO
+from pathlib import Path
+from typing import List, Optional
+
+from rich.console import Console
+from rich.table import Table
+
+from sqlglider.dissection.models import QueryDissectionResult
+
+
+class DissectionTextFormatter:
+    """Format dissection results as Rich tables for terminal display."""
+
+    @staticmethod
+    def format(results: List[QueryDissectionResult], console: Console) -> None:
+        """
+        Format and print dissection results as Rich tables.
+
+        Creates a styled table for each query showing all extracted components.
+
+        Args:
+            results: List of QueryDissectionResult objects
+            console: Rich Console instance for output
+        """
+        if not results:
+            console.print("[yellow]No dissection results found.[/yellow]")
+            return
+
+        for i, result in enumerate(results):
+            # Add spacing between queries (except for first)
+            if i > 0:
+                console.print()
+
+            # Create table with query info as title
+            title = (
+                f"Query {result.metadata.query_index} "
+                f"({result.metadata.statement_type}): "
+                f"{result.metadata.query_preview}"
+            )
+            table = Table(title=title, title_style="bold")
+
+            table.add_column("Index", style="dim", width=6)
+            table.add_column("Type", style="cyan", width=16)
+            table.add_column("Name", style="green", min_width=10)
+            table.add_column("Depth", style="yellow", width=6)
+            table.add_column("Exec?", style="magenta", width=6)
+            table.add_column("Location", style="blue", min_width=15)
+            table.add_column("SQL Preview", style="dim", min_width=30)
+
+            # Add rows for each component
+            for component in result.components:
+                # Truncate SQL for preview
+                sql_preview = " ".join(component.sql.split())[:50]
+                if len(component.sql) > 50:
+                    sql_preview += "..."
+
+                table.add_row(
+                    str(component.component_index),
+                    component.component_type.value,
+                    component.name or "-",
+                    str(component.depth),
+                    "Yes" if component.is_executable else "No",
+                    component.location[:35] + "..."
+                    if len(component.location) > 35
+                    else component.location,
+                    sql_preview,
+                )
+
+            console.print(table)
+            console.print(
+                f"[dim]Total components: {result.metadata.total_components}[/dim]"
+            )
+
+
+class DissectionJsonFormatter:
+    """Format dissection results as JSON."""
+
+    @staticmethod
+    def format(results: List[QueryDissectionResult]) -> str:
+        """
+        Format dissection results as JSON.
+
+        Output format:
+        {
+          "queries": [
+            {
+              "query_index": 0,
+              "query_preview": "SELECT ...",
+              "statement_type": "INSERT",
+              "total_components": 5,
+              "components": [
+                {
+                  "component_type": "CTE",
+                  "component_index": 0,
+                  "name": "order_totals",
+                  "sql": "SELECT ...",
+                  "parent_index": null,
+                  "depth": 0,
+                  "is_executable": true,
+                  "dependencies": [],
+                  "location": "WITH clause"
+                }
+              ],
+              "original_sql": "WITH order_totals AS ..."
+            }
+          ]
+        }
+
+        Args:
+            results: List of QueryDissectionResult objects
+
+        Returns:
+            JSON-formatted string
+        """
+        queries = []
+        for result in results:
+            query_data = {
+                "query_index": result.metadata.query_index,
+                "query_preview": result.metadata.query_preview,
+                "statement_type": result.metadata.statement_type,
+                "total_components": result.metadata.total_components,
+                "components": [
+                    {
+                        "component_type": component.component_type.value,
+                        "component_index": component.component_index,
+                        "name": component.name,
+                        "sql": component.sql,
+                        "parent_index": component.parent_index,
+                        "depth": component.depth,
+                        "is_executable": component.is_executable,
+                        "dependencies": component.dependencies,
+                        "location": component.location,
+                    }
+                    for component in result.components
+                ],
+                "original_sql": result.original_sql,
+            }
+            queries.append(query_data)
+
+        return json.dumps({"queries": queries}, indent=2)
+
+
+class DissectionCsvFormatter:
+    """Format dissection results as CSV."""
+
+    @staticmethod
+    def format(results: List[QueryDissectionResult]) -> str:
+        """
+        Format dissection results as CSV.
+
+        Output format:
+        query_index,component_index,component_type,name,depth,is_executable,location,dependencies,sql
+        0,0,CTE,order_totals,0,true,WITH clause,,"SELECT ..."
+
+        Args:
+            results: List of QueryDissectionResult objects
+
+        Returns:
+            CSV-formatted string
+        """
+        if not results:
+            return ""
+
+        output = StringIO()
+        headers = [
+            "query_index",
+            "component_index",
+            "component_type",
+            "name",
+            "depth",
+            "is_executable",
+            "location",
+            "dependencies",
+            "sql",
+        ]
+
+        writer = csv.writer(output)
+        writer.writerow(headers)
+
+        # Write data rows
+        for result in results:
+            query_index = result.metadata.query_index
+            for component in result.components:
+                # Join dependencies with semicolon
+                deps_str = ";".join(component.dependencies)
+                writer.writerow(
+                    [
+                        query_index,
+                        component.component_index,
+                        component.component_type.value,
+                        component.name or "",
+                        component.depth,
+                        "true" if component.is_executable else "false",
+                        component.location,
+                        deps_str,
+                        component.sql,
+                    ]
+                )
+
+        return output.getvalue()
+
+
+class OutputWriter:
+    """Write formatted output to file or stdout."""
+
+    @staticmethod
+    def write(content: str, output_file: Optional[Path] = None) -> None:
+        """
+        Write content to file or stdout.
+
+        Args:
+            content: The content to write
+            output_file: Optional file path. If None, writes to stdout.
+        """
+        if output_file:
+            output_file.write_text(content, encoding="utf-8")
+        else:
+            print(content)

sqlglider/dissection/models.py

@@ -0,0 +1,112 @@
+"""Pydantic models for SQL dissection results."""
+
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ComponentType(str, Enum):
+    """Type of SQL component extracted from a query."""
+
+    CTE = "CTE"
+    MAIN_QUERY = "MAIN_QUERY"
+    SUBQUERY = "SUBQUERY"
+    SCALAR_SUBQUERY = "SCALAR_SUBQUERY"
+    TARGET_TABLE = "TARGET_TABLE"
+    SOURCE_QUERY = "SOURCE_QUERY"
+    UNION_BRANCH = "UNION_BRANCH"
+
+
+class SQLComponent(BaseModel):
+    """Represents a single SQL component extracted from a query."""
+
+    component_type: ComponentType = Field(
+        ..., description="Type of component (CTE, SUBQUERY, etc.)"
+    )
+    component_index: int = Field(
+        ..., description="0-based index within query (order of extraction)"
+    )
+    name: Optional[str] = Field(
+        None,
+        description="Name/alias of component (CTE name, subquery alias, target table)",
+    )
+    sql: str = Field(
+        ..., description="Extracted SQL for this component (executable if applicable)"
+    )
+    parent_index: Optional[int] = Field(
+        None, description="Index of parent component (for nested subqueries)"
+    )
+    depth: int = Field(
+        default=0, description="Nesting depth (0 = top-level, 1+ = nested)"
+    )
+    is_executable: bool = Field(
+        default=True, description="Whether this SQL can be executed standalone"
+    )
+    dependencies: List[str] = Field(
+        default_factory=list,
+        description="Names of CTEs this component depends on",
+    )
+    location: str = Field(
+        ..., description="Human-readable location context (e.g., 'WITH clause')"
+    )
+
+
+class QueryMetadata(BaseModel):
+    """Metadata about a dissected query."""
+
+    query_index: int = Field(..., description="0-based query index in multi-query file")
+    query_preview: str = Field(..., description="First 100 chars of original query")
+    statement_type: str = Field(
+        ..., description="Type of SQL statement (SELECT, INSERT, CREATE, etc.)"
+    )
+    total_components: int = Field(
+        ..., description="Total number of components extracted"
+    )
+
+
+class QueryDissectionResult(BaseModel):
+    """Complete dissection result for a single query."""
+
+    metadata: QueryMetadata
+    components: List[SQLComponent] = Field(
+        default_factory=list,
+        description="All extracted components in order",
+    )
+    original_sql: str = Field(
+        ..., description="Original SQL query for reference/validation"
+    )
+
+    def get_component_by_name(self, name: str) -> Optional[SQLComponent]:
+        """Find a component by name (case-insensitive).
+
+        Args:
+            name: The component name to search for.
+
+        Returns:
+            The matching SQLComponent or None if not found.
+        """
+        name_lower = name.lower()
+        for component in self.components:
+            if component.name and component.name.lower() == name_lower:
+                return component
+        return None
+
+    def get_components_by_type(self, comp_type: ComponentType) -> List[SQLComponent]:
+        """Get all components of a specific type.
+
+        Args:
+            comp_type: The ComponentType to filter by.
+
+        Returns:
+            List of matching SQLComponent objects.
+        """
+        return [c for c in self.components if c.component_type == comp_type]
+
+    def get_executable_components(self) -> List[SQLComponent]:
+        """Get all executable components.
+
+        Returns:
+            List of SQLComponent objects that can be executed standalone.
+        """
+        return [c for c in self.components if c.is_executable]

sqlglider/graph/builder.py

@@ -6,6 +6,7 @@ from typing import Callable, Dict, List, Optional, Set
 
 import rustworkx as rx
 from rich.console import Console
+from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
 
 from sqlglider.global_models import AnalysisLevel, NodeFormat
 from sqlglider.graph.models import (
@@ -166,11 +167,8 @@ class GraphBuilder:
         else:
             pattern = glob_pattern
 
-        for sql_file in sorted(dir_path.glob(pattern)):
-            if sql_file.is_file():
-                self.add_file(sql_file, dialect)
-
-        return self
+        sql_files = [f for f in sorted(dir_path.glob(pattern)) if f.is_file()]
+        return self.add_files(sql_files, dialect)
 
     def add_manifest(
         self,
@@ -194,6 +192,8 @@ class GraphBuilder:
         manifest = Manifest.from_csv(manifest_path)
         base_dir = manifest_path.parent
 
+        # Collect files with their dialects
+        files_with_dialects: List[tuple[Path, str]] = []
         for entry in manifest.entries:
             # Resolve file path relative to manifest location
             file_path = Path(entry.file_path)
@@ -202,7 +202,25 @@ class GraphBuilder:
 
             # Use entry dialect, then CLI dialect, then builder default
             entry_dialect = entry.dialect or dialect or self.dialect
-            self.add_file(file_path, entry_dialect)
+            files_with_dialects.append((file_path, entry_dialect))
+
+        # Process with progress
+        if files_with_dialects:
+            total = len(files_with_dialects)
+            with Progress(
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                TaskProgressColumn(),
+                console=console,
+                transient=False,
+            ) as progress:
+                task = progress.add_task("Parsing", total=total)
+                for i, (file_path, file_dialect) in enumerate(
+                    files_with_dialects, start=1
+                ):
+                    console.print(f"Parsing file {i}/{total}: {file_path.name}")
+                    self.add_file(file_path, file_dialect)
+                    progress.advance(task)
 
         return self
 
@@ -210,6 +228,7 @@ class GraphBuilder:
         self,
         file_paths: List[Path],
         dialect: Optional[str] = None,
+        show_progress: bool = True,
     ) -> "GraphBuilder":
         """
         Add lineage from multiple SQL files.
@@ -217,12 +236,31 @@ class GraphBuilder:
         Args:
             file_paths: List of paths to SQL files
             dialect: SQL dialect (uses builder default if not specified)
+            show_progress: Whether to print progress messages
 
         Returns:
             self for method chaining
         """
-        for file_path in file_paths:
-            self.add_file(file_path, dialect)
+        if not file_paths:
+            return self
+
+        if show_progress:
+            total = len(file_paths)
+            with Progress(
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                TaskProgressColumn(),
+                console=console,
+                transient=False,
+            ) as progress:
+                task = progress.add_task("Parsing", total=total)
+                for i, file_path in enumerate(file_paths, start=1):
+                    console.print(f"Parsing file {i}/{total}: {file_path.name}")
+                    self.add_file(file_path, dialect)
+                    progress.advance(task)
+        else:
+            for file_path in file_paths:
+                self.add_file(file_path, dialect)
         return self
 
     def _ensure_node(

sqlglider/lineage/analyzer.py

@@ -1,7 +1,7 @@
 """Core lineage analysis using SQLGlot."""
 
 from enum import Enum
-from typing import Callable, Iterator, List, Optional, Set, Tuple
+from typing import Callable, Iterator, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel, Field
 from sqlglot import exp, parse
@@ -155,7 +155,8 @@ class LineageAnalyzer:
         if target_table:
             # DML/DDL: Use target table for output column qualification
             # The columns are from the SELECT, but qualified with the target table
-            for projection in select_node.expressions:
+            projections = self._get_select_projections(select_node)
+            for projection in projections:
                 # Get the underlying expression (unwrap alias if present)
                 if isinstance(projection, exp.Alias):
                     # For aliased columns, use the alias as the column name
@@ -178,7 +179,10 @@ class LineageAnalyzer:
 
         else:
             # DQL (pure SELECT): Use the SELECT columns as output
-            for projection in select_node.expressions:
+            projections = self._get_select_projections(select_node)
+            # Get the first SELECT for table resolution (handles UNION case)
+            first_select = self._get_first_select(select_node)
+            for projection in projections:
                 # Get the underlying expression (unwrap alias if present)
                 if isinstance(projection, exp.Alias):
                     source_expr = projection.this
@@ -195,20 +199,20 @@ class LineageAnalyzer:
                     table_name = source_expr.table
                     col_name = column_name or source_expr.name
 
-                    if table_name:
+                    if table_name and first_select:
                         # Resolve table reference (could be table, CTE, or subquery alias)
                         # This works at any nesting level because we're only looking at the immediate context
                         resolved_table = self._resolve_table_reference(
-                            table_name, select_node
+                            table_name, first_select
                         )
                         qualified_name = f"{resolved_table}.{col_name}"
                         columns.append(qualified_name)
                         # Map qualified name to what lineage expects
                         self._column_mapping[qualified_name] = lineage_name or col_name
-                    else:
+                    elif first_select:
                         # No table qualifier - try to infer from FROM clause
                         # This handles "SELECT col FROM single_source" cases
-                        inferred_table = self._infer_single_table_source(select_node)
+                        inferred_table = self._infer_single_table_source(first_select)
                         if inferred_table:
                             qualified_name = f"{inferred_table}.{col_name}"
                             columns.append(qualified_name)
@@ -219,6 +223,10 @@ class LineageAnalyzer:
                             # Can't infer table, just use column name
                             columns.append(col_name)
                             self._column_mapping[col_name] = lineage_name or col_name
+                    else:
+                        # No SELECT found, just use column name
+                        columns.append(col_name)
+                        self._column_mapping[col_name] = lineage_name or col_name
                 else:
                     # For other expressions (literals, functions, etc.)
                     # Use the alias if available, otherwise the SQL representation
@@ -232,6 +240,46 @@ class LineageAnalyzer:
 
         return columns
 
+    def _get_select_projections(self, node: exp.Expression) -> List[exp.Expression]:
+        """
+        Get the SELECT projections from a SELECT or set operation node.
+
+        For set operations (UNION, INTERSECT, EXCEPT), returns projections from
+        the first branch since all branches must have the same number of columns
+        with compatible types.
+
+        Args:
+            node: A SELECT or set operation (UNION/INTERSECT/EXCEPT) expression
+
+        Returns:
+            List of projection expressions from the SELECT clause
+        """
+        if isinstance(node, exp.Select):
+            return list(node.expressions)
+        elif isinstance(node, (exp.Union, exp.Intersect, exp.Except)):
+            # Recursively get from the left branch (could be nested set operations)
+            return self._get_select_projections(node.left)
+        return []
+
+    def _get_first_select(self, node: exp.Expression) -> Optional[exp.Select]:
+        """
+        Get the first SELECT node from a SELECT or set operation expression.
+
+        For set operations (UNION, INTERSECT, EXCEPT), returns the leftmost
+        SELECT branch.
+
+        Args:
+            node: A SELECT or set operation (UNION/INTERSECT/EXCEPT) expression
+
+        Returns:
+            The first SELECT node, or None if not found
+        """
+        if isinstance(node, exp.Select):
+            return node
+        elif isinstance(node, (exp.Union, exp.Intersect, exp.Except)):
+            return self._get_first_select(node.left)
+        return None
+
     def analyze_queries(
         self,
         level: AnalysisLevel = AnalysisLevel.COLUMN,
@@ -795,7 +843,9 @@ class LineageAnalyzer:
 
     def _get_target_and_select(
         self,
-    ) -> Optional[tuple[Optional[str], exp.Select]]:
+    ) -> Optional[
+        tuple[Optional[str], Union[exp.Select, exp.Union, exp.Intersect, exp.Except]]
+    ]:
         """
         Detect if this is a DML/DDL statement and extract the target table and SELECT node.
 
@@ -817,9 +867,11 @@ class LineageAnalyzer:
             target = self.expr.this
             if isinstance(target, exp.Table):
                 target_name = self._get_qualified_table_name(target)
-                # Find the SELECT within the INSERT
+                # Find the SELECT within the INSERT (may be a set operation)
                 select_node = self.expr.expression
-                if isinstance(select_node, exp.Select):
+                if isinstance(
+                    select_node, (exp.Select, exp.Union, exp.Intersect, exp.Except)
+                ):
                     return (target_name, select_node)
 
         # Check for CREATE TABLE AS SELECT (CTAS) or CREATE VIEW AS SELECT
@@ -831,9 +883,11 @@ class LineageAnalyzer:
                     target = target.this
                 if isinstance(target, exp.Table):
                     target_name = self._get_qualified_table_name(target)
-                    # Find the SELECT in the expression
+                    # Find the SELECT in the expression (may be a set operation)
                     select_node = self.expr.expression
-                    if isinstance(select_node, exp.Select):
+                    if isinstance(
+                        select_node, (exp.Select, exp.Union, exp.Intersect, exp.Except)
+                    ):
                         return (target_name, select_node)
 
         # Check for MERGE statement

sqlglider/utils/config.py

@@ -23,6 +23,28 @@ class TemplatingConfig(BaseModel):
     variables: Optional[Dict[str, Any]] = None
 
 
+class DatabricksCatalogConfig(BaseModel):
+    """Configuration for Databricks catalog provider.
+
+    All fields are optional - they can also be set via environment variables.
+    The SDK supports unified authentication with multiple methods.
+    """
+
+    warehouse_id: Optional[str] = None
+    profile: Optional[str] = None  # Databricks CLI profile from ~/.databrickscfg
+    host: Optional[str] = None
+    token: Optional[str] = None  # Legacy PAT, prefer OAuth or profile
+
+
+class CatalogConfig(BaseModel):
+    """Configuration for catalog providers.
+
+    Contains provider-specific configuration under sub-keys.
+    """
+
+    databricks: Optional[DatabricksCatalogConfig] = None
+
+
 class ConfigSettings(BaseModel):
     """Configuration settings for SQL Glider.
 
@@ -35,6 +57,9 @@ class ConfigSettings(BaseModel):
     output_format: Optional[str] = None
     templater: Optional[str] = None
     templating: Optional[TemplatingConfig] = None
+    catalog_type: Optional[str] = None
+    ddl_folder: Optional[str] = None
+    catalog: Optional[CatalogConfig] = None
 
 
 def find_config_file(start_path: Optional[Path] = None) -> Optional[Path]:

sql_glider-0.1.2.dist-info/RECORD

@@ -1,26 +0,0 @@
-sqlglider/__init__.py,sha256=gDf7s52dMcX7JuCZ1SLawcB1vb3U0yJCohu9RQAATBY,125
-sqlglider/_version.py,sha256=Ok5oAXdWgR9aghaFXTafTeDW6sYO3uVe6d2Nket57R4,704
-sqlglider/cli.py,sha256=POWIhv0jfvoNtwSoURpxJydco1rvxX9rAvyjuA9FGC8,36445
-sqlglider/global_models.py,sha256=2vyJXAuXOsXQpE-D3F0ejj7eR9z0nDWFjTkielhzM8k,356
-sqlglider/graph/__init__.py,sha256=4DDdrPM75CmeQWt7wHdBsjCm1s70BHGLYdijIbaUEKY,871
-sqlglider/graph/builder.py,sha256=rrcpGAXLz-VHZ1Y73uw6R7kMXHpzBz7tQ2tdV5BY05w,10202
-sqlglider/graph/merge.py,sha256=uUZlm4BN3S9gRL66Cc2mzhbtuh4SVAv2n4cN4eUEQBU,4077
-sqlglider/graph/models.py,sha256=EYmjv_WzDSNp_WfhJ6H-qBIOkAcoNKS7GRUryfKrHuY,9330
-sqlglider/graph/query.py,sha256=LHU8Cvn7ZPPSEnqdDn2pF8f1_LQjIvNIrZqs8cFlb6U,9433
-sqlglider/graph/serialization.py,sha256=7JJo31rwSlxnDhdqdTJdK4Dr_ZcSYetXfx3_CmndSac,2662
-sqlglider/lineage/__init__.py,sha256=llXMeI5_PIZaiBo8tKk3-wOubF4m_6QBHbn1FtWxT7k,256
-sqlglider/lineage/analyzer.py,sha256=58lyrUc0XsCUrYSb23A02OSBmq7eCtJwc477PbjS3c0,45905
-sqlglider/lineage/formatters.py,sha256=_Y9wcTX4JXn1vVnZ1xI656g1FF2rMjcAVc-GHjbd9QA,10389
-sqlglider/templating/__init__.py,sha256=g3_wb6rSDI0usq2UUMDpn-J5kVwlAw3NtLdwbxL6UHs,1435
-sqlglider/templating/base.py,sha256=y5bWAW7qXl_4pPyo5KycfHwNVvt1-7slZ63DAsvTE1s,2902
-sqlglider/templating/jinja.py,sha256=o01UG72N4G1-tOT5LKK1Wkccv4nJH2VN4VFaMi5c1-g,5220
-sqlglider/templating/registry.py,sha256=BJU3N2qNVMTUtkgbibyqo8Wme_acXQRw5XI-6ZVgyac,3476
-sqlglider/templating/variables.py,sha256=5593PtLBcOxsnMCSRm2pGAD5I0Y9f__VV3_J_HfXVlQ,8010
-sqlglider/utils/__init__.py,sha256=KGp9-UzKz_OFBOTFoSy-g-NXDZsvyWXG_9-1zcC6ePE,276
-sqlglider/utils/config.py,sha256=mkven_CcE_dNfKiHi0h2CsE5TMQDX9XqbU7GGEELwEY,3959
-sqlglider/utils/file_utils.py,sha256=5_ff28E0r1R7emZzsOnRuHd-7zIX6873eyr1SuPEr4E,1093
-sql_glider-0.1.2.dist-info/METADATA,sha256=JUXRDvhfnJBj2owWMaupDugZj4Y6uDv1R7RCCkaEWlw,22349
-sql_glider-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-sql_glider-0.1.2.dist-info/entry_points.txt,sha256=LWVdQEfvDT5uZ2RQ4Rse8m0HxBCOMbbqDkxdwUh9d78,169
-sql_glider-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sql_glider-0.1.2.dist-info/RECORD,,