ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/unused_deps.py

@@ -0,0 +1,555 @@
+"""Unused dependency detection for SQL optimization.
+
+This module analyzes SQL to find:
+- Tables that are joined but no columns are actually used from them
+- CTEs that are defined but never referenced
+- Subqueries that don't contribute to the output
+
+This helps with:
+1. Fewer upstream dependencies to maintain
+2. Tech debt cleanup - Identify dead code in SQL
+3. Performance - Unnecessary joins waste compute
+"""
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+import sqlglot
+from sqlglot import exp
+
+from ripple.engine.safe_gen import safe_sql
+
+
+@dataclass
+class TableUsage:
+    """Track how a table is used in a query."""
+
+    name: str
+    alias: str | None
+    schema_name: str | None = None
+    database: str | None = None
+
+    # Usage tracking
+    columns_selected: list[str] = field(default_factory=list)
+    columns_in_where: list[str] = field(default_factory=list)
+    columns_in_group_by: list[str] = field(default_factory=list)
+    columns_in_order_by: list[str] = field(default_factory=list)
+    columns_in_having: list[str] = field(default_factory=list)
+    columns_in_join_condition: list[str] = field(default_factory=list)
+
+    # Join info
+    join_type: str | None = None  # LEFT, RIGHT, INNER, CROSS, etc.
+    is_from_table: bool = False  # True if this is the main FROM table
+
+    @property
+    def identifier(self) -> str:
+        """Return the identifier used in the query (alias if present, else name)."""
+        return self.alias or self.name
+
+    @property
+    def full_name(self) -> str:
+        """Return fully qualified name."""
+        parts = []
+        if self.database:
+            parts.append(self.database)
+        if self.schema_name:
+            parts.append(self.schema_name)
+        parts.append(self.name)
+        return ".".join(parts)
+
+    @property
+    def total_column_uses(self) -> int:
+        """Total number of column references from this table."""
+        return (
+            len(self.columns_selected)
+            + len(self.columns_in_where)
+            + len(self.columns_in_group_by)
+            + len(self.columns_in_order_by)
+            + len(self.columns_in_having)
+        )
+
+    @property
+    def is_used(self) -> bool:
+        """Check if this table contributes to the query output."""
+        # A table is "used" if:
+        # 1. It's the main FROM table (always needed)
+        # 2. Columns from it are selected
+        # 3. Columns from it are used in WHERE/GROUP BY/HAVING/ORDER BY
+        if self.is_from_table:
+            return True
+        return self.total_column_uses > 0
+
+    @property
+    def only_used_for_join(self) -> bool:
+        """Check if table is only used in join condition, not in output."""
+        return (
+            not self.is_from_table
+            and self.total_column_uses == 0
+            and len(self.columns_in_join_condition) > 0
+        )
+
+
+@dataclass
+class CTEUsage:
+    """Track CTE definition and usage."""
+
+    name: str
+    sql: str
+    referenced_count: int = 0
+    referenced_in: list[str] = field(default_factory=list)  # Where it's used
+
+    @property
+    def is_used(self) -> bool:
+        return self.referenced_count > 0
+
+
+@dataclass
+class UnusedDependency:
+    """A detected unused dependency."""
+
+    type: Literal["join", "cte", "subquery"]
+    name: str
+    location: str  # Human-readable location in query
+    reason: str  # Why it's considered unused
+    recommendation: str  # What to do about it
+    impact: str  # Potential impact of removal
+    confidence: float = 1.0  # How confident we are (0-1)
+    columns_in_join_only: list[str] = field(default_factory=list)
+
+
+@dataclass
+class UnusedDepsAnalysisResult:
+    """Result of unused dependency analysis."""
+
+    unused_joins: list[UnusedDependency] = field(default_factory=list)
+    unused_ctes: list[UnusedDependency] = field(default_factory=list)
+    table_usage: list[TableUsage] = field(default_factory=list)
+    cte_usage: list[CTEUsage] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def total_issues(self) -> int:
+        return len(self.unused_joins) + len(self.unused_ctes)
+
+    @property
+    def has_issues(self) -> bool:
+        return self.total_issues > 0
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "total_issues": self.total_issues,
+            "has_issues": self.has_issues,
+            "unused_joins": [
+                {
+                    "type": dep.type,
+                    "name": dep.name,
+                    "location": dep.location,
+                    "reason": dep.reason,
+                    "recommendation": dep.recommendation,
+                    "impact": dep.impact,
+                    "confidence": dep.confidence,
+                    "columns_in_join_only": dep.columns_in_join_only,
+                }
+                for dep in self.unused_joins
+            ],
+            "unused_ctes": [
+                {
+                    "type": dep.type,
+                    "name": dep.name,
+                    "location": dep.location,
+                    "reason": dep.reason,
+                    "recommendation": dep.recommendation,
+                    "impact": dep.impact,
+                    "confidence": dep.confidence,
+                }
+                for dep in self.unused_ctes
+            ],
+            "table_usage": [
+                {
+                    "name": t.name,
+                    "alias": t.alias,
+                    "full_name": t.full_name,
+                    "join_type": t.join_type,
+                    "is_from_table": t.is_from_table,
+                    "columns_selected": t.columns_selected,
+                    "columns_in_where": t.columns_in_where,
+                    "columns_in_join_condition": t.columns_in_join_condition,
+                    "total_uses": t.total_column_uses,
+                    "is_used": t.is_used,
+                    "only_used_for_join": t.only_used_for_join,
+                }
+                for t in self.table_usage
+            ],
+            "cte_usage": [
+                {
+                    "name": c.name,
+                    "referenced_count": c.referenced_count,
+                    "referenced_in": c.referenced_in,
+                    "is_used": c.is_used,
+                }
+                for c in self.cte_usage
+            ],
+            "warnings": self.warnings,
+        }
+
+
+class UnusedDependencyDetector:
+    """Detect unused dependencies in SQL queries.
+
+    This analyzer finds:
+    1. JOINed tables where no columns are actually used in the output
+    2. CTEs that are defined but never referenced
+    3. Tables only used for filtering that could be optimized
+
+    Usage:
+        detector = UnusedDependencyDetector(dialect="snowflake")
+        result = detector.analyze(sql)
+        for issue in result.unused_joins:
+            print(f"Unused join: {issue.name} - {issue.reason}")
+    """
+
+    def __init__(self, dialect: str = "snowflake"):
+        """Initialize detector.
+
+        Args:
+            dialect: SQL dialect (snowflake, bigquery, databricks, postgres, etc.)
+        """
+        self.dialect = dialect
+        self._has_unqualified_columns = False
+        self._has_select_star = False
+        self._has_using_join = False
+        self._has_natural_join = False
+        self._has_lateral_join = False
+        self._has_table_function = False
+
+    def analyze(self, sql: str) -> UnusedDepsAnalysisResult:
+        """Analyze SQL for unused dependencies.
+
+        Args:
+            sql: The SQL query to analyze
+
+        Returns:
+            UnusedDepsAnalysisResult with detected issues
+        """
+        result = UnusedDepsAnalysisResult()
+
+        try:
+            parsed = sqlglot.parse_one(sql, dialect=self.dialect)
+        except Exception as e:
+            result.warnings.append(f"Failed to parse SQL: {str(e)}")
+            return result
+
+        # Analyze CTEs
+        self._analyze_ctes(parsed, result)
+
+        # Analyze table usage (JOINs)
+        self._analyze_table_usage(parsed, result)
+
+        # Detect unused joins
+        self._detect_unused_joins(result)
+
+        # Detect unused CTEs
+        self._detect_unused_ctes(result)
+
+        if self._has_unqualified_columns:
+            result.warnings.append("unqualified_columns")
+        if self._has_select_star:
+            result.warnings.append("select_star")
+        if self._has_using_join:
+            result.warnings.append("using_join")
+        if self._has_natural_join:
+            result.warnings.append("natural_join")
+        if self._has_lateral_join:
+            result.warnings.append("lateral_join")
+        if self._has_table_function:
+            result.warnings.append("table_function")
+
+        return result
+
+    def _analyze_ctes(self, parsed: exp.Expression, result: UnusedDepsAnalysisResult) -> None:
+        """Analyze CTE definitions and usage."""
+        # Find all CTE definitions
+        cte_names: dict[str, CTEUsage] = {}
+
+        for cte in parsed.find_all(exp.CTE):
+            cte_name = cte.alias
+            cte_sql = (safe_sql(cte.this, self.dialect) or "") if cte.this else ""
+            cte_names[cte_name.lower()] = CTEUsage(
+                name=cte_name,
+                sql=cte_sql[:200] + "..." if len(cte_sql) > 200 else cte_sql,
+            )
+
+        if not cte_names:
+            return
+
+        # Find all table references
+        for table in parsed.find_all(exp.Table):
+            table_name = table.name.lower()
+
+            if table_name in cte_names:
+                cte_names[table_name].referenced_count += 1
+
+                # Try to determine where it's referenced
+                parent = table.parent
+                location = "unknown"
+                if parent:
+                    if isinstance(parent, exp.From):
+                        location = "FROM clause"
+                    elif isinstance(parent, exp.Join):
+                        location = "JOIN"
+                    elif isinstance(parent, exp.Subquery):
+                        location = "subquery"
+
+                cte_names[table_name].referenced_in.append(location)
+
+        # Also check CTEs referencing other CTEs
+        for cte in parsed.find_all(exp.CTE):
+            cte_name = cte.alias.lower()
+            for inner_table in cte.find_all(exp.Table):
+                inner_name = inner_table.name.lower()
+                if inner_name in cte_names and inner_name != cte_name:
+                    cte_names[inner_name].referenced_count += 1
+                    cte_names[inner_name].referenced_in.append(f"CTE '{cte.alias}'")
+
+        result.cte_usage = list(cte_names.values())
+
+    def _analyze_table_usage(
+        self, parsed: exp.Expression, result: UnusedDepsAnalysisResult
+    ) -> None:
+        """Analyze how each table is used in the query."""
+        tables: dict[str, TableUsage] = {}
+
+        # Find all tables in FROM and JOINs
+        for from_clause in parsed.find_all(exp.From):
+            table_expr = from_clause.this
+            if isinstance(table_expr, exp.Table):
+                usage = self._create_table_usage(table_expr, is_from=True)
+                tables[usage.identifier.lower()] = usage
+
+        for join in parsed.find_all(exp.Join):
+            table_expr = join.this
+            if isinstance(table_expr, exp.Table):
+                usage = self._create_table_usage(table_expr, is_from=False)
+                usage.join_type = join.kind or "INNER"
+                tables[usage.identifier.lower()] = usage
+                if join.args.get("using"):
+                    self._has_using_join = True
+                if join.args.get("method") == "natural":
+                    self._has_natural_join = True
+
+                # Track columns used in join condition
+                if join.args.get("on"):
+                    for col in join.args["on"].find_all(exp.Column):
+                        col_table = (col.table or "").lower()
+                        if col_table in tables:
+                            tables[col_table].columns_in_join_condition.append(col.name)
+
+            # Detect LATERAL joins and table functions (can't reliably analyze)
+            if join.args.get("side") == "LATERAL" or isinstance(table_expr, exp.Lateral):
+                self._has_lateral_join = True
+
+        # Detect table functions (FLATTEN, UNNEST, etc.)
+        for _lateral in parsed.find_all(exp.Lateral):
+            self._has_lateral_join = True
+        for _func in parsed.find_all(exp.Unnest):
+            self._has_table_function = True
+        for _func in parsed.find_all(exp.Explode):
+            self._has_table_function = True
+
+        if not tables:
+            return
+
+        # Find column usage in different clauses
+        self._track_column_usage(parsed, tables, "select", "columns_selected")
+        self._track_column_usage(parsed, tables, "where", "columns_in_where")
+        self._track_column_usage(parsed, tables, "group", "columns_in_group_by")
+        self._track_column_usage(parsed, tables, "order", "columns_in_order_by")
+        self._track_column_usage(parsed, tables, "having", "columns_in_having")
+
+        result.table_usage = list(tables.values())
+
+    def _create_table_usage(self, table_expr: exp.Table, is_from: bool) -> TableUsage:
+        """Create TableUsage from a table expression."""
+        return TableUsage(
+            name=table_expr.name,
+            alias=table_expr.alias if table_expr.alias else None,
+            schema_name=table_expr.db,
+            database=table_expr.catalog,
+            is_from_table=is_from,
+        )
+
+    def _track_column_usage(
+        self,
+        parsed: exp.Expression,
+        tables: dict[str, TableUsage],
+        clause_type: str,
+        attr_name: str,
+    ) -> None:
+        """Track column usage in a specific clause type.
+
+        IMPORTANT: We need to be precise about what we search.
+        For SELECT, we only want the actual selected expressions, not JOINs.
+        For WHERE/GROUP/ORDER/HAVING, we want those specific clauses only.
+        """
+        for select_stmt in parsed.find_all(exp.Select):
+            if clause_type == "select":
+                # Only look at the SELECT expressions (not JOINs, WHERE, etc.)
+                for select_expr in select_stmt.expressions:
+                    if isinstance(select_expr, exp.Star):
+                        self._has_select_star = True
+                        for table in tables.values():
+                            col_list = getattr(table, attr_name)
+                            if "*" not in col_list:
+                                col_list.append("*")
+                        continue
+                    if isinstance(select_expr, exp.Column) and select_expr.name == "*":
+                        table_name = (select_expr.table or "").lower()
+                        if table_name and table_name in tables:
+                            col_list = getattr(tables[table_name], attr_name)
+                            if "*" not in col_list:
+                                col_list.append("*")
+                        else:
+                            self._has_select_star = True
+                        continue
+                    self._extract_columns_from_expr(select_expr, tables, attr_name)
+            elif clause_type == "where":
+                where_clause = select_stmt.args.get("where")
+                if where_clause:
+                    self._extract_columns_from_expr(where_clause, tables, attr_name)
+            elif clause_type == "group":
+                group_clause = select_stmt.args.get("group")
+                if group_clause:
+                    self._extract_columns_from_expr(group_clause, tables, attr_name)
+            elif clause_type == "order":
+                order_clause = select_stmt.args.get("order")
+                if order_clause:
+                    self._extract_columns_from_expr(order_clause, tables, attr_name)
+            elif clause_type == "having":
+                having_clause = select_stmt.args.get("having")
+                if having_clause:
+                    self._extract_columns_from_expr(having_clause, tables, attr_name)
+
+    def _extract_columns_from_expr(
+        self,
+        expr: exp.Expression,
+        tables: dict[str, TableUsage],
+        attr_name: str,
+    ) -> None:
+        """Extract column references from an expression and add to table usage."""
+        for col in expr.find_all(exp.Column):
+            col_table = (col.table or "").lower()
+
+            # If no table qualifier, we can't reliably attribute the column
+            if not col_table:
+                self._has_unqualified_columns = True
+                continue
+
+            if col_table in tables:
+                col_list = getattr(tables[col_table], attr_name)
+                if col.name not in col_list:
+                    col_list.append(col.name)
+
+    def _detect_unused_joins(self, result: UnusedDepsAnalysisResult) -> None:
+        """Detect joins that don't contribute to the output."""
+        for table in result.table_usage:
+            if table.is_from_table:
+                continue  # Main table is always needed
+
+            if table.only_used_for_join:
+                # Table columns only appear in JOIN condition, not in output
+                dep = UnusedDependency(
+                    type="join",
+                    name=table.full_name,
+                    location=f"{table.join_type} JOIN {table.full_name}"
+                    + (f" AS {table.alias}" if table.alias else ""),
+                    reason=f"Columns from this table ({', '.join(table.columns_in_join_condition)}) are only used in the JOIN condition, not in the query output",
+                    recommendation=(
+                        "Consider if this JOIN is necessary. If you're only using it for filtering, "
+                        "an EXISTS subquery or IN clause might be more efficient"
+                    ),
+                    impact="This table is an upstream dependency but doesn't directly contribute data to the output",
+                    confidence=0.7,  # Lower confidence - might be intentional
+                    columns_in_join_only=table.columns_in_join_condition,
+                )
+                result.unused_joins.append(dep)
+                continue
+
+            if not table.is_used:
+                # Table is joined but no columns are used anywhere
+                dep = UnusedDependency(
+                    type="join",
+                    name=table.full_name,
+                    location=f"{table.join_type} JOIN {table.full_name}"
+                    + (f" AS {table.alias}" if table.alias else ""),
+                    reason="No columns from this table are used in SELECT, WHERE, GROUP BY, or HAVING",
+                    recommendation=f"Remove the {table.join_type} JOIN to '{table.full_name}' - it adds overhead without contributing to results",
+                    impact="Removing this join will improve query performance and reduce upstream dependencies",
+                    confidence=0.95,
+                    columns_in_join_only=table.columns_in_join_condition,
+                )
+                result.unused_joins.append(dep)
+
+    def _detect_unused_ctes(self, result: UnusedDepsAnalysisResult) -> None:
+        """Detect CTEs that are defined but never used."""
+        for cte in result.cte_usage:
+            if not cte.is_used:
+                dep = UnusedDependency(
+                    type="cte",
+                    name=cte.name,
+                    location=f"WITH {cte.name} AS (...)",
+                    reason="This CTE is defined but never referenced in the query",
+                    recommendation=f"Remove the CTE '{cte.name}' - it's dead code that adds complexity",
+                    impact="Removing unused CTEs improves query readability and may improve planning time",
+                    confidence=1.0,  # Very confident - unused CTE is clearly dead code
+                )
+                result.unused_ctes.append(dep)
+
+
+def analyze_sql_for_unused_deps(sql: str, dialect: str = "snowflake") -> UnusedDepsAnalysisResult:
+    """Convenience function to analyze SQL for unused dependencies.
+
+    Args:
+        sql: SQL query to analyze
+        dialect: SQL dialect
+
+    Returns:
+        Analysis result with any detected issues
+    """
+    detector = UnusedDependencyDetector(dialect=dialect)
+    return detector.analyze(sql)
+
+
+def get_optimization_summary(result: UnusedDepsAnalysisResult) -> str:
+    """Generate a human-readable summary of optimization opportunities.
+
+    Args:
+        result: Analysis result
+
+    Returns:
+        Formatted summary string
+    """
+    if not result.has_issues:
+        return "No unused dependencies detected."
+
+    lines = [
+        f"Found {result.total_issues} potential optimization(s):",
+        "",
+    ]
+
+    if result.unused_joins:
+        lines.append(f"## Unused JOINs ({len(result.unused_joins)})")
+        for dep in result.unused_joins:
+            lines.append(f"  - {dep.name}")
+            lines.append(f"    Reason: {dep.reason}")
+            lines.append(f"    Recommendation: {dep.recommendation}")
+            lines.append("")
+
+    if result.unused_ctes:
+        lines.append(f"## Unused CTEs ({len(result.unused_ctes)})")
+        for dep in result.unused_ctes:
+            lines.append(f"  - {dep.name}")
+            lines.append(f"    Reason: {dep.reason}")
+            lines.append("")
+
+    return "\n".join(lines)