sql-dialect-master 1.1.1__py3-none-any.whl

backend/core/parser.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""SQL Parser - Parse and analyze SQL statements using sqlglot.
+
+Provides comprehensive SQL parsing with:
+- 12 database dialects support
+- AST extraction and analysis
+- Element extraction (tables, columns, functions, joins)
+- Query type detection
+"""
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import sqlglot
+from sqlglot import exp
+
+from .config import SUPPORTED_DIALECTS, settings
+
+__all__ = ['SUPPORTED_DIALECTS', 'ParseResult', 'SQLParser']
+
+
+@dataclass
+class ParseResult:
+    """Result of SQL parsing operation."""
+    success: bool
+    ast: Optional[exp.Expression] = None
+    dialect: str = ""
+    error: Optional[str] = None
+    tables: List[Dict[str, Any]] = field(default_factory=list)
+    columns: List[Dict[str, Any]] = field(default_factory=list)
+    functions: List[Dict[str, Any]] = field(default_factory=list)
+    joins: List[Dict[str, Any]] = field(default_factory=list)
+    query_type: str = ""
+
+
+class SQLParser:
+    """SQL Parser using sqlglot for multi-dialect support."""
+
+    def __init__(self, dialect: str = "hive"):
+        """Initialize parser with a normalized supported dialect."""
+        dialect_lower = dialect.strip().lower()
+        if dialect_lower not in SUPPORTED_DIALECTS:
+            raise ValueError(
+                f"Unsupported dialect: {dialect}. "
+                f"Supported: {', '.join(SUPPORTED_DIALECTS)}"
+            )
+        self.dialect = dialect_lower
+
+    def parse(self, sql: str) -> ParseResult:
+        """Parse SQL string into AST."""
+        if not isinstance(sql, str):
+            return ParseResult(success=False, error="SQL statement must be a string", dialect=self.dialect)
+        max_sql_length = getattr(settings, "parser_max_sql_length", settings.transpiler_max_sql_length)
+        if len(sql) > max_sql_length:
+            return ParseResult(
+                success=False,
+                error=f"SQL exceeds maximum length of {max_sql_length} characters",
+                dialect=self.dialect,
+            )
+        if not sql or not sql.strip():
+            return ParseResult(
+                success=False,
+                error="Empty SQL statement",
+                dialect=self.dialect
+            )
+
+        try:
+            ast = sqlglot.parse_one(sql, read=self.dialect)
+            result = ParseResult(
+                success=True,
+                ast=ast,
+                dialect=self.dialect,
+                query_type=self._get_query_type(ast)
+            )
+            result.tables = self._extract_tables(ast)
+            result.columns = self._extract_columns(ast)
+            result.functions = self._extract_functions(ast)
+            result.joins = self._extract_joins(ast)
+            return result
+        except Exception as e:
+            return ParseResult(
+                success=False,
+                error=str(e),
+                dialect=self.dialect
+            )
+
+    def validate(self, sql: str) -> tuple:
+        """Validate SQL syntax."""
+        if not isinstance(sql, str):
+            return False, "SQL statement must be a string"
+        max_sql_length = getattr(settings, "parser_max_sql_length", settings.transpiler_max_sql_length)
+        if len(sql) > max_sql_length:
+            return False, f"SQL exceeds maximum length of {max_sql_length} characters"
+        if not sql or not sql.strip():
+            return False, "Empty SQL statement"
+
+        try:
+            sqlglot.parse_one(sql, read=self.dialect)
+            return True, None
+        except Exception as e:
+            return False, str(e)
+
+    def extract_elements(self, sql: str) -> Dict[str, Any]:
+        """Extract all SQL elements for analysis."""
+        result = self.parse(sql)
+        if not result.success:
+            return {"error": result.error}
+
+        return {
+            "query_type": result.query_type,
+            "tables": result.tables,
+            "columns": result.columns,
+            "functions": result.functions,
+            "joins": result.joins,
+            "has_subquery": self._has_subquery(result.ast),
+            "has_cte": self._has_cte(result.ast),
+            "has_window": self._has_window_function(result.ast),
+            "has_aggregation": self._has_aggregation(result.ast),
+            "has_distinct": self._has_distinct(result.ast),
+            "has_union": self._has_union(result.ast),
+            "has_order_by": self._has_order_by(result.ast),
+            "has_limit": self._has_limit(result.ast),
+        }
+
+    def get_complexity_score(self, sql: str) -> Dict[str, Any]:
+        """Calculate SQL complexity score."""
+        result = self.parse(sql)
+        if not result.success:
+            return {"error": result.error, "score": 0}
+
+        score = 1
+        factors = []
+        if result.joins:
+            join_score = len(result.joins) * 2
+            score += join_score
+            factors.append(f"Joins: +{join_score}")
+        if self._has_subquery(result.ast):
+            score += 3
+            factors.append("Subquery: +3")
+        if self._has_cte(result.ast):
+            score += 2
+            factors.append("CTE: +2")
+        if self._has_window_function(result.ast):
+            score += 2
+            factors.append("Window function: +2")
+        if self._has_aggregation(result.ast):
+            score += 1
+            factors.append("Aggregation: +1")
+        if self._has_union(result.ast):
+            score += 2
+            factors.append("UNION: +2")
+
+        if score <= 2:
+            level = "Simple"
+        elif score <= 5:
+            level = "Moderate"
+        elif score <= 10:
+            level = "Complex"
+        else:
+            level = "Very Complex"
+
+        return {
+            "score": score,
+            "level": level,
+            "factors": factors,
+            "tables_count": len(result.tables),
+            "columns_count": len(result.columns),
+            "functions_count": len(result.functions),
+            "joins_count": len(result.joins)
+        }
+
+    def _get_query_type(self, ast: exp.Expression) -> str:
+        """Determine the type of SQL query."""
+        type_map = {
+            exp.Select: "SELECT",
+            exp.Insert: "INSERT",
+            exp.Update: "UPDATE",
+            exp.Delete: "DELETE",
+            exp.Create: "CREATE",
+            exp.Drop: "DROP",
+            exp.Alter: "ALTER",
+            exp.Merge: "MERGE",
+            exp.Union: "UNION",
+        }
+        for exp_type, name in type_map.items():
+            if isinstance(ast, exp_type):
+                return name
+        return "UNKNOWN"
+
+    def _extract_tables(self, ast: exp.Expression) -> List[Dict[str, Any]]:
+        """Extract all table references."""
+        tables = []
+        seen = set()
+        for table in ast.find_all(exp.Table):
+            key = (table.name, table.alias)
+            if key not in seen:
+                seen.add(key)
+                tables.append({
+                    "name": table.name,
+                    "alias": table.alias if table.alias else None,
+                    "schema": table.db if hasattr(table, 'db') and table.db else None,
+                    "catalog": table.catalog if hasattr(table, 'catalog') and table.catalog else None
+                })
+        return tables
+
+    def _extract_columns(self, ast: exp.Expression) -> List[Dict[str, Any]]:
+        """Extract all column references."""
+        columns = []
+        seen = set()
+        for col in ast.find_all(exp.Column):
+            key = (col.name, col.table)
+            if key not in seen:
+                seen.add(key)
+                columns.append({
+                    "name": col.name,
+                    "table": col.table if col.table else None,
+                    "alias": None
+                })
+        return columns
+
+    def _extract_functions(self, ast: exp.Expression) -> List[Dict[str, Any]]:
+        """Extract all function calls."""
+        functions = []
+        seen = set()
+        for func in ast.find_all(exp.Func):
+            name = func.sql_name() if hasattr(func, 'sql_name') else type(func).__name__
+            if name not in seen:
+                seen.add(name)
+                functions.append({
+                    "name": name,
+                    "args_count": len(func.args) if hasattr(func, 'args') else 0,
+                    "is_aggregate": isinstance(func, (exp.AggFunc,)),
+                    "is_window": isinstance(getattr(func, 'parent', None), exp.Window)
+                })
+        return functions
+
+    def _extract_joins(self, ast: exp.Expression) -> List[Dict[str, Any]]:
+        """Extract all JOIN operations."""
+        joins = []
+        for join in ast.find_all(exp.Join):
+            join_type = "INNER"
+            if join.kind:
+                join_type = join.kind.upper()
+            elif join.side:
+                join_type = f"{join.side.upper()} OUTER"
+            if hasattr(join.this, 'name'):
+                table_name = join.this.name
+            elif hasattr(join.this, 'alias'):
+                table_name = str(join.this)
+            else:
+                table_name = str(join.this)
+            joins.append({
+                "type": join_type,
+                "table": table_name,
+                "on_condition": str(join.args.get('on')) if join.args.get('on') else None
+            })
+        return joins
+
+    def _has_subquery(self, ast: exp.Expression) -> bool:
+        return len(list(ast.find_all(exp.Subquery))) > 0
+
+    def _has_cte(self, ast: exp.Expression) -> bool:
+        return ast.find(exp.With) is not None
+
+    def _has_window_function(self, ast: exp.Expression) -> bool:
+        return len(list(ast.find_all(exp.Window))) > 0
+
+    def _has_aggregation(self, ast: exp.Expression) -> bool:
+        return ast.find(exp.Group) is not None or ast.find(exp.AggFunc) is not None
+
+    def _has_distinct(self, ast: exp.Expression) -> bool:
+        return ast.find(exp.Distinct) is not None
+
+    def _has_union(self, ast: exp.Expression) -> bool:
+        return isinstance(ast, exp.Union) or ast.find(exp.Union) is not None
+
+    def _has_order_by(self, ast: exp.Expression) -> bool:
+        return ast.find(exp.Order) is not None
+
+    def _has_limit(self, ast: exp.Expression) -> bool:
+        return ast.find(exp.Limit) is not None or ast.find(exp.Fetch) is not None

backend/core/post_processor.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Post-processor for dialect-specific SQL transformations.
+
+Uses the rule engine for declarative transformations and handles
+complex cases that require custom logic.
+"""
+import logging
+from typing import Tuple, List, Callable
+
+import sqlglot
+from sqlglot import exp
+
+from .rules import rule_engine, RuleEngine
+from .function_call_scanner import replace_function_calls
+from .p1_sql_scanner import mask_non_executable
+
+logger = logging.getLogger(__name__)
+
+
+class PostProcessor:
+    """Apply dialect-specific fixes after sqlglot transpilation."""
+
+    def __init__(self, engine: RuleEngine = None):
+        self.engine = engine or rule_engine
+
+    def process(self, sql: str, source: str, target: str) -> Tuple[str, List[str]]:
+        result = sql
+        all_notes = []
+        result, rule_notes = self.engine.apply_rules(result, source, target)
+        all_notes.extend(rule_notes)
+        result, custom_notes = self._apply_custom_transformations(result, source, target)
+        all_notes.extend(custom_notes)
+        warnings = self._check_warnings(sql, source, target)
+        all_notes.extend(warnings)
+        return result, all_notes
+
+    def _apply_custom_transformations(self, sql: str, source: str, target: str) -> Tuple[str, List[str]]:
+        result = sql
+        notes = []
+        if source == "oracle" and target in ("hive", "spark", "databricks"):
+            result, listagg_notes = self._convert_listagg_to_array_join(result)
+            notes.extend(listagg_notes)
+        return result, notes
+
+    def _replace_function_calls(self, sql: str, function_name: str, replacer: Callable[[str, str], str]) -> str:
+        """Replace function calls using the shared quote/comment-aware scanner."""
+        return replace_function_calls(sql, function_name, replacer)
+
+    def _check_warnings(self, sql: str, source: str, target: str) -> List[str]:
+        warnings = []
+        sql_upper = mask_non_executable(sql).upper()
+        if source == "hive" and "INSERT OVERWRITE" in sql_upper and target in ("mysql", "postgres", "tsql", "oracle"):
+            warnings.append(f"WARNING: INSERT OVERWRITE not supported in {target}. Use TRUNCATE + INSERT or MERGE instead.")
+        if source == "oracle" and "CONNECT BY" in sql_upper and target in ("hive", "postgres", "mysql"):
+            warnings.append("WARNING: CONNECT BY requires manual conversion to WITH RECURSIVE CTE")
+        if source == "hive" and target in ("mysql", "postgres", "oracle", "tsql") and ("DISTRIBUTE BY" in sql_upper or "CLUSTER BY" in sql_upper):
+            warnings.append("WARNING: DISTRIBUTE BY/CLUSTER BY are Hive-specific hints, removed in target")
+        if any(kw in sql_upper for kw in ["CREATE PROCEDURE", "CREATE FUNCTION", "BEGIN", "DECLARE"]):
+            warnings.append("WARNING: Procedural code detected. Stored procedure syntax varies significantly between databases.")
+        if "MATERIALIZED VIEW" in sql_upper:
+            warnings.append("WARNING: Materialized view syntax and refresh mechanisms vary by database.")
+        return warnings
+
+    def _convert_listagg_to_array_join(self, sql: str) -> Tuple[str, List[str]]:
+        """Convert Oracle LISTAGG→Hive-family aggregated output to ARRAY_JOIN(COLLECT_LIST(...)).
+
+        sqlglot transpiles Oracle LISTAGG to GROUP_CONCAT for hive/spark/databricks,
+        but Hive does not natively support GROUP_CONCAT. This method rewrites the
+        sqlglot-produced GROUP_CONCAT(... ORDER BY ..., sep) form to
+        ARRAY_JOIN(COLLECT_LIST(expr), sep). Ordering semantics are explicitly
+        noted as lost because Hive's COLLECT_LIST does not preserve input order.
+
+        The separator is the last Ordered expression when it is a string Literal.
+        """
+        try:
+            tree = sqlglot.parse_one(sql, read="hive")
+        except sqlglot.errors.ParseError:
+            return sql, []
+
+        notes = []
+        rewritten = False
+
+        for node in list(tree.walk()):
+            if not isinstance(node, exp.GroupConcat):
+                continue
+            expr = node.this
+            sep = node.args.get("separator")
+
+            has_order = isinstance(expr, exp.Order)
+            if has_order:
+                inner_expr = expr.this
+                # The separator is encoded as the last Ordered expression when it's a string literal
+                order_exprs = expr.expressions
+                if (
+                    order_exprs
+                    and isinstance(order_exprs[-1].this, exp.Literal)
+                    and order_exprs[-1].this.is_string
+                ):
+                    sep = order_exprs[-1].this
+            else:
+                inner_expr = expr
+
+            collect_list = exp.Anonymous(
+                this="COLLECT_LIST",
+                expressions=[inner_expr],
+            )
+            array_join = exp.Anonymous(
+                this="ARRAY_JOIN",
+                expressions=[collect_list, sep or exp.Literal.string(",")],
+            )
+            node.replace(array_join)
+            if has_order:
+                notes.append(
+                    "LISTAGG ordering semantics cannot be preserved in Hive; "
+                    "converted to ARRAY_JOIN(COLLECT_LIST()) which does not guarantee order"
+                )
+            rewritten = True
+
+        if rewritten:
+            return tree.sql(dialect="hive"), notes
+        return sql, notes
+
+    def get_stats(self) -> dict:
+        return {
+            "rule_engine": self.engine.get_stats(),
+            "custom_handlers": [
+                "LISTAGG to ARRAY_JOIN (Hive)",
+            ],
+        }