sql-dialect-master 1.1.1__py3-none-any.whl

backend/core/functions_lookup.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+"""Function Encyclopedia - Search and compare SQL functions across dialects.
+
+Provides comprehensive SQL function lookup with:
+- 50+ SQL functions across 11 categories
+- Cross-database syntax comparison for 12 dialects
+- Fuzzy search with relevance scoring
+- Category-based filtering
+- Function examples and parameter documentation
+"""
+import json
+import logging
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from difflib import SequenceMatcher
+
+from .config import (
+    SUPPORTED_DIALECTS,
+    FUNCTION_CATEGORIES,
+    CATEGORY_DESCRIPTIONS,
+    FunctionInfo,
+    settings
+)
+from .exceptions import ConfigurationError
+from .function_semantics import FunctionSemanticRegistry
+from .p2_data_validation import _validate_function_entry
+
+# Configure module logger
+logger = logging.getLogger(__name__)
+
+
+class FunctionEncyclopedia:
+    """SQL Function Encyclopedia with multi-dialect comparison.
+    
+    Features:
+    - 50+ SQL functions
+    - 11 categories: string, date, math, aggregate, window, conditional, conversion, json, array, system, geo
+    - 12 database dialects support
+    - Fuzzy search with relevance scoring
+    - Function examples and parameter documentation
+    - Strict semantic equivalence classification for reviewed function pairs
+    """
+    
+    CATEGORIES = FUNCTION_CATEGORIES
+    CATEGORY_DESCRIPTIONS = CATEGORY_DESCRIPTIONS
+    
+    def __init__(self, data_path: Optional[Path] = None, semantic_registry: Optional[FunctionSemanticRegistry] = None):
+        """Initialize encyclopedia with function data.
+        
+        Args:
+            data_path: Path to functions_db.json. Uses default if None.
+            semantic_registry: Optional reviewed semantic registry. Defaults to the shared registry.
+        """
+        if data_path is None:
+            data_path = Path(__file__).parent / "functions_db.json"
+        self.data_path = data_path
+        self.semantic_registry = semantic_registry or FunctionSemanticRegistry()
+        self.functions: List[FunctionInfo] = self._load_data()
+        self._build_index()
+    
+    def _load_data(self) -> List[FunctionInfo]:
+        """Load function data from JSON file.
+
+        Missing or malformed packaged data is a configuration error and must
+        not be silently converted into an empty encyclopedia.
+        """
+        try:
+            with open(self.data_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ConfigurationError(
+                f"Unable to load function data from {self.data_path}: {exc}",
+                details={"data_path": str(self.data_path)},
+            ) from exc
+
+        if not isinstance(data, dict):
+            raise ConfigurationError(
+                f"Invalid function data in {self.data_path}: top-level JSON value must be an object",
+                details={"data_path": str(self.data_path)},
+            )
+
+        functions = data.get("functions", [])
+        if not isinstance(functions, list):
+            raise ConfigurationError(
+                f"Invalid function data in {self.data_path}: 'functions' must be a list",
+                details={"data_path": str(self.data_path)},
+            )
+        for index, entry in enumerate(functions):
+            _validate_function_entry(self.data_path, index, entry)
+        return functions
+    
+    def _build_index(self) -> None:
+        """Build search index for fast lookup."""
+        self.name_index: Dict[str, FunctionInfo] = {}
+        self.category_index: Dict[str, List[FunctionInfo]] = {cat: [] for cat in self.CATEGORIES}
+        
+        for func in self.functions:
+            name = func.get("name", "").upper()
+            self.name_index[name] = func
+            
+            category = func.get("category", "").lower()
+            if category in self.category_index:
+                self.category_index[category].append(func)
+    
+    def search(self, query: str, limit: int = None) -> List[FunctionInfo]:
+        """Search functions with fuzzy matching.
+        
+        Args:
+            query: Search query
+            limit: Maximum results to return
+            
+        Returns:
+            List of matching functions sorted by relevance
+        """
+        limit = limit or settings.function_search_limit
+        query = query.upper().strip()
+        
+        if not query:
+            return self.functions[:limit]
+        
+        results: List[tuple] = []
+        
+        for func in self.functions:
+            name = func.get("name", "").upper()
+            description = func.get("description", "").upper()
+            
+            if query == name:
+                results.append((func, 1.0))
+            elif name.startswith(query):
+                results.append((func, 0.9))
+            elif query in name:
+                results.append((func, 0.7))
+            elif query in description:
+                results.append((func, 0.5))
+            else:
+                ratio = SequenceMatcher(None, query, name).ratio()
+                if ratio > settings.function_fuzzy_threshold:
+                    results.append((func, ratio))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        return [r[0] for r in results[:limit]]
+    
+    def get_function(self, name: str) -> Optional[FunctionInfo]:
+        """Get detailed function information by exact name."""
+        return self.name_index.get(name.upper())
+    
+    def list_by_category(self, category: str) -> List[FunctionInfo]:
+        """List all functions in a category."""
+        category = category.lower()
+        return self.category_index.get(category, [])
+    
+    def get_dialect_syntax(self, func_name: str, dialect: str) -> Optional[str]:
+        """Get function syntax for specific dialect."""
+        func = self.get_function(func_name)
+        if func:
+            return func.get("dialects", {}).get(dialect.lower())
+        return None
+
+    def resolve_function_semantics(self, func_name: str, source_dialect: str, target_dialect: str) -> Dict[str, Any]:
+        """Resolve reviewed semantic equivalence without inferring from syntax presence."""
+        return self.semantic_registry.resolve(func_name, source_dialect, target_dialect)
+    
+    def compare_dialects(self, func_name: str, dialects: List[str] = None) -> Dict[str, Any]:
+        """Compare function syntax across multiple dialects."""
+        func = self.get_function(func_name)
+        if not func:
+            return {"error": f"Function {func_name} not found"}
+        
+        if dialects is None:
+            dialects = SUPPORTED_DIALECTS
+        
+        comparison = {
+            "name": func.get("name"),
+            "description": func.get("description"),
+            "category": func.get("category"),
+            "parameters": func.get("parameters", []),
+            "examples": func.get("examples", {}),
+            "notes": func.get("notes", ""),
+            "dialects": {}
+        }
+        
+        for dialect in dialects:
+            syntax = func.get("dialects", {}).get(dialect)
+            comparison["dialects"][dialect] = syntax if syntax else "N/A"
+        
+        available = [d for d in dialects if comparison["dialects"].get(d, "N/A") != "N/A"]
+        comparison["availability"] = {
+            "supported": available,
+            "not_supported": [d for d in dialects if d not in available],
+            "coverage": f"{len(available)}/{len(dialects)}"
+        }
+        
+        return comparison
+    
+    def get_all_categories(self) -> List[Dict[str, Any]]:
+        """Get list of all categories with function counts and descriptions."""
+        return [
+            {
+                "name": cat,
+                "count": len(funcs),
+                "description": self.CATEGORY_DESCRIPTIONS.get(cat, ""),
+                "icon": self._get_category_icon(cat)
+            }
+            for cat, funcs in self.category_index.items()
+            if funcs
+        ]
+    
+    def _get_category_icon(self, category: str) -> str:
+        """Get icon for category."""
+        icons = {
+            "string": "📝", "date": "📅", "math": "🔢", "aggregate": "📊",
+            "window": "🪟", "conditional": "❓", "conversion": "🔄", "json": "📋",
+            "array": "📦", "system": "⚙️", "geo": "🌍"
+        }
+        return icons.get(category, "📄")
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get comprehensive encyclopedia statistics."""
+        dialect_coverage = {d: 0 for d in SUPPORTED_DIALECTS}
+        for func in self.functions:
+            for dialect, syntax in func.get("dialects", {}).items():
+                if syntax and syntax != "N/A":
+                    dialect_coverage[dialect] = dialect_coverage.get(dialect, 0) + 1
+        
+        return {
+            "total_functions": len(self.functions),
+            "categories": len([c for c in self.category_index.values() if c]),
+            "by_category": {cat: len(funcs) for cat, funcs in self.category_index.items()},
+            "dialects_covered": len(SUPPORTED_DIALECTS),
+            "dialect_coverage": dialect_coverage,
+            "top_categories": sorted(
+                [(cat, len(funcs)) for cat, funcs in self.category_index.items()],
+                key=lambda x: x[1], reverse=True
+            )[:5]
+        }
+
+    def find_equivalent(self, func_name: str, source_dialect: str, target_dialect: str) -> Optional[Dict[str, Any]]:
+        """Find target syntax while exposing strict semantic classification."""
+        func = self.get_function(func_name)
+        if not func:
+            return None
+        
+        dialects = func.get("dialects", {})
+        source_dialect = source_dialect.lower()
+        target_dialect = target_dialect.lower()
+        source_syntax = dialects.get(source_dialect)
+        target_syntax = dialects.get(target_dialect)
+        
+        if not source_syntax or source_syntax == "N/A":
+            return None
+
+        semantic = self.resolve_function_semantics(func_name, source_dialect, target_dialect)
+        
+        return {
+            "function": func_name,
+            "source_dialect": source_dialect,
+            "target_dialect": target_dialect,
+            "source_syntax": source_syntax,
+            "target_syntax": target_syntax if target_syntax and target_syntax != "N/A" else None,
+            "supported_in_target": target_syntax is not None and target_syntax != "N/A",
+            "semantic_status": semantic["status"],
+            "semantic_equivalent": semantic["equivalent"],
+            "semantic_argument_semantics": semantic.get("argument_semantics", ""),
+            "semantic_return_shape": semantic.get("return_shape", ""),
+            "semantic_null_behavior": semantic.get("null_behavior", ""),
+            "semantic_notes": semantic.get("notes", ""),
+            "notes": func.get("notes", "")
+        }
+    
+    def get_functions_by_dialect(self, dialect: str) -> List[Dict[str, Any]]:
+        """Get all functions available in a specific dialect."""
+        dialect = dialect.lower()
+        available = []
+        
+        for func in self.functions:
+            syntax = func.get("dialects", {}).get(dialect)
+            if syntax and syntax != "N/A":
+                available.append({
+                    "name": func.get("name"),
+                    "category": func.get("category"),
+                    "syntax": syntax,
+                    "description": func.get("description", ""),
+                    "parameters": func.get("parameters", [])
+                })
+        
+        return available
+    
+    def search_by_description(self, query: str, limit: int = 10) -> List[FunctionInfo]:
+        """Search functions by description text."""
+        query = query.lower()
+        results = []
+        
+        for func in self.functions:
+            desc = func.get("description", "").lower()
+            name = func.get("name", "").lower()
+            
+            if query in desc or query in name:
+                score = 1.0 if query in name else 0.7
+                results.append((func, score))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        return [r[0] for r in results[:limit]]
+    
+    def get_function_alternatives(self, func_name: str, dialect: str) -> List[Dict[str, Any]]:
+        """Get alternative functions for a dialect that doesn't support the original."""
+        func = self.get_function(func_name)
+        if not func:
+            return []
+        
+        category = func.get("category", "")
+        dialect = dialect.lower()
+        
+        alternatives = []
+        for other_func in self.category_index.get(category, []):
+            if other_func.get("name") == func_name:
+                continue
+            
+            syntax = other_func.get("dialects", {}).get(dialect)
+            if syntax and syntax != "N/A":
+                alternatives.append({
+                    "name": other_func.get("name"),
+                    "syntax": syntax,
+                    "description": other_func.get("description", ""),
+                    "similarity": SequenceMatcher(
+                        None,
+                        func.get("description", ""),
+                        other_func.get("description", "")
+                    ).ratio()
+                })
+        
+        alternatives.sort(key=lambda x: x["similarity"], reverse=True)
+        return alternatives[:5]

backend/core/metadata.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Canonical metadata for SQL Dialect Master.
+
+Single source of truth for product counts, version facts, and
+dialect information. Every public surface (API, README verifier,
+frontend) must derive its displayed numbers from this module rather
+than from hardcoded literals.
+
+Design rules:
+- Counts (functions, types, rules, dialects) are computed at import
+  time from the canonical data files / engine objects.
+- Version constants live here and are the sole release source
+  (``PROJECT_VERSION`` / ``API_VERSION``).
+- The frontend architecture version is deliberately separate from
+  the product version: "Frontend v2" is not "release v2.0".
+"""
+from __future__ import annotations
+
+# ── Release version facts (sole source of truth) ────────────────────────
+PROJECT_VERSION = "1.1.1"
+API_VERSION = "1.1.1"
+# Frontend v2 architecture — independent of the product release version.
+FRONTEND_ARCHITECTURE_VERSION = 2
+
+
+def _canonical_counts() -> dict:
+    """Compute dialect / function / type / rule counts from data files."""
+    import json
+    from pathlib import Path
+
+    base = Path(__file__).parent
+    dialects, functions, types = 0, 0, 0
+    type_file = base / "type_mapping.json"
+    if type_file.exists():
+        with type_file.open(encoding="utf-8") as f:
+            data = json.load(f)
+        dialects = len(data.get("dialects", []))
+        types = len(data.get("mappings", {}))
+    func_file = base / "functions_db.json"
+    if func_file.exists():
+        with func_file.open(encoding="utf-8") as f:
+            fdata = json.load(f)
+        functions = len(fdata.get("functions", []))
+    return {"dialects": dialects, "functions": functions, "types": types}
+
+
+_counts = _canonical_counts()
+
+SUPPORTED_DIALECTS_COUNT = _counts["dialects"]
+FUNCTION_COUNT = _counts["functions"]
+TYPE_COUNT = _counts["types"]
+
+# Rule count depends on the RuleEngine; compute lazily to keep the
+# import graph clean. Uses the same engine instance the transpiler
+# uses, so RULE_COUNT() matches the API's reported rule count.
+def RULE_COUNT() -> int:
+    """Number of active conversion rules in the RuleEngine."""
+    from backend.core.post_processor import PostProcessor
+
+    return len(PostProcessor().engine.rules)
+
+
+__all__ = [
+    "PROJECT_VERSION",
+    "API_VERSION",
+    "FRONTEND_ARCHITECTURE_VERSION",
+    "SUPPORTED_DIALECTS_COUNT",
+    "FUNCTION_COUNT",
+    "TYPE_COUNT",
+    "RULE_COUNT",
+]

backend/core/nl2sql.py

@@ -0,0 +1,85 @@
+"""Stable NL2SQL facade with focused semantic extraction fixes."""
+import re
+
+from .nl2sql_pipeline.orchestrator import PipelineNL2SQLGenerator as _NL2SQLGenerator
+from .nl2sql_pipeline.models import NL2SQLResult
+
+
+class NL2SQLGenerator(_NL2SQLGenerator):
+    """Keep the public generator API while fixing count/group semantics."""
+
+    def _match_templates(self, text: str):
+        """Route count-by-group requests through enhanced semantic extraction."""
+        template, match_groups = super()._match_templates(text)
+        if template and template.name == "count_by_group":
+            return None, None
+        return template, match_groups
+
+    def _extract_aggregations(self, text: str) -> list:
+        """Preserve the established ``count`` alias for COUNT aggregates."""
+        aggregations = super()._extract_aggregations(text)
+        return [
+            "COUNT(*) AS count" if aggregation == "COUNT(*)" else aggregation
+            for aggregation in aggregations
+        ]
+
+    def _extract_group_by(self, text: str) -> list:
+        """Extract explicit per-group intent such as '每个部门' when needed."""
+        group_cols = super()._extract_group_by(text)
+        if group_cols:
+            return group_cols
+
+        match = re.search(r"(?:每个|各个)\s*([^的\s]+?)\s*的", text, re.IGNORECASE)
+        if not match:
+            match = re.search(r"(?:per|each)\s+(\w+)\b", text, re.IGNORECASE)
+        if not match:
+            return []
+
+        group_text = match.group(1).strip()
+        for keyword, column in self.COLUMN_PATTERNS.items():
+            if keyword in group_text:
+                return [column]
+        if "部门" in group_text or "department" in group_text:
+            return ["department"]
+        if "用户" in group_text or "user" in group_text:
+            return ["user_id"]
+        if "日期" in group_text or "date" in group_text:
+            return ["date"]
+        return []
+
+    def _extract_ordering(self, text: str):
+        """Order aggregate-count requests by the requested count deterministically."""
+        ordering = super()._extract_ordering(text)
+        count_requested = any(
+            keyword in text
+            for keyword in ["数量", "计数", "多少", "count", "how many"]
+        )
+        sorting_requested = any(
+            keyword in text
+            for keyword in ["排序", "排列", "sort", "order", "sorted"]
+        )
+        if count_requested and sorting_requested:
+            descending = any(
+                keyword in text
+                for keyword in [
+                    "降序",
+                    "从大到小",
+                    "递减",
+                    "desc",
+                    "descending",
+                    "decreasing",
+                ]
+            )
+            ascending = any(
+                keyword in text
+                for keyword in [
+                    "升序",
+                    "从小到大",
+                    "递增",
+                    "asc",
+                    "ascending",
+                    "increasing",
+                ]
+            )
+            return "count", "ASC" if ascending and not descending else "DESC"
+        return ordering

backend/core/nl2sql_components/__init__.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""NL2SQL Components Package - Modular components for NL2SQL generation.
+
+This package provides modular components for natural language to SQL
+generation:
+- Tokenizer: Smart tokenization with Chinese and English support
+- QueryTemplate: Pattern-based SQL generation using prioritized templates
+- Mappings: Language mappings for tables, columns, and SQL operations
+- CoreExtract: Operation, table, column, limit, distinct extraction
+- Relations: Join extraction, join-key inference, relational keywords
+- Aggregations: SQL aggregate function extraction
+
+Example:
+    from backend.core.nl2sql_components import Tokenizer, KEYWORDS
+
+    tokens = Tokenizer.tokenize("查询所有用户")
+    print(tokens)  # ['查询', '所有', '用户']
+"""
+
+# Import main components for convenient access
+from .aggregations import extract_aggregations
+from .boolean_conditions import extract_boolean_conditions
+from .core_extract import (
+    check_distinct,
+    detect_operation,
+    extract_columns,
+    extract_limit,
+    extract_table,
+    extract_table_subject,
+    named_tables,
+)
+from .mappings import COLUMN_PATTERNS, KEYWORDS, TABLE_PATTERNS
+from .relations import (
+    JoinSpec,
+    extract_joins,
+    guess_join_key,
+    has_relational_keyword,
+    parse_join_spec,
+)
+from .templates import DEFAULT_QUERY_TEMPLATES, QueryTemplate
+from .tokenizer import JIEBA_AVAILABLE, Tokenizer
+
+__all__ = [
+    # tokenizer
+    "Tokenizer",
+    "JIEBA_AVAILABLE",
+    # templates
+    "QueryTemplate",
+    "DEFAULT_QUERY_TEMPLATES",
+    # mappings
+    "KEYWORDS",
+    "TABLE_PATTERNS",
+    "COLUMN_PATTERNS",
+    # core_extract
+    "detect_operation",
+    "extract_table",
+    "extract_table_subject",
+    "named_tables",
+    "extract_columns",
+    "extract_limit",
+    "check_distinct",
+    # relations
+    "extract_joins",
+    "guess_join_key",
+    "has_relational_keyword",
+    "JoinSpec",
+    "parse_join_spec",
+    # aggregations
+    "extract_aggregations",
+    # boolean conditions
+    "extract_boolean_conditions",
+]

backend/core/nl2sql_components/aggregations.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""NL2SQL aggregation extraction.
+
+Extracts aggregate function calls (COUNT, SUM, AVG, MAX, MIN) from
+natural-language text using keyword matching. Uses the shared
+COLUMN_PATTERNS to resolve the target column for value aggregates.
+"""
+from typing import Dict, List
+
+
+def extract_aggregations(text: str, column_patterns: Dict[str, str]) -> List[str]:
+    """Extract aggregation expressions from text.
+
+    Returns a list of SQL aggregate expressions, e.g.
+    ['COUNT(*)', 'SUM(amount)']. An empty list means no aggregation
+    was requested.
+    """
+    aggs: List[str] = []
+    agg_column = None
+    for pattern, column in column_patterns.items():
+        if pattern in text:
+            agg_column = column
+            break
+    col = agg_column or "amount"
+    if any(k in text for k in ["统计", "计数", "数量", "多少", "几个", "count", "total", "how many"]):
+        aggs.append("COUNT(*)")
+    if any(k in text for k in ["求和", "总和", "合计", "总计", "sum", "total of"]):
+        aggs.append(f"SUM({col})")
+    if any(k in text for k in ["平均", "均值", "平均值", "average", "avg", "mean"]):
+        aggs.append(f"AVG({col})")
+    if any(k in text for k in ["最大", "最高", "最多", "max", "maximum", "highest", "largest"]):
+        aggs.append(f"MAX({col})")
+    if any(k in text for k in ["最小", "最低", "最少", "min", "minimum", "lowest", "smallest"]):
+        aggs.append(f"MIN({col})")
+    return aggs