sql-dialect-master 1.1.1__py3-none-any.whl

backend/core/nl2sql_components/mappings.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Language mappings for NL2SQL.
+
+Contains Chinese-English keyword mappings for tables, columns, and SQL operations.
+"""
+from typing import Dict
+
+
+# Chinese-English keyword mappings
+KEYWORDS: Dict[str, str] = {
+    # Query operations
+    "查询": "SELECT", "获取": "SELECT", "查找": "SELECT", "显示": "SELECT", "列出": "SELECT",
+    "select": "SELECT", "get": "SELECT", "find": "SELECT", "show": "SELECT", "list": "SELECT", "fetch": "SELECT",
+    # Insert operations
+    "插入": "INSERT", "添加": "INSERT", "新增": "INSERT", "创建": "INSERT",
+    "insert": "INSERT", "add": "INSERT", "create": "INSERT",
+    # Update operations
+    "更新": "UPDATE", "修改": "UPDATE", "改变": "UPDATE", "设置": "UPDATE",
+    "update": "UPDATE", "modify": "UPDATE", "change": "UPDATE", "set": "UPDATE",
+    # Delete operations
+    "删除": "DELETE", "移除": "DELETE", "清除": "DELETE",
+    "delete": "DELETE", "remove": "DELETE", "drop": "DELETE",
+    # Aggregations
+    "统计": "COUNT", "计数": "COUNT", "数量": "COUNT", "多少": "COUNT", "几个": "COUNT",
+    "count": "COUNT", "total": "COUNT", "how many": "COUNT",
+    "求和": "SUM", "总和": "SUM", "合计": "SUM", "总计": "SUM",
+    "sum": "SUM", "total of": "SUM",
+    "平均": "AVG", "均值": "AVG", "平均值": "AVG",
+    "average": "AVG", "avg": "AVG", "mean": "AVG",
+    "最大": "MAX", "最高": "MAX", "最多": "MAX",
+    "max": "MAX", "maximum": "MAX", "highest": "MAX", "largest": "MAX",
+    "最小": "MIN", "最低": "MIN", "最少": "MIN",
+    "min": "MIN", "minimum": "MIN", "lowest": "MIN", "smallest": "MIN",
+    # Conditions
+    "大于": ">", "超过": ">", "高于": ">", "多于": ">",
+    "greater": ">", "more than": ">", "above": ">", "over": ">",
+    "小于": "<", "低于": "<", "少于": "<", "不足": "<",
+    "less": "<", "less than": "<", "below": "<", "under": "<",
+    "等于": "=", "是": "=", "为": "=",
+    "equals": "=", "equal": "=", "is": "=",
+    "不等于": "!=", "不是": "!=", "不为": "!=",
+    "not equal": "!=", "not": "!=", "isn't": "!=",
+    "包含": "LIKE", "含有": "LIKE", "像": "LIKE",
+    "contains": "LIKE", "like": "LIKE", "includes": "LIKE",
+    "开头": "LIKE_START", "以...开头": "LIKE_START",
+    "starts with": "LIKE_START", "beginning with": "LIKE_START",
+    "结尾": "LIKE_END", "以...结尾": "LIKE_END",
+    "ends with": "LIKE_END", "ending with": "LIKE_END",
+    "之间": "BETWEEN", "范围": "BETWEEN",
+    "between": "BETWEEN", "range": "BETWEEN",
+    "为空": "IS NULL", "空值": "IS NULL",
+    "is null": "IS NULL", "null": "IS NULL", "empty": "IS NULL",
+    "非空": "IS NOT NULL", "不为空": "IS NOT NULL",
+    "is not null": "IS NOT NULL", "not null": "IS NOT NULL", "not empty": "IS NOT NULL",
+    # Sorting
+    "排序": "ORDER BY", "按": "ORDER BY", "排列": "ORDER BY",
+    "sort": "ORDER BY", "order": "ORDER BY", "sorted": "ORDER BY",
+    "升序": "ASC", "从小到大": "ASC", "递增": "ASC",
+    "ascending": "ASC", "asc": "ASC", "increasing": "ASC",
+    "降序": "DESC", "从大到小": "DESC", "递减": "DESC",
+    "descending": "DESC", "desc": "DESC", "decreasing": "DESC",
+    # Grouping
+    "分组": "GROUP BY", "按...分组": "GROUP BY", "汇总": "GROUP BY",
+    "group": "GROUP BY", "group by": "GROUP BY", "grouped": "GROUP BY",
+    # Limiting
+    "前": "LIMIT", "限制": "LIMIT", "只要": "LIMIT", "仅": "LIMIT",
+    "top": "LIMIT", "limit": "LIMIT", "first": "LIMIT", "only": "LIMIT",
+    # Joins
+    "关联": "JOIN", "连接": "JOIN", "合并": "JOIN", "联合": "JOIN",
+    "join": "JOIN", "combine": "JOIN", "merge": "JOIN",
+    "左连接": "LEFT JOIN", "left join": "LEFT JOIN",
+    "右连接": "RIGHT JOIN", "right join": "RIGHT JOIN",
+    "内连接": "INNER JOIN", "inner join": "INNER JOIN",
+    # Time
+    "今天": "CURRENT_DATE", "today": "CURRENT_DATE",
+    "昨天": "DATE_SUB(CURRENT_DATE, 1)", "yesterday": "DATE_SUB(CURRENT_DATE, 1)",
+    "前天": "DATE_SUB(CURRENT_DATE, 2)", "day before yesterday": "DATE_SUB(CURRENT_DATE, 2)",
+    "本周": "WEEK", "this week": "WEEK",
+    "本月": "MONTH", "this month": "MONTH",
+    "本年": "YEAR", "this year": "YEAR",
+    "最近": ">=", "recent": ">=", "last": ">=", "past": ">=",
+    # Distinct
+    "去重": "DISTINCT", "唯一": "DISTINCT", "不重复": "DISTINCT",
+    "distinct": "DISTINCT", "unique": "DISTINCT",
+}
+
+# Common table patterns (expanded)
+TABLE_PATTERNS: Dict[str, str] = {
+    # Users
+    "用户": "users", "user": "users", "users": "users", "会员": "users", "member": "users",
+    "账户": "accounts", "account": "accounts", "accounts": "accounts",
+    # Orders
+    "订单": "orders", "order": "orders", "orders": "orders", "购买": "orders",
+    "销售": "sales", "sale": "sales", "sales": "sales",
+    # Products
+    "产品": "products", "product": "products", "products": "products",
+    "商品": "items", "item": "items", "items": "items", "货物": "items",
+    "库存": "inventory", "inventory": "inventory", "stock": "inventory",
+    # Logs
+    "日志": "logs", "log": "logs", "logs": "logs", "记录": "logs",
+    "操作日志": "operation_logs", "审计": "audit_logs", "audit": "audit_logs",
+    # HR
+    "员工": "employees", "employee": "employees", "employees": "employees", "职员": "employees",
+    "部门": "departments", "department": "departments", "departments": "departments",
+    "薪资": "salaries", "salary": "salaries", "工资": "salaries",
+    # Customers
+    "客户": "customers", "customer": "customers", "customers": "customers",
+    "供应商": "suppliers", "supplier": "suppliers", "vendors": "suppliers",
+    # Finance
+    "交易": "transactions", "transaction": "transactions", "transactions": "transactions",
+    "支付": "payments", "payment": "payments", "payments": "payments",
+    "发票": "invoices", "invoice": "invoices", "invoices": "invoices",
+    # Events
+    "事件": "events", "event": "events", "events": "events",
+    "活动": "activities", "activity": "activities",
+    # Data
+    "数据": "data", "data": "data", "表": "table", "table": "table",
+    "配置": "configs", "config": "configs", "settings": "configs",
+    # Categories
+    "分类": "categories", "categories": "categories",
+    "标签": "tags", "tag": "tags", "tags": "tags",
+    # Messages
+    "消息": "messages", "message": "messages", "messages": "messages",
+    "通知": "notifications", "notification": "notifications",
+    "评论": "comments", "comment": "comments", "comments": "comments",
+}
+
+# Column patterns (expanded)
+COLUMN_PATTERNS: Dict[str, str] = {
+    # Identity
+    "id": "id", "编号": "id", "标识": "id", "主键": "id",
+    "uuid": "uuid", "guid": "guid",
+    # Personal info
+    "姓名": "name", "名字": "name", "name": "name", "名称": "name",
+    "用户名": "username", "username": "username", "账号": "username",
+    "邮箱": "email", "email": "email", "电子邮件": "email",
+    "电话": "phone", "phone": "phone", "手机": "mobile", "mobile": "mobile",
+    "地址": "address", "address": "address", "住址": "address",
+    "年龄": "age", "age": "age",
+    "性别": "gender", "gender": "gender", "sex": "gender",
+    "生日": "birthday", "birthday": "birthday", "出生日期": "birth_date",
+    # Financial
+    "金额": "amount", "amount": "amount", "总额": "total_amount",
+    "价格": "price", "price": "price", "单价": "unit_price",
+    "数量": "quantity", "quantity": "quantity", "qty": "quantity",
+    "成本": "cost", "cost": "cost",
+    "利润": "profit", "profit": "profit",
+    "折扣": "discount", "discount": "discount",
+    "税": "tax", "tax": "tax",
+    "余额": "balance", "balance": "balance",
+    "薪水": "salary", "salary": "salary", "工资": "salary",
+    # Time
+    "日期": "date", "date": "date",
+    "时间": "time", "time": "time",
+    "创建时间": "created_at", "created_at": "created_at", "create_time": "created_at",
+    "更新时间": "updated_at", "updated_at": "updated_at", "update_time": "updated_at",
+    "开始时间": "start_time", "start_time": "start_time", "start_date": "start_date",
+    "结束时间": "end_time", "end_time": "end_time", "end_date": "end_date",
+    "过期时间": "expire_time", "expire_at": "expire_at",
+    # Status
+    "状态": "status", "status": "status",
+    "类型": "type", "type": "type", "种类": "type",
+    "级别": "level", "level": "level", "等级": "level",
+    "优先级": "priority", "priority": "priority",
+    "是否": "is_", "是否有效": "is_valid", "是否删除": "is_deleted",
+    # Content
+    "标题": "title", "title": "title",
+    "内容": "content", "content": "content",
+    "描述": "description", "description": "description", "desc": "description",
+    "备注": "remark", "remark": "remark", "note": "note", "notes": "notes",
+    # Relations
+    "用户id": "user_id", "user_id": "user_id",
+    "订单id": "order_id", "order_id": "order_id",
+    "产品id": "product_id", "product_id": "product_id",
+    "部门id": "dept_id", "dept_id": "dept_id", "department_id": "department_id",
+    "父级": "parent_id", "parent_id": "parent_id",
+    # Categorization
+    "分类": "category", "category": "category",
+    # Metrics
+    "得分": "score", "score": "score", "分数": "score",
+    "评分": "rating", "rating": "rating",
+    "浏览量": "views", "views": "views", "view_count": "view_count",
+    "点击量": "clicks", "clicks": "clicks",
+    "下载量": "downloads", "downloads": "downloads",
+}

backend/core/nl2sql_components/relations.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""NL2SQL relation extraction: joins, join-key inference, relational keywords.
+
+The join-key knowledge base (``JOIN_KEY_PATTERNS``) is the single
+canonical source for inter-table relationships. Unknown relationships
+return ``None`` so the caller can fail closed instead of fabricating
+conditions.
+"""
+import re
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+# Canonical join-key knowledge base.
+# Pairs are stored both ways so lookups are order-independent.
+JOIN_KEY_PATTERNS: Dict[tuple, str] = {
+    ("users", "orders"): "users.id = orders.user_id",
+    ("orders", "users"): "orders.user_id = users.id",
+    ("users", "transactions"): "users.id = transactions.user_id",
+    ("transactions", "users"): "transactions.user_id = users.id",
+    ("orders", "products"): "orders.product_id = products.id",
+    ("products", "orders"): "products.id = orders.product_id",
+    ("users", "products"): "users.id = products.user_id",
+    ("products", "users"): "products.user_id = users.id",
+    ("employees", "departments"): "employees.dept_id = departments.id",
+    ("departments", "employees"): "departments.id = employees.dept_id",
+    ("orders", "payments"): "orders.id = payments.order_id",
+    ("payments", "orders"): "payments.order_id = orders.id",
+    ("customers", "orders"): "customers.id = orders.customer_id",
+    ("orders", "customers"): "orders.customer_id = customers.id",
+}
+
+# Negative relational existence keywords -> NOT EXISTS
+_NOT_EXISTS_KEYWORDS = [
+    "without", "not have", "no ",
+    "who don't have", "who doesn't have",
+    "who do not have", "who does not have",
+]
+
+# Positive relational existence keywords -> EXISTS
+_EXISTS_KEYWORDS = [
+    "who have", "who has", "that have", "that has",
+    "with orders", "with customers",
+    "with users", "with employees",
+    "with invoices", "with payments", "with transactions",
+    "with suppliers", "with products",
+    "who placed", "who made", "who created", "who submitted",
+]
+
+
+# Explicit physical JOIN keywords, each mapped to its structured kind.
+# Matched as whole words / substrings against the full text — never as
+# "the first token of the sentence" (the legacy split()[0] bug).
+_JOIN_KIND_KEYWORDS: Dict[str, str] = {
+    "left join": "LEFT",
+    "右连接": "RIGHT",
+    "right join": "RIGHT",
+    "inner join": "INNER",
+    "内连接": "INNER",
+    "outer join": "OUTER",
+    "cross join": "CROSS",
+}
+
+
+@dataclass(frozen=True)
+class JoinSpec:
+    """Structured physical-join intent parsed from natural language.
+
+    ``kind`` is one of ``"LEFT" | "RIGHT" | "INNER" | "OUTER" | "CROSS"
+    | "JOIN"`` (JOIN = unqualified, defaults to inner). ``matched_text``
+    records which keyword fired, for evidence.
+    """
+
+    kind: str
+    matched_text: str
+
+
+def parse_join_spec(text: str) -> Optional[JoinSpec]:
+    """Parse an explicit physical-join keyword into a structured JoinSpec.
+
+    Returns ``None`` when the text carries no physical-join intent
+    (e.g. relational-existence phrases like "users who have orders"), in
+    which case the caller falls back to its EXISTS/JOIN inference.
+
+    Keywords are located anywhere in the text via whole-phrase
+    matching — the join type is never inferred from the first word of the
+    sentence.
+    """
+    lowered = text.lower()
+    for keyword, kind in _JOIN_KIND_KEYWORDS.items():
+        if keyword.isascii():
+            # English: whole-word, case-insensitive.
+            if re.search(r"\b" + re.escape(keyword) + r"\b", lowered):
+                return JoinSpec(kind=kind, matched_text=keyword)
+        else:
+            # Chinese (or any non-ASCII keyword): direct substring.
+            if keyword in text:
+                return JoinSpec(kind=kind, matched_text=keyword)
+    # Unqualified physical "join" that is not part of a known qualified
+    # keyword — default to inner.
+    if re.search(r"\bjoin\b", lowered):
+        return JoinSpec(kind="JOIN", matched_text="join")
+    return None
+
+
+def guess_join_key(table1: str, table2: str) -> Optional[str]:
+    """Return the canonical join condition between two tables, or None.
+
+    None means the relationship is unknown; the caller must not
+    fabricate a condition.
+    """
+    return JOIN_KEY_PATTERNS.get((table1, table2))
+
+
+def extract_joins(
+    text: str,
+    table_patterns: Dict[str, str],
+) -> List[Dict]:
+    """Extract structured join contexts from text.
+
+    Detects relational existence patterns (e.g. "users who have orders")
+    and returns one dict per additional table, keyed as:
+
+        type      - "EXISTS", "NOT EXISTS" or "JOIN"
+        table     - the related table name
+        subject   - the main (first-appearing) table name
+        condition - join key string, or None for unknown relationships
+        relational - True when type is EXISTS/NOT EXISTS
+    """
+    joins = []
+    # Collect unique tables in text-appearance order (first occurrence wins).
+    # Exclude "order" when it is part of an "order by" sorting clause.
+    table_occurrences = []
+    for pattern, table in table_patterns.items():
+        if pattern == "order" and re.search(r"\border\b\s+by\b", text, re.IGNORECASE):
+            continue
+        if re.search(r"\b" + re.escape(pattern) + r"\b", text) and table not in [t for _, t in table_occurrences]:
+            table_occurrences.append((text.find(pattern), table))
+    tables_found = [t for _, t in sorted(table_occurrences)]
+    if len(tables_found) >= 2:
+        join_spec = parse_join_spec(text)
+        if any(k in text for k in _NOT_EXISTS_KEYWORDS):
+            join_type = "NOT EXISTS"
+            relational = True
+        elif any(k in text for k in _EXISTS_KEYWORDS):
+            join_type = "EXISTS"
+            relational = True
+        elif join_spec is not None:
+            # Structured physical-join intent.  Unqualified "JOIN" normalizes
+            # to inner; qualified kinds map directly to their SQL keyword.
+            join_type = "INNER" if join_spec.kind == "JOIN" else join_spec.kind
+            relational = False
+        else:
+            join_type = "JOIN"
+            relational = False
+
+        main_table = tables_found[0]
+        for other_table in tables_found[1:]:
+            join_key = guess_join_key(main_table, other_table)
+            record = {
+                "type": join_type,
+                "table": other_table,
+                "subject": main_table,
+                "condition": join_key,
+                "relational": relational,
+                "join_spec": join_spec,
+            }
+            if join_key is None:
+                # Unknown relationship - do not fabricate a condition.
+                # Carry both sides of the pair so downstream fail-closed
+                # messages can name the actual unresolved relationship.
+                joins.append(record)
+                continue
+            joins.append(record)
+    return joins
+
+
+def has_relational_keyword(text: str) -> bool:
+    """Check if text contains any relational existence keyword."""
+    all_keywords = _NOT_EXISTS_KEYWORDS + _EXISTS_KEYWORDS
+    return any(pattern in text for pattern in all_keywords)

backend/core/nl2sql_components/templates.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Query Template module for NL2SQL.
+
+Provides pattern-based SQL generation using prioritized templates.
+"""
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any, Callable
+
+logger = logging.getLogger(__name__)
+
+
+# Clause markers that mean a single-purpose or catch-all template may truncate
+# part of the user's intent. Composed queries must flow to enhanced parsing.
+_COMPOSITION_MARKERS = (
+    "order by", "sort", "排序", "排列",
+    "group by", "grouped by", "分组", "汇总",
+    "limit", "top ", "top\t", "only ", "前",
+    " and ", " or ", "且", "并且", "或者", "或",
+)
+
+_TEMPLATE_BLOCKERS = {
+    "condition_query": _COMPOSITION_MARKERS,
+    "aggregate_query": _COMPOSITION_MARKERS,
+    "time_range_query": _COMPOSITION_MARKERS,
+    "select_with_columns": _COMPOSITION_MARKERS,
+    "simple_select": _COMPOSITION_MARKERS,
+}
+
+# Keep template captures bounded. These patterns run against natural-language
+# query text, so unbounded adjacent wildcards can cause excessive backtracking
+# on adversarial input. 512 characters is ample for each individual template
+# field while making the regex work bounded by construction.
+_TEMPLATE_CAPTURE_LIMIT = 512
+
+
+@dataclass
+class QueryTemplate:
+    """Template for pattern-based SQL generation with priority.
+
+    Templates are matched in priority order (highest first).
+    Each template has a regex pattern and a SQL template with placeholders.
+    """
+    name: str
+    pattern: str
+    sql_template: str
+    priority: int = 50
+    extractor: Optional[Callable] = None
+    _compiled_pattern: re.Pattern = field(init=False, repr=False, default=None)
+
+    def __post_init__(self):
+        """Pre-compile regex pattern for better performance."""
+        try:
+            self._compiled_pattern = re.compile(self.pattern, re.IGNORECASE)
+        except re.error as e:
+            logger.warning(f"Failed to compile pattern for template {self.name}: {e}")
+            self._compiled_pattern = None
+
+    def match(self, text: str) -> Optional[Dict[str, Any]]:
+        """Try to match this template against input text."""
+        normalized = text.lower()
+        blockers = _TEMPLATE_BLOCKERS.get(self.name, ())
+        if any(marker in normalized for marker in blockers):
+            logger.debug(
+                "Skipping template '%s' because composition markers require enhanced parsing",
+                self.name,
+            )
+            return None
+
+        if self._compiled_pattern is None:
+            match = re.search(self.pattern, text, re.IGNORECASE)
+        else:
+            match = self._compiled_pattern.search(text)
+
+        if match:
+            return match.groupdict() if match.groupdict() else {"match": match.group(0)}
+        return None
+
+
+DEFAULT_QUERY_TEMPLATES = [
+    QueryTemplate(
+        name="top_n_query",
+        pattern=rf"(?:查询|获取|get|show|find)?\s*(?:前|top|bottom|最低|最少|lowest|smallest)\s*(\d+)\s*(?:个|条|名)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)(?:按|by)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}})?\s*(?:排序|排列|order)?",
+        sql_template="SELECT * FROM {table} ORDER BY {order_col} DESC LIMIT {n}",
+        priority=90,
+    ),
+    QueryTemplate(
+        name="count_by_group",
+        pattern=rf"(?:统计|计算|count)\s*(?:每个|各个|each)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)\s*(?:的|的数量|数量|有多少)",
+        sql_template="SELECT {group_col}, COUNT(*) AS count FROM {table} GROUP BY {group_col}",
+        priority=85,
+    ),
+    QueryTemplate(
+        name="time_range_query",
+        pattern=r"(?:查询|获取|get|find)?\s*(?:最近|过去|last|past)\s*(\d+)\s*(天|周|月|年|days?|weeks?|months?|years?)\s*(?:的|内的)?\s*(.+)",
+        sql_template="SELECT * FROM {table} WHERE {date_col} >= DATE_SUB(CURRENT_DATE, {interval})",
+        priority=85,
+    ),
+    QueryTemplate(
+        name="aggregate_query",
+        pattern=rf"(?:计算|求|get)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)\s*(?:的)?\s*(平均|总和|最大|最小|average|sum|max|min)\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}})",
+        sql_template="SELECT {agg_func}({col}) FROM {table}",
+        priority=80,
+    ),
+    QueryTemplate(
+        name="condition_query",
+        pattern=rf"(?:查询|获取|find|get)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)\s*(大于|小于|等于|超过|不等于|greater(?:\s+than)?|less(?:\s+than)?|equal(?:\s+to)?|>|<|=)\s*(\d+\.?\d*)\s*(?:的)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}})?",
+        sql_template="SELECT * FROM {table} WHERE {col} {op} {value}",
+        priority=75,
+    ),
+    QueryTemplate(
+        name="join_query",
+        pattern=rf"(?:查询|获取)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)\s*(?:和|与|关联|连接|join)\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)(?:的|数据)?",
+        sql_template="SELECT * FROM {table1} JOIN {table2} ON {join_condition}",
+        priority=70,
+    ),
+    QueryTemplate(
+        name="select_with_columns",
+        pattern=rf"(?:查询|获取|显示|select|get|show)\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)\s*(?:的|from)?\s*(.{{1,{_TEMPLATE_CAPTURE_LIMIT}}}?)(?:表|table)?$",
+        sql_template="SELECT {columns} FROM {table}",
+        priority=60,
+    ),
+    QueryTemplate(
+        name="simple_select",
+        pattern=r"(?:查询|获取|显示|列出|select|get|show|list|find)\s*(?:所有|全部|all)?\s*(.+)",
+        sql_template="SELECT * FROM {table}",
+        priority=40,
+    ),
+]