sql-dialect-master 1.1.1__py3-none-any.whl

backend/core/nl2sql_components/boolean_conditions.py

@@ -0,0 +1,279 @@
+"""Boolean condition extraction for NL2SQL.
+
+Extract explicit comparison, range, text, null, status, and set-membership
+predicates while preserving normal SQL AND-over-OR precedence and
+user-supplied parentheses.
+"""
+import re
+from typing import List, Optional, Tuple
+
+
+_COMPARISON_PATTERNS = (
+    re.compile(
+        r"(?<![A-Za-z0-9_])(price|quantity|amount|age|score|rating|views|clicks)\s+"
+        r"(greater(?:\s+than)?|more\s+than|above|over|less(?:\s+than)?|"
+        r"below|under|equal(?:\s+to)?|equals)\s+"
+        r"([0-9]+(?:\.[0-9]+)?)",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"(?<![A-Za-z0-9_])(price|quantity|amount|age|score|rating|views|clicks)\s*"
+        r"(>=|<=|!=|=|>|<)\s*([0-9]+(?:\.[0-9]+)?)",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"(价格|数量|金额|年龄|得分|评分|浏览量|点击量)\s*"
+        r"(大于等于|小于等于|不等于|大于|小于|超过|低于|等于)\s*"
+        r"([0-9]+(?:\.[0-9]+)?)"
+    ),
+)
+
+_RANGE_PATTERN = re.compile(
+    r"(?<![A-Za-z0-9_])(price|quantity|amount|age|score|rating|views|clicks)\s+"
+    r"(?:between\s+([0-9]+(?:\.[0-9]+)?)\s+and\s+([0-9]+(?:\.[0-9]+)?)|"
+    r"in\s+(?:the\s+)?range\s+([0-9]+(?:\.[0-9]+)?)\s*(?:to|-)\s*"
+    r"([0-9]+(?:\.[0-9]+)?))",
+    re.IGNORECASE,
+)
+
+_CN_RANGE_PATTERN = re.compile(
+    r"(价格|数量|金额|年龄|得分|评分|浏览量|点击量)\s*"
+    r"(?:在|介于)\s*([0-9]+(?:\.[0-9]+)?)\s*(?:和|到|至|-)\s*"
+    r"([0-9]+(?:\.[0-9]+)?)\s*(?:之间|范围内|以内)?"
+)
+
+_TEXT_PATTERN = re.compile(
+    r"(?<![A-Za-z0-9_])(name|title|description|category|brand)\s+"
+    r"(?:contains|containing|includes|including)\s+"
+    r"(?:the\s+)?[\"']?([^\"']+?)[\"']?(?=\s+(?:and|or)\s+|$)",
+    re.IGNORECASE,
+)
+
+_CN_TEXT_PATTERN = re.compile(
+    r"(名称|名字|标题|描述|分类|品牌)\s*(?:包含|含有|包括)\s*[“\"]?([^”\"]+?)[”\"]?(?=(?:且|并且|或者|或|和)|$)"
+)
+
+_NULL_PATTERN = re.compile(
+    r"(?<![A-Za-z0-9_])(name|title|description|category|brand|price|quantity|amount|"
+    r"age|score|rating|views|clicks)\s+(is\s+(?:not\s+)?null)\b",
+    re.IGNORECASE,
+)
+
+_CN_NULL_PATTERN = re.compile(
+    r"(名称|名字|标题|描述|分类|品牌|价格|数量|金额|年龄|得分|评分|浏览量|点击量)\s*"
+    r"(为空|为\s*空|是空|不为空|不为\s*空|非空)"
+)
+
+_SET_PREFIX_PATTERN = re.compile(
+    r"(?<![A-Za-z0-9_])(category|type|department|region|status)\s+"
+    r"(?:(not)\s+)?in\s*\(",
+    re.IGNORECASE,
+)
+
+_CN_SET_PREFIX_PATTERN = re.compile(
+    r"(类别|类型|部门|地区|状态)\s*(不在|在)\s*"
+)
+
+_STATUS_PATTERN = re.compile(
+    r"(?:\bstatus\s+|状态\s*(?:为|是|=)\s*)"
+    r"(active|enabled|valid|inactive|disabled|invalid|completed|done|pending|paid|unpaid|"
+    r"有效|无效|已完成|未完成|已支付|未支付)",
+    re.IGNORECASE,
+)
+
+_STATUS_CONDITIONS = {
+    "active": "status = 'active'",
+    "enabled": "status = 'active'",
+    "valid": "status = 'active'",
+    "有效": "status = 'active'",
+    "inactive": "status = 'inactive'",
+    "disabled": "status = 'inactive'",
+    "invalid": "status = 'inactive'",
+    "无效": "status = 'inactive'",
+    "completed": "status = 'completed'",
+    "done": "status = 'completed'",
+    "已完成": "status = 'completed'",
+    "pending": "status = 'pending'",
+    "未完成": "status = 'pending'",
+    "paid": "status = 'paid'",
+    "已支付": "status = 'paid'",
+    "unpaid": "status = 'unpaid'",
+    "未支付": "status = 'unpaid'",
+}
+
+_CN_COLUMNS = {
+    "价格": "price",
+    "数量": "quantity",
+    "金额": "amount",
+    "年龄": "age",
+    "得分": "score",
+    "评分": "rating",
+    "浏览量": "views",
+    "点击量": "clicks",
+    "名称": "name",
+    "名字": "name",
+    "标题": "title",
+    "描述": "description",
+    "分类": "category",
+    "品牌": "brand",
+}
+
+_CN_SET_COLUMNS = {
+    "类别": "category",
+    "类型": "type",
+    "部门": "department",
+    "地区": "region",
+    "状态": "status",
+}
+
+_CN_OPERATORS = {
+    "大于等于": ">=",
+    "小于等于": "<=",
+    "不等于": "!=",
+    "大于": ">",
+    "小于": "<",
+    "超过": ">",
+    "低于": "<",
+    "等于": "=",
+}
+
+_CONNECTOR_PATTERN = re.compile(r"\b(?:and|or)\b|以及|并且|且|或者|或|和", re.IGNORECASE)
+
+
+def _normalize_operator(raw: str) -> str:
+    normalized = raw.lower()
+    if normalized.startswith(("greater", "more", "above", "over")):
+        return ">"
+    if normalized.startswith(("less", "below", "under")):
+        return "<"
+    if normalized.startswith(("equal", "equals")):
+        return "="
+    return _CN_OPERATORS.get(raw, normalized.upper())
+
+
+def _text_sql(column: str, value: str) -> str:
+    escaped = value.strip().replace("'", "''")
+    return f"{column} LIKE '%{escaped}%'"
+
+
+def _format_set_values(raw_values: str) -> str:
+    values = []
+    for raw in re.split(r"[,，]", raw_values):
+        value = raw.strip().strip("\"'")
+        if not value:
+            continue
+        values.append("'" + value.replace("'", "''") + "'")
+    return ", ".join(values)
+
+
+def _comparison_matches(text: str) -> List[Tuple[int, int, str]]:
+    matches: List[Tuple[int, int, str]] = []
+
+    for pattern in _COMPARISON_PATTERNS:
+        for match in pattern.finditer(text):
+            first, raw_operator, value = match.groups()
+            column = _CN_COLUMNS.get(first, first)
+            matches.append((match.start(), match.end(), f"{column} {_normalize_operator(raw_operator)} {value}"))
+
+    for pattern in (_RANGE_PATTERN, _CN_RANGE_PATTERN):
+        for match in pattern.finditer(text):
+            groups = match.groups()
+            if len(groups) == 5:
+                first, low1, high1, low2, high2 = groups
+                low, high = low1 or low2, high1 or high2
+            else:
+                first, low, high = groups
+            column = _CN_COLUMNS.get(first, first)
+            matches.append((match.start(), match.end(), f"{column} BETWEEN {low} AND {high}"))
+
+    for pattern in (_TEXT_PATTERN, _CN_TEXT_PATTERN):
+        for match in pattern.finditer(text):
+            first, value = match.groups()
+            column = _CN_COLUMNS.get(first, first)
+            matches.append((match.start(), match.end(), _text_sql(column, value)))
+
+    for pattern in (_NULL_PATTERN, _CN_NULL_PATTERN):
+        for match in pattern.finditer(text):
+            first, raw_predicate = match.groups()
+            column = _CN_COLUMNS.get(first, first)
+            normalized = raw_predicate.replace(" ", "").lower()
+            is_not_null = normalized.startswith(("isnotnull", "不为空", "非空", "不为"))
+            matches.append((match.start(), match.end(), f"{column} IS {'NOT ' if is_not_null else ''}NULL"))
+
+    for match in _SET_PREFIX_PATTERN.finditer(text):
+        close_paren = text.find(")", match.end())
+        if close_paren == -1:
+            continue
+        column, negation = match.groups()
+        raw_values = text[match.end() : close_paren]
+        values = _format_set_values(raw_values)
+        if values:
+            operator = "NOT IN" if negation else "IN"
+            matches.append((match.start(), close_paren + 1, f"{column.lower()} {operator} ({values})"))
+
+    for match in _CN_SET_PREFIX_PATTERN.finditer(text):
+        column, operator_text = match.groups()
+        value_end = len(text)
+        connector = _CONNECTOR_PATTERN.search(text, match.end())
+        if connector is not None:
+            value_end = connector.start()
+        raw_values = text[match.end() : value_end].strip()
+        for suffix in ("之中", "其中", "中"):
+            if raw_values.endswith(suffix):
+                raw_values = raw_values[: -len(suffix)].rstrip()
+                break
+        if not raw_values or any(char in raw_values for char in "()"):
+            continue
+        values = _format_set_values(raw_values)
+        if values:
+            operator = "NOT IN" if operator_text == "不在" else "IN"
+            matches.append((match.start(), value_end, f"{_CN_SET_COLUMNS[column]} {operator} ({values})"))
+
+    for match in _STATUS_PATTERN.finditer(text):
+        status = match.group(1).lower()
+        matches.append((match.start(), match.end(), _STATUS_CONDITIONS[status]))
+
+    matches.sort(key=lambda item: (item[0], item[1]))
+    deduped: List[Tuple[int, int, str]] = []
+    for item in matches:
+        if deduped and item[0] == deduped[-1][0] and item[1] <= deduped[-1][1]:
+            continue
+        deduped.append(item)
+    return deduped
+
+
+def extract_boolean_conditions(text: str) -> Optional[List[str]]:
+    """Return one boolean SQL expression preserving connector precedence/grouping."""
+    matches = _comparison_matches(text)
+    if len(matches) < 2:
+        return None
+
+    tokens: List[str] = []
+    prefix = text[: matches[0][0]]
+    tokens.extend(char for char in prefix if char in "()")
+
+    for index, current in enumerate(matches):
+        tokens.append(f"({current[2]})")
+        if index == len(matches) - 1:
+            suffix = text[current[1] :]
+            tokens.extend(char for char in suffix if char in "()")
+            continue
+
+        separator = text[current[1] : matches[index + 1][0]]
+        events = []
+        for paren in re.finditer(r"[()]", separator):
+            events.append((paren.start(), paren.group(0)))
+        connector_matches = list(_CONNECTOR_PATTERN.finditer(separator))
+        if len(connector_matches) != 1:
+            return None
+        connector = connector_matches[0]
+        events.append((connector.start(), connector.group(0)))
+        events.sort(key=lambda item: item[0])
+        for _, event in events:
+            if event == "(" or event == ")":
+                tokens.append(event)
+            elif event:
+                normalized = event.lower()
+                tokens.append("OR" if normalized in {"or", "或者", "或"} else "AND")
+
+    return [" ".join(tokens)]

backend/core/nl2sql_components/core_extract.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Core NL2SQL extraction primitives: operation, table, column, limit, distinct.
+
+These are stateless, dependency-free functions that operate on raw text
+and the shared mapping dicts (KEYWORDS, TABLE_PATTERNS, COLUMN_PATTERNS
+from ``backend.core.nl2sql_components.mappings``).
+"""
+import re
+from typing import Dict, List, Optional
+
+
+def detect_operation(text: str, keywords: Dict[str, str]) -> str:
+    """Detect SQL operation type from text.
+
+    Scans the keyword mapping for operation keywords present in the text
+    and returns the first matching operation (SELECT/INSERT/UPDATE/DELETE).
+    Defaults to SELECT.
+    """
+    for keyword, op in keywords.items():
+        if keyword in text and op in ["SELECT", "INSERT", "UPDATE", "DELETE"]:
+            return op
+    return "SELECT"
+
+
+def extract_table(text: str, table_patterns: Dict[str, str]) -> str:
+    """Extract a single table name from text.
+
+    Prefers the longest matching pattern; ties broken by first position.
+    Returns the fallback name "table_name" when no pattern matches.
+    """
+    tables_found = []
+    for pattern, table in table_patterns.items():
+        if pattern in text:
+            tables_found.append((text.find(pattern), table, len(pattern)))
+    if tables_found:
+        tables_found.sort(key=lambda item: (-item[2], item[0]))
+        return tables_found[0][1]
+    return "table_name"
+
+
+def extract_table_subject(text: str, table_patterns: Dict[str, str]) -> Optional[str]:
+    """Extract the subject table (first appearing in text) for D2 relational queries.
+
+    For queries like 'users who have orders', returns 'users' (the subject)
+    rather than 'orders' (the relation). Uses first occurrence position.
+    """
+    candidates = []
+    for pattern, table in table_patterns.items():
+        if table not in candidates:
+            pos = text.find(pattern)
+            if pos != -1:
+                candidates.append((pos, table))
+    if candidates:
+        candidates.sort(key=lambda x: x[0])
+        return candidates[0][1]
+    return None
+
+
+def named_tables(text: str, table_patterns: Dict[str, str]) -> List[str]:
+    """Return the distinct table names the text mentions, in order of first appearance."""
+    occurrences = []
+    seen = set()
+    for pattern, table in table_patterns.items():
+        if table in seen:
+            continue
+        pos = text.find(pattern)
+        if pos != -1:
+            occurrences.append((pos, table))
+            seen.add(table)
+    return [t for _, t in sorted(occurrences)]
+
+
+def extract_columns(text: str, column_patterns: Dict[str, str]) -> List[str]:
+    """Extract column names from text.
+
+    Returns a list of distinct column names, or ["*"] when no column
+    keyword is present.
+    """
+    columns = []
+    for pattern, column in column_patterns.items():
+        if pattern in text and column not in columns:
+            columns.append(column)
+    return columns if columns else ["*"]
+
+
+def extract_limit(text: str) -> Optional[int]:
+    """Extract LIMIT value from text.
+
+    Supports Chinese and English cardinality phrasings such as
+    '前10', 'top 10', 'limit 10', '10条', '10行'.
+    """
+    patterns = [
+        r"前\s*(\d+)", r"top\s*(\d+)", r"limit\s*(\d+)",
+        r"first\s*(\d+)", r"(\d+)\s*条", r"(\d+)\s*个",
+        r"(\d+)\s*行", r"(\d+)\s*rows?", r"only\s*(\d+)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            return int(match.group(1))
+    return None
+
+
+def check_distinct(text: str) -> bool:
+    """Check if DISTINCT is requested in the text."""
+    return any(k in text for k in ["去重", "唯一", "不重复", "distinct", "unique"])

backend/core/nl2sql_components/evidence.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Evidence-based confidence model for NL2SQL generation.
+
+Replaces opaque ``confidence += 0.1`` heuristic increments with a
+structured list of named evidence items.  Each item records:
+
+    evidence_id  - stable identifier for the evidence source
+    label        - human-readable description
+    weight       - contribution to the confidence score (signed)
+
+The confidence score is the sum of all item weights, clamped to [0, 1],
+exactly matching the previous heuristic's numeric output for the
+same inputs.  Consumers can inspect the items list to understand
+*why* a result was given its score.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+
+@dataclass
+class EvidenceItem:
+    """A single evidence item contributing to the confidence score.
+
+    Attributes:
+        evidence_id: Stable identifier, e.g. "table_known" or "parse_ok".
+        label:      Human-readable description shown to the user.
+        weight:      Signed contribution to the confidence score.
+        detail:      Optional machine-readable context (table name, parse error, etc.).
+    """
+    evidence_id: str
+    label: str
+    weight: float
+    detail: Optional[str] = None
+
+
+@dataclass
+class GenerationEvidence:
+    """Structured evidence backing a generated SQL confidence score.
+
+    Holds the full list of EvidenceItems and a computed score.
+    The score is the sum of all weights, clamped to [0.0, 1.0],
+    matching the previous heuristic confidence exactly.
+
+    Use ``GenerationEvidence.score`` as the ``confidence`` value and
+    pass ``GenerationEvidence.items`` as the evidence trail.
+    """
+    items: List[EvidenceItem] = field(default_factory=list)
+
+    @property
+    def score(self) -> float:
+        """Clamped sum of all item weights (matches legacy heuristic range)."""
+        return max(0.0, min(1.0, sum(i.weight for i in self.items)))
+
+    def to_dict(self) -> dict:
+        return {
+            "score": self.score,
+            "items": [
+                {
+                    "evidence_id": i.evidence_id,
+                    "label": i.label,
+                    "weight": i.weight,
+                    "detail": i.detail,
+                }
+                for i in self.items
+            ],
+        }
+
+
+# ── Evidence ID constants ──────────────────────────────────────────────
+# Use these instead of raw string literals so the IDs are stable
+# across refactors and can be grepped.
+
+# Table / schema evidence
+EVIDENCE_TABLE_KNOWN = "table_known"
+EVIDENCE_TABLE_UNKNOWN = "table_unknown"
+
+# Column / aggregate evidence
+EVIDENCE_COLUMNS_EXPLICIT = "columns_explicit"
+EVIDENCE_COLUMNS_WILDCARD = "columns_wildcard"
+
+# Predicate / condition evidence
+EVIDENCE_CONDITIONS_PRESENT = "conditions_present"
+
+# Relationship / join evidence
+EVIDENCE_JOIN_KNOWN = "join_known"
+EVIDENCE_JOIN_UNKNOWN = "join_unknown"
+
+# Aggregation evidence
+EVIDENCE_AGGREGATION_PRESENT = "aggregation_present"
+
+# Ordering / limit / group-by evidence
+EVIDENCE_GROUP_BY = "group_by"
+EVIDENCE_ORDERING = "ordering"
+EVIDENCE_LIMIT = "limit"
+
+# Generated SQL structural evidence
+EVIDENCE_PARSE_OK = "target_parse_ok"
+EVIDENCE_PARSE_FAIL = "target_parse_fail"
+
+# Template evidence
+EVIDENCE_TEMPLATE_MATCHED = "template_matched"
+
+__all__ = [
+    "EvidenceItem",
+    "GenerationEvidence",
+    "EVIDENCE_TABLE_KNOWN",
+    "EVIDENCE_TABLE_UNKNOWN",
+    "EVIDENCE_COLUMNS_EXPLICIT",
+    "EVIDENCE_COLUMNS_WILDCARD",
+    "EVIDENCE_CONDITIONS_PRESENT",
+    "EVIDENCE_JOIN_KNOWN",
+    "EVIDENCE_JOIN_UNKNOWN",
+    "EVIDENCE_AGGREGATION_PRESENT",
+    "EVIDENCE_GROUP_BY",
+    "EVIDENCE_ORDERING",
+    "EVIDENCE_LIMIT",
+    "EVIDENCE_PARSE_OK",
+    "EVIDENCE_PARSE_FAIL",
+    "EVIDENCE_TEMPLATE_MATCHED",
+]