sql-dialect-master 1.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backend/__init__.py +1 -0
- backend/api/__init__.py +1 -0
- backend/api/main.py +732 -0
- backend/api/middleware.py +274 -0
- backend/api/rate_limit_store.py +105 -0
- backend/api/readiness.py +178 -0
- backend/core/__init__.py +57 -0
- backend/core/cache.py +261 -0
- backend/core/column_hint_validation.py +22 -0
- backend/core/config.py +546 -0
- backend/core/constants.py +84 -0
- backend/core/exceptions.py +124 -0
- backend/core/function_call_scanner.py +64 -0
- backend/core/function_semantics.py +148 -0
- backend/core/functions_db.json +601 -0
- backend/core/functions_lookup.py +332 -0
- backend/core/metadata.py +71 -0
- backend/core/nl2sql.py +85 -0
- backend/core/nl2sql_components/__init__.py +72 -0
- backend/core/nl2sql_components/aggregations.py +35 -0
- backend/core/nl2sql_components/boolean_conditions.py +279 -0
- backend/core/nl2sql_components/core_extract.py +106 -0
- backend/core/nl2sql_components/evidence.py +122 -0
- backend/core/nl2sql_components/mappings.py +185 -0
- backend/core/nl2sql_components/relations.py +180 -0
- backend/core/nl2sql_components/templates.py +130 -0
- backend/core/nl2sql_components/tokenizer.py +187 -0
- backend/core/nl2sql_legacy.py +22 -0
- backend/core/nl2sql_pipeline/__init__.py +13 -0
- backend/core/nl2sql_pipeline/aggregations.py +26 -0
- backend/core/nl2sql_pipeline/clause_extract.py +97 -0
- backend/core/nl2sql_pipeline/dialect.py +139 -0
- backend/core/nl2sql_pipeline/intent.py +85 -0
- backend/core/nl2sql_pipeline/ir.py +70 -0
- backend/core/nl2sql_pipeline/models.py +30 -0
- backend/core/nl2sql_pipeline/orchestrator.py +576 -0
- backend/core/nl2sql_pipeline/predicates.py +182 -0
- backend/core/nl2sql_pipeline/renderer.py +248 -0
- backend/core/nl2sql_pipeline/resolution.py +69 -0
- backend/core/nl2sql_pipeline/suggestions.py +30 -0
- backend/core/nl2sql_pipeline/templates.py +371 -0
- backend/core/nl2sql_pipeline/validation.py +28 -0
- backend/core/p1_sql_scanner.py +196 -0
- backend/core/p2_data_validation.py +72 -0
- backend/core/parser.py +280 -0
- backend/core/post_processor.py +129 -0
- backend/core/rules.py +808 -0
- backend/core/semantic_diff.py +385 -0
- backend/core/transpiler.py +791 -0
- backend/core/type_mapping.json +593 -0
- backend/core/type_mapping.py +765 -0
- backend/utils/__init__.py +2 -0
- backend/utils/validators.py +407 -0
- frontend/__init__.py +10 -0
- frontend/app_context_v2.py +201 -0
- frontend/core/__init__.py +33 -0
- frontend/core/design_tokens.py +127 -0
- frontend/core/escaping.py +13 -0
- frontend/core/navigation.py +135 -0
- frontend/core/state.py +612 -0
- frontend/core/styles.py +523 -0
- frontend/core/themes.py +69 -0
- frontend/core/viewmodels.py +499 -0
- frontend/pages/__init__.py +0 -0
- frontend/pages/convert.py +283 -0
- frontend/pages/diff.py +157 -0
- frontend/pages/functions.py +100 -0
- frontend/pages/history.py +144 -0
- frontend/pages/lineage.py +168 -0
- frontend/pages/nl2sql.py +305 -0
- frontend/pages/query_analysis.py +120 -0
- frontend/pages/runtime.py +66 -0
- frontend/pages/settings.py +60 -0
- frontend/pages/templates.py +67 -0
- frontend/pages/types.py +110 -0
- frontend/templates_v2.py +53 -0
- frontend/ui/__init__.py +0 -0
- frontend/ui/command_palette.py +153 -0
- frontend/ui/diff.py +275 -0
- frontend/ui/editor.py +102 -0
- frontend/ui/status.py +255 -0
- sql_dialect_master-1.1.1.dist-info/METADATA +360 -0
- sql_dialect_master-1.1.1.dist-info/RECORD +86 -0
- sql_dialect_master-1.1.1.dist-info/WHEEL +5 -0
- sql_dialect_master-1.1.1.dist-info/licenses/LICENSE +21 -0
- sql_dialect_master-1.1.1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Tokenizer module for NL2SQL.
|
|
3
|
+
|
|
4
|
+
Provides smart tokenization with Chinese and English support,
|
|
5
|
+
optimized for natural language SQL generation.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
import logging
|
|
9
|
+
from typing import List
|
|
10
|
+
|
|
11
|
+
# Configure module logger
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
# Try to import jieba for better Chinese tokenization
|
|
15
|
+
try:
|
|
16
|
+
import jieba
|
|
17
|
+
jieba.setLogLevel(logging.WARNING) # Suppress jieba logs
|
|
18
|
+
JIEBA_AVAILABLE = True
|
|
19
|
+
logger.info("jieba tokenizer available for Chinese NL2SQL")
|
|
20
|
+
except ImportError:
|
|
21
|
+
JIEBA_AVAILABLE = False
|
|
22
|
+
logger.debug("jieba not installed, using basic tokenization for Chinese")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Tokenizer:
|
|
26
|
+
"""Smart tokenizer with Chinese and English support.
|
|
27
|
+
|
|
28
|
+
Performance: All static regex patterns are pre-compiled at class level.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
# Chinese punctuation to remove
|
|
32
|
+
CN_PUNCTUATION = r"[,。!?、;:\"'()【】《》]"
|
|
33
|
+
|
|
34
|
+
# Pre-compiled regex patterns for performance
|
|
35
|
+
_RE_CN_PUNCT = re.compile(CN_PUNCTUATION)
|
|
36
|
+
_RE_EN_PUNCT = re.compile(r'[,\.!?;:\'\"()\[\]{}]')
|
|
37
|
+
_RE_CHINESE_CHARS = re.compile(r'[\u4e00-\u9fff]')
|
|
38
|
+
_RE_LETTER = re.compile(r'[a-zA-Z]')
|
|
39
|
+
_RE_ALNUM = re.compile(r'[a-zA-Z0-9]')
|
|
40
|
+
_RE_DIGIT = re.compile(r'\d')
|
|
41
|
+
_RE_DIGIT_DOT = re.compile(r'[\d\.]')
|
|
42
|
+
_RE_NUMBERS = re.compile(r'\d+\.?\d*')
|
|
43
|
+
_RE_QUOTED = re.compile(r'["\']([^"\']+)["\']')
|
|
44
|
+
|
|
45
|
+
# Common Chinese compound words that should not be split
|
|
46
|
+
# These are keywords important for NL2SQL
|
|
47
|
+
CHINESE_COMPOUNDS = [
|
|
48
|
+
# Comparison operators
|
|
49
|
+
"大于", "小于", "等于", "不等于", "大于等于", "小于等于", "超过", "低于",
|
|
50
|
+
# Aggregations
|
|
51
|
+
"平均", "总和", "最大", "最小", "合计", "统计", "计数", "数量",
|
|
52
|
+
# Time
|
|
53
|
+
"最近", "过去", "今天", "昨天", "前天", "本周", "本月", "本年",
|
|
54
|
+
# Operations
|
|
55
|
+
"查询", "获取", "显示", "列出", "插入", "添加", "更新", "修改", "删除", "移除",
|
|
56
|
+
# Grouping/Sorting
|
|
57
|
+
"分组", "汇总", "排序", "排列", "升序", "降序", "去重",
|
|
58
|
+
# Tables
|
|
59
|
+
"用户", "订单", "产品", "商品", "员工", "部门", "客户", "日志",
|
|
60
|
+
# Columns
|
|
61
|
+
"姓名", "名字", "邮箱", "电话", "地址", "年龄", "性别", "金额", "价格", "数量",
|
|
62
|
+
"状态", "类型", "时间", "日期", "创建时间", "更新时间",
|
|
63
|
+
# Joins
|
|
64
|
+
"关联", "连接", "左连接", "右连接", "内连接",
|
|
65
|
+
# Misc
|
|
66
|
+
"所有", "全部", "每个", "各个",
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def tokenize(text: str) -> List[str]:
|
|
71
|
+
"""Tokenize text into words/tokens.
|
|
72
|
+
|
|
73
|
+
Uses jieba for Chinese if available, otherwise falls back to
|
|
74
|
+
compound-word aware splitting for Chinese and word-based for English.
|
|
75
|
+
"""
|
|
76
|
+
# Clean punctuation using pre-compiled patterns
|
|
77
|
+
text = Tokenizer._RE_CN_PUNCT.sub(' ', text)
|
|
78
|
+
text = Tokenizer._RE_EN_PUNCT.sub(' ', text)
|
|
79
|
+
|
|
80
|
+
# Check if text contains Chinese characters
|
|
81
|
+
has_chinese = bool(Tokenizer._RE_CHINESE_CHARS.search(text))
|
|
82
|
+
|
|
83
|
+
if has_chinese and JIEBA_AVAILABLE:
|
|
84
|
+
# Use jieba for Chinese tokenization
|
|
85
|
+
tokens = list(jieba.cut(text, cut_all=False))
|
|
86
|
+
elif has_chinese:
|
|
87
|
+
# Fallback: compound-word aware tokenization
|
|
88
|
+
tokens = Tokenizer._tokenize_chinese_fallback(text)
|
|
89
|
+
else:
|
|
90
|
+
# English: simple word tokenization
|
|
91
|
+
tokens = text.lower().split()
|
|
92
|
+
|
|
93
|
+
# Filter empty tokens
|
|
94
|
+
return [t.strip() for t in tokens if t.strip()]
|
|
95
|
+
|
|
96
|
+
@staticmethod
|
|
97
|
+
def _tokenize_chinese_fallback(text: str) -> List[str]:
|
|
98
|
+
"""Fallback Chinese tokenization that preserves compound words.
|
|
99
|
+
|
|
100
|
+
Uses a greedy longest-match approach for known compound words.
|
|
101
|
+
"""
|
|
102
|
+
tokens = []
|
|
103
|
+
i = 0
|
|
104
|
+
text_len = len(text)
|
|
105
|
+
|
|
106
|
+
while i < text_len:
|
|
107
|
+
char = text[i]
|
|
108
|
+
|
|
109
|
+
# Skip whitespace
|
|
110
|
+
if char.isspace():
|
|
111
|
+
i += 1
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
# Check if this is the start of a known compound word
|
|
115
|
+
matched_compound = None
|
|
116
|
+
for compound in sorted(Tokenizer.CHINESE_COMPOUNDS, key=len, reverse=True):
|
|
117
|
+
if text[i:].startswith(compound):
|
|
118
|
+
matched_compound = compound
|
|
119
|
+
break
|
|
120
|
+
|
|
121
|
+
if matched_compound:
|
|
122
|
+
tokens.append(matched_compound)
|
|
123
|
+
i += len(matched_compound)
|
|
124
|
+
elif Tokenizer._RE_CHINESE_CHARS.match(char):
|
|
125
|
+
# Single Chinese character
|
|
126
|
+
tokens.append(char)
|
|
127
|
+
i += 1
|
|
128
|
+
elif Tokenizer._RE_LETTER.match(char):
|
|
129
|
+
# English word - collect until non-letter
|
|
130
|
+
word_start = i
|
|
131
|
+
while i < text_len and Tokenizer._RE_ALNUM.match(text[i]):
|
|
132
|
+
i += 1
|
|
133
|
+
tokens.append(text[word_start:i].lower())
|
|
134
|
+
elif Tokenizer._RE_DIGIT.match(char):
|
|
135
|
+
# Number - collect until non-digit
|
|
136
|
+
num_start = i
|
|
137
|
+
while i < text_len and Tokenizer._RE_DIGIT_DOT.match(text[i]):
|
|
138
|
+
i += 1
|
|
139
|
+
tokens.append(text[num_start:i])
|
|
140
|
+
else:
|
|
141
|
+
i += 1
|
|
142
|
+
|
|
143
|
+
return tokens
|
|
144
|
+
|
|
145
|
+
@staticmethod
|
|
146
|
+
def extract_numbers(text: str) -> List[str]:
|
|
147
|
+
"""Extract numbers, preserving a simple Chinese boolean comparison tail."""
|
|
148
|
+
numbers = Tokenizer._RE_NUMBERS.findall(text)
|
|
149
|
+
if "且" in text and len(numbers) >= 2:
|
|
150
|
+
boolean_tail = re.search(
|
|
151
|
+
r"且\s*(数量|价格|金额|年龄|数量)\s*(大于|超过|高于|多于|小于|低于|少于|不足|等于|不等于)\s*(\d+(?:\.\d+)?)",
|
|
152
|
+
text,
|
|
153
|
+
)
|
|
154
|
+
if boolean_tail:
|
|
155
|
+
column_map = {
|
|
156
|
+
"数量": "quantity",
|
|
157
|
+
"价格": "price",
|
|
158
|
+
"金额": "amount",
|
|
159
|
+
"年龄": "age",
|
|
160
|
+
}
|
|
161
|
+
operator_map = {
|
|
162
|
+
"大于": ">",
|
|
163
|
+
"超过": ">",
|
|
164
|
+
"高于": ">",
|
|
165
|
+
"多于": ">",
|
|
166
|
+
"小于": "<",
|
|
167
|
+
"低于": "<",
|
|
168
|
+
"少于": "<",
|
|
169
|
+
"不足": "<",
|
|
170
|
+
"等于": "=",
|
|
171
|
+
"不等于": "!=",
|
|
172
|
+
}
|
|
173
|
+
column = column_map[boolean_tail.group(1)]
|
|
174
|
+
operator = operator_map[boolean_tail.group(2)]
|
|
175
|
+
second_value = boolean_tail.group(3)
|
|
176
|
+
return [f"{numbers[0]} AND {column} {operator} {second_value}"] + numbers[1:]
|
|
177
|
+
return numbers
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def extract_quoted_strings(text: str) -> List[str]:
|
|
181
|
+
"""Extract quoted strings from text."""
|
|
182
|
+
return Tokenizer._RE_QUOTED.findall(text)
|
|
183
|
+
|
|
184
|
+
@staticmethod
|
|
185
|
+
def is_chinese(text: str) -> bool:
|
|
186
|
+
"""Check if text contains Chinese characters."""
|
|
187
|
+
return bool(Tokenizer._RE_CHINESE_CHARS.search(text))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""NL2SQL legacy module — P8 import-compatibility shim.
|
|
3
|
+
|
|
4
|
+
All implementation has moved to ``backend.core.nl2sql_pipeline``. This
|
|
5
|
+
module is a thin re-export shim kept so existing import sites
|
|
6
|
+
(``from backend.core.nl2sql_legacy import NL2SQLGenerator`` /
|
|
7
|
+
``NL2SQLResult``) continue to work unchanged. It is NOT deleted;
|
|
8
|
+
deletion happens in a future phase once every import/call site is
|
|
9
|
+
confirmed migrated.
|
|
10
|
+
|
|
11
|
+
``NL2SQLResult`` is re-exported from the single canonical source
|
|
12
|
+
(``nl2sql_pipeline.models``) so
|
|
13
|
+
``from backend.core.nl2sql import NL2SQLResult as A`` and
|
|
14
|
+
``from backend.core.nl2sql_legacy import NL2SQLResult as B`` satisfy
|
|
15
|
+
``A is B``.
|
|
16
|
+
"""
|
|
17
|
+
from .nl2sql_pipeline.models import NL2SQLResult
|
|
18
|
+
from .nl2sql_pipeline.orchestrator import (
|
|
19
|
+
PipelineNL2SQLGenerator as NL2SQLGenerator,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = ["NL2SQLGenerator", "NL2SQLResult"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""P8 NL2SQL pipeline: named-stage modules extracted from nl2sql_legacy.
|
|
2
|
+
|
|
3
|
+
Each module corresponds to one stage of the target pipeline
|
|
4
|
+
(Intent → Entity Resolution → Relation Resolution → Predicate Resolution
|
|
5
|
+
→ Query IR → Dialect Renderer → AST Validation → Evidence/Confidence).
|
|
6
|
+
|
|
7
|
+
This package is internal to ``backend.core``; it is NOT re-exported from
|
|
8
|
+
``backend.core.__init__``. The public facade is ``backend.core.nl2sql``.
|
|
9
|
+
"""
|
|
10
|
+
from .models import NL2SQLResult
|
|
11
|
+
from .orchestrator import PipelineNL2SQLGenerator
|
|
12
|
+
|
|
13
|
+
__all__ = ["NL2SQLResult", "PipelineNL2SQLGenerator"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: aggregation extraction (moved from nl2sql_legacy).
|
|
3
|
+
|
|
4
|
+
``extract_aggregations`` is a thin parameterized wrapper over
|
|
5
|
+
``nl2sql_components.aggregations.extract_aggregations`` — it fills the
|
|
6
|
+
canonical ``COLUMN_PATTERNS`` when none are supplied so the legacy
|
|
7
|
+
``self._extract_aggregations(text)`` call-shape works unchanged.
|
|
8
|
+
"""
|
|
9
|
+
from typing import Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
from ..nl2sql_components.aggregations import (
|
|
12
|
+
extract_aggregations as _extract_aggregations_impl,
|
|
13
|
+
)
|
|
14
|
+
from ..nl2sql_components.mappings import COLUMN_PATTERNS
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def extract_aggregations(
|
|
18
|
+
text: str, column_patterns: Optional[Dict[str, str]] = None
|
|
19
|
+
) -> List[str]:
|
|
20
|
+
"""Extract aggregation expressions from text.
|
|
21
|
+
|
|
22
|
+
Returns a list of SQL aggregate expressions, e.g.
|
|
23
|
+
``['COUNT(*)', 'SUM(amount)']``. An empty list means no
|
|
24
|
+
aggregation was requested.
|
|
25
|
+
"""
|
|
26
|
+
return _extract_aggregations_impl(text, column_patterns or COLUMN_PATTERNS)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: clause extraction (moved from nl2sql_legacy).
|
|
3
|
+
|
|
4
|
+
Holds the ``extract_group_by`` and ``extract_ordering`` stage functions
|
|
5
|
+
(verbatim moves of the legacy instance methods, parameterized on
|
|
6
|
+
``COLUMN_PATTERNS`` / ``TABLE_PATTERNS``). ``extract_limit`` and
|
|
7
|
+
``check_distinct`` are re-exports of the stateless ``core_extract``
|
|
8
|
+
primitives the legacy class already delegated to.
|
|
9
|
+
"""
|
|
10
|
+
import re
|
|
11
|
+
from typing import Dict, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
from ..nl2sql_components.core_extract import check_distinct, extract_limit
|
|
14
|
+
from ..nl2sql_components.mappings import COLUMN_PATTERNS, TABLE_PATTERNS
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"check_distinct",
|
|
18
|
+
"extract_group_by",
|
|
19
|
+
"extract_limit",
|
|
20
|
+
"extract_ordering",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def extract_group_by(
|
|
25
|
+
text: str,
|
|
26
|
+
column_patterns: Optional[Dict[str, str]] = None,
|
|
27
|
+
table_patterns: Optional[Dict[str, str]] = None,
|
|
28
|
+
) -> List[str]:
|
|
29
|
+
"""Extract GROUP BY columns from text.
|
|
30
|
+
|
|
31
|
+
Verbatim move of ``nl2sql_legacy._extract_group_by``; ``column_patterns``
|
|
32
|
+
and ``table_patterns`` replace the instance attributes.
|
|
33
|
+
"""
|
|
34
|
+
column_patterns = column_patterns or COLUMN_PATTERNS
|
|
35
|
+
table_patterns = table_patterns or TABLE_PATTERNS
|
|
36
|
+
group_cols: List[str] = []
|
|
37
|
+
patterns = [
|
|
38
|
+
r"按(.+?)(分组|汇总|统计)",
|
|
39
|
+
r"group\s*by\s*(\w+)",
|
|
40
|
+
r"grouped\s*by\s*(\w+)",
|
|
41
|
+
r"per\s+(\w+)",
|
|
42
|
+
r"each\s+(\w+)",
|
|
43
|
+
]
|
|
44
|
+
for pattern in patterns:
|
|
45
|
+
match = re.search(pattern, text, re.IGNORECASE)
|
|
46
|
+
if not match:
|
|
47
|
+
continue
|
|
48
|
+
group_text = match.group(1).strip()
|
|
49
|
+
for kw, col in column_patterns.items():
|
|
50
|
+
if kw in group_text:
|
|
51
|
+
group_cols.append(col)
|
|
52
|
+
break
|
|
53
|
+
else:
|
|
54
|
+
for kw in table_patterns:
|
|
55
|
+
if kw in group_text:
|
|
56
|
+
if "部门" in group_text or "department" in group_text:
|
|
57
|
+
group_cols.append("department")
|
|
58
|
+
elif "用户" in group_text or "user" in group_text:
|
|
59
|
+
group_cols.append("user_id")
|
|
60
|
+
elif "日期" in group_text or "date" in group_text:
|
|
61
|
+
group_cols.append("date")
|
|
62
|
+
elif "月" in group_text or "month" in group_text:
|
|
63
|
+
group_cols.append("MONTH(date)")
|
|
64
|
+
elif "年" in group_text or "year" in group_text:
|
|
65
|
+
group_cols.append("YEAR(date)")
|
|
66
|
+
break
|
|
67
|
+
return group_cols
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def extract_ordering(
|
|
71
|
+
text: str,
|
|
72
|
+
column_patterns: Optional[Dict[str, str]] = None,
|
|
73
|
+
) -> Optional[Tuple[str, str]]:
|
|
74
|
+
"""Extract ORDER BY clause from text.
|
|
75
|
+
|
|
76
|
+
Verbatim move of ``nl2sql_legacy._extract_ordering``; ``column_patterns``
|
|
77
|
+
replaces the instance attribute.
|
|
78
|
+
"""
|
|
79
|
+
column_patterns = column_patterns or COLUMN_PATTERNS
|
|
80
|
+
order_col = None
|
|
81
|
+
for pattern, column in column_patterns.items():
|
|
82
|
+
if pattern in text:
|
|
83
|
+
order_col = column
|
|
84
|
+
break
|
|
85
|
+
# Use word-boundary matching to avoid false positives (e.g., "orders" contains "order")
|
|
86
|
+
if any(re.search(rf'\b{k}\b', text, re.IGNORECASE) for k in ["排序", "排列", "sort", "order", "sorted"]):
|
|
87
|
+
return (
|
|
88
|
+
order_col or "id",
|
|
89
|
+
"DESC"
|
|
90
|
+
if any(re.search(rf'\b{k}\b', text, re.IGNORECASE) for k in ["降序", "从大到小", "递减", "desc", "descending", "decreasing"])
|
|
91
|
+
else "ASC",
|
|
92
|
+
)
|
|
93
|
+
if any(re.search(rf'\b{k}\b', text, re.IGNORECASE) for k in ["最大", "最高", "最多", "max", "highest", "top"]):
|
|
94
|
+
return (order_col or "amount", "DESC")
|
|
95
|
+
if any(re.search(rf'\b{k}\b', text, re.IGNORECASE) for k in ["最小", "最低", "最少", "min", "lowest"]):
|
|
96
|
+
return (order_col or "amount", "ASC")
|
|
97
|
+
return None
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: dialect rewriter (moved UNCHANGED from nl2sql_legacy).
|
|
3
|
+
|
|
4
|
+
Regex-based, executable-region-aware dialect adjustment. In P8 this module
|
|
5
|
+
is a verbatim move; AST-based rewriting is deferred to a future phase.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
from typing import Callable
|
|
9
|
+
|
|
10
|
+
from ..p1_sql_scanner import executable_segments
|
|
11
|
+
|
|
12
|
+
# Pattern types shared by the per-dialect replacement tables.
|
|
13
|
+
_re_p = re.Pattern
|
|
14
|
+
_re_s = str | Callable[[re.Match], str]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def apply_dialect_adjustments(sql: str, dialect: str) -> str:
|
|
18
|
+
"""Apply dialect-specific syntax adjustments.
|
|
19
|
+
|
|
20
|
+
Uses executable-region-aware replacement so literals, comments, and
|
|
21
|
+
quoted identifiers are never modified.
|
|
22
|
+
"""
|
|
23
|
+
dialect = dialect.lower()
|
|
24
|
+
|
|
25
|
+
if dialect == "hive":
|
|
26
|
+
return sql
|
|
27
|
+
|
|
28
|
+
replacements: list[tuple[_re_p, _re_s]] = []
|
|
29
|
+
|
|
30
|
+
if dialect == "oracle":
|
|
31
|
+
# Handle DATE_SUB/DATE_ADD/ADD_MONTHS with CURRENT_DATE first,
|
|
32
|
+
# then handle the already-converted forms (TRUNC(SYSDATE)).
|
|
33
|
+
replacements = [
|
|
34
|
+
(
|
|
35
|
+
re.compile(
|
|
36
|
+
r"\bDATE_SUB\(\s*(?:TRUNC\(SYSDATE\)|CURRENT_DATE)\s*,\s*(\d+)\s*\)",
|
|
37
|
+
re.IGNORECASE,
|
|
38
|
+
),
|
|
39
|
+
r"TRUNC(SYSDATE) - \1",
|
|
40
|
+
),
|
|
41
|
+
(
|
|
42
|
+
re.compile(
|
|
43
|
+
r"\bDATE_ADD\(\s*(?:TRUNC\(SYSDATE\)|CURRENT_DATE)\s*,\s*(\d+)\s*\)",
|
|
44
|
+
re.IGNORECASE,
|
|
45
|
+
),
|
|
46
|
+
r"TRUNC(SYSDATE) + \1",
|
|
47
|
+
),
|
|
48
|
+
(
|
|
49
|
+
re.compile(
|
|
50
|
+
r"\bADD_MONTHS\(\s*(?:TRUNC\(SYSDATE\)|CURRENT_DATE)\s*,\s*([+-]?\d+)\s*\)",
|
|
51
|
+
re.IGNORECASE,
|
|
52
|
+
),
|
|
53
|
+
r"ADD_MONTHS(TRUNC(SYSDATE), \1)",
|
|
54
|
+
),
|
|
55
|
+
(re.compile(r"\bCURRENT_TIMESTAMP\b", re.IGNORECASE), "SYSTIMESTAMP"),
|
|
56
|
+
(re.compile(r"\bCURRENT_DATE\b", re.IGNORECASE), "TRUNC(SYSDATE)"),
|
|
57
|
+
]
|
|
58
|
+
elif dialect == "tsql":
|
|
59
|
+
replacements = [
|
|
60
|
+
(
|
|
61
|
+
re.compile(
|
|
62
|
+
r"\bDATE_SUB\(\s*(?:CAST\(GETDATE\(\)\s+AS\s+DATE\)|CURRENT_DATE)\s*,\s*(\d+)\s*\)",
|
|
63
|
+
re.IGNORECASE,
|
|
64
|
+
),
|
|
65
|
+
r"DATEADD(DAY, -\1, CAST(GETDATE() AS DATE))",
|
|
66
|
+
),
|
|
67
|
+
(
|
|
68
|
+
re.compile(
|
|
69
|
+
r"\bDATE_ADD\(\s*(?:CAST\(GETDATE\(\)\s+AS\s+DATE\)|CURRENT_DATE)\s*,\s*(\d+)\s*\)",
|
|
70
|
+
re.IGNORECASE,
|
|
71
|
+
),
|
|
72
|
+
r"DATEADD(DAY, \1, CAST(GETDATE() AS DATE))",
|
|
73
|
+
),
|
|
74
|
+
(
|
|
75
|
+
re.compile(
|
|
76
|
+
r"\bADD_MONTHS\(\s*(?:CAST\(GETDATE\(\)\s+AS\s+DATE\)|CURRENT_DATE)\s*,\s*([+-]?\d+)\s*\)",
|
|
77
|
+
re.IGNORECASE,
|
|
78
|
+
),
|
|
79
|
+
r"DATEADD(MONTH, \1, CAST(GETDATE() AS DATE))",
|
|
80
|
+
),
|
|
81
|
+
(re.compile(r"\bCURRENT_TIMESTAMP\b", re.IGNORECASE), "SYSDATETIME()"),
|
|
82
|
+
(re.compile(r"\bCURRENT_DATE\b", re.IGNORECASE), "CAST(GETDATE() AS DATE)"),
|
|
83
|
+
]
|
|
84
|
+
elif dialect == "mysql":
|
|
85
|
+
replacements = [
|
|
86
|
+
(
|
|
87
|
+
re.compile(r"\bDATE_SUB\(\s*CURRENT_DATE\s*,\s*(\d+)\s*\)", re.IGNORECASE),
|
|
88
|
+
r"DATE_SUB(CURRENT_DATE, INTERVAL \1 DAY)",
|
|
89
|
+
),
|
|
90
|
+
(
|
|
91
|
+
re.compile(r"\bDATE_ADD\(\s*CURRENT_DATE\s*,\s*(\d+)\s*\)", re.IGNORECASE),
|
|
92
|
+
r"DATE_ADD(CURRENT_DATE, INTERVAL \1 DAY)",
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
re.compile(
|
|
96
|
+
r"\bADD_MONTHS\(\s*CURRENT_DATE\s*,\s*([+-]?\d+)\s*\)",
|
|
97
|
+
re.IGNORECASE,
|
|
98
|
+
),
|
|
99
|
+
lambda match: (
|
|
100
|
+
f"DATE_SUB(CURRENT_DATE, INTERVAL {abs(int(match.group(1)))} MONTH)"
|
|
101
|
+
if int(match.group(1)) < 0
|
|
102
|
+
else f"DATE_ADD(CURRENT_DATE, INTERVAL {match.group(1)} MONTH)"
|
|
103
|
+
),
|
|
104
|
+
),
|
|
105
|
+
]
|
|
106
|
+
elif dialect in ("postgres", "duckdb"):
|
|
107
|
+
replacements = [
|
|
108
|
+
(
|
|
109
|
+
re.compile(r"\bDATE_SUB\(\s*CURRENT_DATE\s*,\s*(\d+)\s*\)", re.IGNORECASE),
|
|
110
|
+
r"CURRENT_DATE - INTERVAL '\1 days'",
|
|
111
|
+
),
|
|
112
|
+
(
|
|
113
|
+
re.compile(r"\bDATE_ADD\(\s*CURRENT_DATE\s*,\s*(\d+)\s*\)", re.IGNORECASE),
|
|
114
|
+
r"CURRENT_DATE + INTERVAL '\1 days'",
|
|
115
|
+
),
|
|
116
|
+
(
|
|
117
|
+
re.compile(
|
|
118
|
+
r"\bADD_MONTHS\(\s*CURRENT_DATE\s*,\s*([+-]?\d+)\s*\)",
|
|
119
|
+
re.IGNORECASE,
|
|
120
|
+
),
|
|
121
|
+
r"CURRENT_DATE + INTERVAL '\1 month'",
|
|
122
|
+
),
|
|
123
|
+
]
|
|
124
|
+
else:
|
|
125
|
+
return sql
|
|
126
|
+
|
|
127
|
+
for pattern, replacement in replacements:
|
|
128
|
+
parts = []
|
|
129
|
+
cursor = 0
|
|
130
|
+
for start, end in executable_segments(sql):
|
|
131
|
+
parts.append(sql[cursor:start])
|
|
132
|
+
segment = sql[start:end]
|
|
133
|
+
segment = pattern.sub(replacement, segment)
|
|
134
|
+
parts.append(segment)
|
|
135
|
+
cursor = end
|
|
136
|
+
parts.append(sql[cursor:])
|
|
137
|
+
sql = "".join(parts)
|
|
138
|
+
|
|
139
|
+
return sql
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: intent analysis (moved from nl2sql_legacy).
|
|
3
|
+
|
|
4
|
+
Holds the tokenization / semantic-analysis step (``analyze_intent``)
|
|
5
|
+
and the text-parsing helpers (``parse_text``, ``detect_operation``)
|
|
6
|
+
that feed the resolution stages.
|
|
7
|
+
"""
|
|
8
|
+
import re
|
|
9
|
+
from typing import Any, Dict, Optional
|
|
10
|
+
|
|
11
|
+
from ..nl2sql_components.core_extract import detect_operation as _detect_operation_impl
|
|
12
|
+
from ..nl2sql_components.mappings import COLUMN_PATTERNS, KEYWORDS, TABLE_PATTERNS
|
|
13
|
+
from ..nl2sql_components.tokenizer import Tokenizer
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def analyze_intent(text: str) -> Dict[str, Any]:
|
|
17
|
+
"""Tokenize text and perform semantic analysis.
|
|
18
|
+
|
|
19
|
+
Returns the same dict shape the legacy
|
|
20
|
+
``_tokenize_and_analyze`` produced:
|
|
21
|
+
``tokens``, ``numbers``, ``quoted_strings``, ``tables``, ``columns``,
|
|
22
|
+
``operations``, ``is_chinese``, ``token_count``.
|
|
23
|
+
"""
|
|
24
|
+
tokenizer = Tokenizer()
|
|
25
|
+
tokens = tokenizer.tokenize(text)
|
|
26
|
+
numbers = tokenizer.extract_numbers(text)
|
|
27
|
+
quoted = tokenizer.extract_quoted_strings(text)
|
|
28
|
+
|
|
29
|
+
detected_tables = [
|
|
30
|
+
TABLE_PATTERNS[token]
|
|
31
|
+
for token in tokens
|
|
32
|
+
if token in TABLE_PATTERNS
|
|
33
|
+
]
|
|
34
|
+
detected_columns = [
|
|
35
|
+
COLUMN_PATTERNS[token]
|
|
36
|
+
for token in tokens
|
|
37
|
+
if token in COLUMN_PATTERNS
|
|
38
|
+
]
|
|
39
|
+
detected_ops = [
|
|
40
|
+
KEYWORDS[token]
|
|
41
|
+
for token in tokens
|
|
42
|
+
if token in KEYWORDS
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
"tokens": tokens,
|
|
47
|
+
"numbers": numbers,
|
|
48
|
+
"quoted_strings": quoted,
|
|
49
|
+
"tables": list(set(detected_tables)),
|
|
50
|
+
"columns": list(set(detected_columns)),
|
|
51
|
+
"operations": list(set(detected_ops)),
|
|
52
|
+
"is_chinese": tokenizer.is_chinese(text),
|
|
53
|
+
"token_count": len(tokens),
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def parse_text(text_lower: str, original: str) -> dict:
|
|
58
|
+
"""Parse and extract all elements from text (legacy ``_parse_text``)."""
|
|
59
|
+
parsed = {
|
|
60
|
+
"tables": [],
|
|
61
|
+
"columns": [],
|
|
62
|
+
"conditions": [],
|
|
63
|
+
"aggregations": [],
|
|
64
|
+
"time_range": None,
|
|
65
|
+
"limit": None,
|
|
66
|
+
"order": None,
|
|
67
|
+
"joins": [],
|
|
68
|
+
}
|
|
69
|
+
# Word-boundary matching for tables to avoid false positives like "order" matching "orders"
|
|
70
|
+
for pattern, table in TABLE_PATTERNS.items():
|
|
71
|
+
if re.search(r'\b' + re.escape(pattern) + r'\b', text_lower) and table not in parsed["tables"]:
|
|
72
|
+
parsed["tables"].append(table)
|
|
73
|
+
# Word-boundary matching for columns
|
|
74
|
+
for pattern, column in COLUMN_PATTERNS.items():
|
|
75
|
+
if re.search(r'\b' + re.escape(pattern) + r'\b', text_lower) and column not in parsed["columns"]:
|
|
76
|
+
parsed["columns"].append(column)
|
|
77
|
+
numbers = re.findall(r"\d+", original)
|
|
78
|
+
if numbers:
|
|
79
|
+
parsed["numbers"] = numbers
|
|
80
|
+
return parsed
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def detect_operation(text: str) -> str:
|
|
84
|
+
"""Detect SQL operation type from text (delegates to core_extract)."""
|
|
85
|
+
return _detect_operation_impl(text, KEYWORDS)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: Query IR.
|
|
3
|
+
|
|
4
|
+
The IR is the structured intermediate representation produced by the
|
|
5
|
+
resolution stages and consumed by the renderer. In P8 it mirrors the
|
|
6
|
+
flat positional-argument shape of the legacy ``_build_sql_enhanced``
|
|
7
|
+
call, plus the ``evidence`` accumulator, so the migration is
|
|
8
|
+
behavior-preserving.
|
|
9
|
+
"""
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Dict, List, Optional, Tuple, Union
|
|
12
|
+
|
|
13
|
+
from ..nl2sql_components.evidence import GenerationEvidence
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class QueryIR:
|
|
18
|
+
"""Structured intermediate representation for an NL2SQL query.
|
|
19
|
+
|
|
20
|
+
Mirrors the positional-argument shape of the legacy
|
|
21
|
+
``_build_sql_enhanced`` call. ``conditions`` may be a ``list`` of
|
|
22
|
+
predicate strings or a single ``str`` — the renderer handles both
|
|
23
|
+
(mirroring current legacy code).
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
operation: str
|
|
27
|
+
table: str
|
|
28
|
+
columns: List[str]
|
|
29
|
+
conditions: Union[List[str], str]
|
|
30
|
+
aggregations: List[str]
|
|
31
|
+
group_by: List[str]
|
|
32
|
+
ordering: Optional[Tuple[str, str]]
|
|
33
|
+
limit: Optional[int]
|
|
34
|
+
joins: List[Dict]
|
|
35
|
+
distinct: bool
|
|
36
|
+
dialect: str
|
|
37
|
+
evidence: GenerationEvidence = field(default_factory=GenerationEvidence)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def build_ir(
|
|
41
|
+
operation: str,
|
|
42
|
+
table: str,
|
|
43
|
+
columns: List[str],
|
|
44
|
+
conditions: Union[List[str], str],
|
|
45
|
+
aggregations: List[str],
|
|
46
|
+
group_by: List[str],
|
|
47
|
+
ordering: Optional[Tuple[str, str]],
|
|
48
|
+
limit: Optional[int],
|
|
49
|
+
joins: List[Dict],
|
|
50
|
+
distinct: bool,
|
|
51
|
+
dialect: str,
|
|
52
|
+
) -> QueryIR:
|
|
53
|
+
"""Thin positional-arg adapter mirroring the legacy call site.
|
|
54
|
+
|
|
55
|
+
Constructs a ``QueryIR`` with a fresh ``GenerationEvidence``
|
|
56
|
+
accumulator, identical to what the legacy code did implicitly.
|
|
57
|
+
"""
|
|
58
|
+
return QueryIR(
|
|
59
|
+
operation=operation,
|
|
60
|
+
table=table,
|
|
61
|
+
columns=columns,
|
|
62
|
+
conditions=conditions,
|
|
63
|
+
aggregations=aggregations,
|
|
64
|
+
group_by=group_by,
|
|
65
|
+
ordering=ordering,
|
|
66
|
+
limit=limit,
|
|
67
|
+
joins=joins,
|
|
68
|
+
distinct=distinct,
|
|
69
|
+
dialect=dialect,
|
|
70
|
+
)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""P8 NL2SQL pipeline: result model.
|
|
3
|
+
|
|
4
|
+
``NL2SQLResult`` is the single canonical result dataclass. It is re-exported
|
|
5
|
+
from every import path (``nl2sql``, ``nl2sql_legacy``, ``nl2sql_pipeline``)
|
|
6
|
+
so that ``from backend.core.nl2sql import NL2SQLResult as A`` and
|
|
7
|
+
``from backend.core.nl2sql_legacy import NL2SQLResult as B`` satisfy
|
|
8
|
+
``A is B``.
|
|
9
|
+
"""
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
|
|
12
|
+
from ..nl2sql_components.evidence import GenerationEvidence
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class NL2SQLResult:
|
|
17
|
+
"""Result of NL2SQL generation."""
|
|
18
|
+
|
|
19
|
+
success: bool = False
|
|
20
|
+
input_text: str = ""
|
|
21
|
+
sql: str | None = None
|
|
22
|
+
dialect: str = ""
|
|
23
|
+
explanation: str = ""
|
|
24
|
+
confidence: float = 0.0
|
|
25
|
+
# Structured evidence trail backing ``confidence``. Optional and
|
|
26
|
+
# backward-compatible: consumers that only need the scalar score can
|
|
27
|
+
# ignore this field.
|
|
28
|
+
evidence: GenerationEvidence | None = None
|
|
29
|
+
suggestions: list = field(default_factory=list)
|
|
30
|
+
parsed_elements: dict = field(default_factory=dict)
|