cdmduck 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cdmduck/__init__.py +2 -0
- cdmduck/cli.py +28 -0
- cdmduck/sqlrender/__init__.py +37 -0
- cdmduck/sqlrender/_bigquery_spark.py +207 -0
- cdmduck/sqlrender/_java.py +124 -0
- cdmduck/sqlrender/_render.py +220 -0
- cdmduck/sqlrender/_strings.py +362 -0
- cdmduck/sqlrender/_translate.py +343 -0
- cdmduck/sqlrender/csv/replacementPatterns.csv +1451 -0
- cdmduck-0.0.1.dist-info/METADATA +74 -0
- cdmduck-0.0.1.dist-info/RECORD +16 -0
- cdmduck-0.0.1.dist-info/WHEEL +5 -0
- cdmduck-0.0.1.dist-info/entry_points.txt +2 -0
- cdmduck-0.0.1.dist-info/licenses/LICENSE +202 -0
- cdmduck-0.0.1.dist-info/licenses/NOTICE +9 -0
- cdmduck-0.0.1.dist-info/top_level.txt +1 -0
cdmduck/__init__.py
ADDED
cdmduck/cli.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""명령어 `cdmduck`."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main(argv=None):
|
|
7
|
+
ap = argparse.ArgumentParser(prog='cdmduck', description='OMOP CDM / ATLAS-compatible tools on DuckDB')
|
|
8
|
+
sub = ap.add_subparsers(dest='cmd', required=True)
|
|
9
|
+
t = sub.add_parser('translate', help='SQL Server 방언 SQL 을 대상 방언으로 번역')
|
|
10
|
+
t.add_argument('file', help="SQL 파일 ('-' 는 표준입력)")
|
|
11
|
+
t.add_argument('dialect')
|
|
12
|
+
t.add_argument('--temp-schema')
|
|
13
|
+
t.add_argument('-p', '--param', action='append', default=[], metavar='KEY=VALUE', help='번역 전에 렌더할 파라미터')
|
|
14
|
+
sub.add_parser('dialects', help='지원 방언 목록')
|
|
15
|
+
a = ap.parse_args(argv)
|
|
16
|
+
from .sqlrender import dialects, render, translate
|
|
17
|
+
if a.cmd == 'dialects':
|
|
18
|
+
print('\n'.join(dialects()))
|
|
19
|
+
elif a.cmd == 'translate':
|
|
20
|
+
sql = sys.stdin.read() if a.file == '-' else open(a.file, encoding='utf-8').read()
|
|
21
|
+
if a.param:
|
|
22
|
+
sql = render(sql, **dict(p.split('=', 1) for p in a.param))
|
|
23
|
+
sys.stdout.write(translate(sql, a.dialect, temp_emulation_schema=a.temp_schema))
|
|
24
|
+
return 0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
if __name__ == '__main__':
|
|
28
|
+
sys.exit(main())
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""OHDSI SqlRender 의 파이썬 이식 — Java 판과 출력이 글자 단위로 같도록 맞춘다.
|
|
2
|
+
|
|
3
|
+
from cdmduck.sqlrender import render, translate
|
|
4
|
+
sql = render("SELECT * FROM @cdm.person {@limit}?{LIMIT 10}", cdm="main", limit=True)
|
|
5
|
+
sql = translate(sql, "duckdb")
|
|
6
|
+
"""
|
|
7
|
+
from ._java import JavaError
|
|
8
|
+
from ._render import check as check_render
|
|
9
|
+
from ._render import render_sql
|
|
10
|
+
from ._strings import split_sql
|
|
11
|
+
from ._translate import (check as check_translate, dialects, generate_session_id, set_replacement_patterns,
|
|
12
|
+
translate_single_statement_sql, translate_sql)
|
|
13
|
+
|
|
14
|
+
__all__ = ['render', 'translate', 'translate_single_statement', 'split_sql', 'render_sql', 'translate_sql',
|
|
15
|
+
'translate_single_statement_sql', 'dialects', 'generate_session_id', 'set_replacement_patterns',
|
|
16
|
+
'check_render', 'check_translate', 'JavaError']
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _value(v):
|
|
20
|
+
"""R SqlRender 와 같은 값 변환: 논리값은 TRUE/FALSE, 목록은 쉼표로 잇는다."""
|
|
21
|
+
if isinstance(v, bool):
|
|
22
|
+
return 'TRUE' if v else 'FALSE'
|
|
23
|
+
if isinstance(v, (list, tuple, set)):
|
|
24
|
+
return ','.join(_value(x) for x in v)
|
|
25
|
+
return str(v)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def render(sql, **params):
|
|
29
|
+
return render_sql(sql, list(params), [_value(v) for v in params.values()])
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def translate(sql, target_dialect, temp_emulation_schema=None, session_id=None):
|
|
33
|
+
return translate_sql(sql, target_dialect, session_id, temp_emulation_schema)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def translate_single_statement(sql, target_dialect, temp_emulation_schema=None, session_id=None):
|
|
37
|
+
return translate_single_statement_sql(sql, target_dialect, session_id, temp_emulation_schema)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""org.ohdsi.sql.BigQuerySparkTranslate 포팅 (DB 연결이 필요한 sparkHandleInsert 는 제외)."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from ._java import JavaError, split, substring, trim
|
|
5
|
+
from ._strings import split_sql, tokenize_sql
|
|
6
|
+
from ._translate import parse_search_pattern, search
|
|
7
|
+
|
|
8
|
+
SELECT, GROUP_BY, ORDER_BY, WITH_COLUMNS, IN = range(5)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _CommaListIterator:
|
|
12
|
+
def __init__(self, expression_list, list_type):
|
|
13
|
+
self.list_type = list_type
|
|
14
|
+
self.expression_list = expression_list
|
|
15
|
+
self._split_list()
|
|
16
|
+
self.expression_list = ',' + self.expression_list + ','
|
|
17
|
+
self.pattern = parse_search_pattern(', @@a ,')
|
|
18
|
+
self.cur = search(self.expression_list, self.pattern, 0)
|
|
19
|
+
if self.cur.start != -1:
|
|
20
|
+
self._split_expression()
|
|
21
|
+
|
|
22
|
+
def done(self):
|
|
23
|
+
return self.cur.start == -1
|
|
24
|
+
|
|
25
|
+
def next(self):
|
|
26
|
+
n = len(tokenize_sql(substring(self.expression_list, self.cur.start, self.cur.end)))
|
|
27
|
+
self.cur = search(self.expression_list, self.pattern, self.cur.start_token + n - 1)
|
|
28
|
+
if self.cur.start != -1:
|
|
29
|
+
self._split_expression()
|
|
30
|
+
|
|
31
|
+
def full(self):
|
|
32
|
+
return _s(self.prefix) + _s(self.suffix)
|
|
33
|
+
|
|
34
|
+
def _split_list(self):
|
|
35
|
+
self.list_prefix = ''
|
|
36
|
+
self.list_suffix = ''
|
|
37
|
+
if self.list_type == SELECT:
|
|
38
|
+
m = search('^' + _s(self.expression_list) + '$', parse_search_pattern('^ distinct @@a $'), 0)
|
|
39
|
+
if m.start != -1:
|
|
40
|
+
self.list_prefix = 'distinct '
|
|
41
|
+
self.expression_list = m.variable_to_value.get('@@a')
|
|
42
|
+
m = search('^' + _s(self.expression_list) + '$', parse_search_pattern('^@@a into @@b$'), 0)
|
|
43
|
+
if m.start != -1:
|
|
44
|
+
self.expression_list = m.variable_to_value.get('@@a')
|
|
45
|
+
self.list_suffix = ' into ' + _s(m.variable_to_value.get('@@b'))
|
|
46
|
+
elif self.list_type == GROUP_BY:
|
|
47
|
+
m = search('^' + _s(self.expression_list) + '$', parse_search_pattern('^@@a order by @@b$'), 0)
|
|
48
|
+
if m.start != -1:
|
|
49
|
+
self.expression_list = m.variable_to_value.get('@@a')
|
|
50
|
+
self.list_suffix = ' order by ' + _s(m.variable_to_value.get('@@b'))
|
|
51
|
+
|
|
52
|
+
def _split_expression(self):
|
|
53
|
+
self.prefix = self.cur.variable_to_value.get('@@a')
|
|
54
|
+
self.suffix = ''
|
|
55
|
+
if self.list_type == SELECT:
|
|
56
|
+
self._split_alias()
|
|
57
|
+
elif self.list_type == ORDER_BY:
|
|
58
|
+
tokens = tokenize_sql(self.full())
|
|
59
|
+
if not tokens:
|
|
60
|
+
raise JavaError('IndexOutOfBoundsException')
|
|
61
|
+
last = tokens[-1]
|
|
62
|
+
if last.text.lower() in ('asc', 'desc'):
|
|
63
|
+
self.prefix = substring(self.full(), 0, last.start - 1)
|
|
64
|
+
self.suffix = ' ' + last.text
|
|
65
|
+
|
|
66
|
+
def _split_alias(self):
|
|
67
|
+
tokens = tokenize_sql(self.prefix)
|
|
68
|
+
m = search('^' + self.prefix + '$', parse_search_pattern('^ @@a as @@b $'), 0)
|
|
69
|
+
if m.start == -1:
|
|
70
|
+
if len(tokens) >= 2:
|
|
71
|
+
alias = tokens[-1]
|
|
72
|
+
preceding = tokens[-2].text
|
|
73
|
+
if alias.is_identifier() and preceding != '.' and preceding != '+':
|
|
74
|
+
self.prefix = substring(self.prefix, 0, alias.start)
|
|
75
|
+
self.suffix = alias.text
|
|
76
|
+
else:
|
|
77
|
+
self.prefix = m.variable_to_value.get('@@a')
|
|
78
|
+
self.suffix = m.variable_to_value.get('@@b')
|
|
79
|
+
|
|
80
|
+
def is_single_column_reference(self):
|
|
81
|
+
t = tokenize_sql(self.prefix)
|
|
82
|
+
return len(t) == 3 and t[0].is_identifier() and t[1].text == '.' and t[2].is_identifier()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _s(x):
|
|
86
|
+
"""Java 문자열 연결: null 은 "null"."""
|
|
87
|
+
return 'null' if x is None else x
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _alias_ctes(sql, pattern):
|
|
91
|
+
pp = parse_search_pattern(pattern)
|
|
92
|
+
m = search(sql, pp, 0)
|
|
93
|
+
while m.start != -1:
|
|
94
|
+
vv = m.variable_to_value
|
|
95
|
+
with_it = _CommaListIterator(vv.get('@@b'), WITH_COLUMNS)
|
|
96
|
+
sel_it = _CommaListIterator(vv.get('@@c'), SELECT)
|
|
97
|
+
repl = ''
|
|
98
|
+
while not with_it.done():
|
|
99
|
+
if sel_it.done():
|
|
100
|
+
break
|
|
101
|
+
expr = _s(sel_it.prefix) + ' as ' + with_it.full()
|
|
102
|
+
if repl:
|
|
103
|
+
repl += ','
|
|
104
|
+
repl += expr
|
|
105
|
+
with_it.next()
|
|
106
|
+
sel_it.next()
|
|
107
|
+
repl = sel_it.list_prefix + repl + sel_it.list_suffix
|
|
108
|
+
a = vv.get('@@a')
|
|
109
|
+
if a is None:
|
|
110
|
+
raise JavaError('NullPointerException')
|
|
111
|
+
body = pattern.replace('@@a', a).replace('(@@b)', '').replace('@@c', repl).replace('@@d', vv.get('@@d') or '')
|
|
112
|
+
sql = substring(sql, 0, m.start) + body + substring(sql, m.end)
|
|
113
|
+
m = search(sql, pp, m.start_token + 1)
|
|
114
|
+
return sql
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _convert_select_list_references(sql, select_pattern, list_type):
|
|
118
|
+
pp = parse_search_pattern(select_pattern)
|
|
119
|
+
m = search(sql, pp, 0)
|
|
120
|
+
while m.start != -1:
|
|
121
|
+
vv = m.variable_to_value
|
|
122
|
+
select_list = vv.get('@@s')
|
|
123
|
+
repl = ''
|
|
124
|
+
it = _CommaListIterator(vv.get('@@r'), list_type)
|
|
125
|
+
while not it.done():
|
|
126
|
+
expr, suffix = it.prefix, it.suffix
|
|
127
|
+
expr_pp = parse_search_pattern(expr)
|
|
128
|
+
if it.is_single_column_reference():
|
|
129
|
+
repl += ', ' + expr + suffix
|
|
130
|
+
else:
|
|
131
|
+
sel_it = _CommaListIterator(select_list, SELECT)
|
|
132
|
+
found = False
|
|
133
|
+
i = 1
|
|
134
|
+
while not sel_it.done():
|
|
135
|
+
if search(sel_it.prefix, expr_pp, 0).start != -1:
|
|
136
|
+
found = True
|
|
137
|
+
repl += ', ' + str(i) + suffix
|
|
138
|
+
break
|
|
139
|
+
i += 1
|
|
140
|
+
sel_it.next()
|
|
141
|
+
if not found:
|
|
142
|
+
repl += ', ' + expr + suffix
|
|
143
|
+
it.next()
|
|
144
|
+
repl = it.list_prefix + substring(repl, 1) + it.list_suffix
|
|
145
|
+
tail = substring(sql, m.end)
|
|
146
|
+
sql = substring(sql, 0, m.start)
|
|
147
|
+
for block in pp:
|
|
148
|
+
if sql:
|
|
149
|
+
sql += ' '
|
|
150
|
+
if block.is_variable:
|
|
151
|
+
sql += repl if block.text == '@@r' else _s(vv.get(block.text))
|
|
152
|
+
else:
|
|
153
|
+
sql += block.text
|
|
154
|
+
sql += tail
|
|
155
|
+
m = search(sql, pp, m.start_token + 1)
|
|
156
|
+
return sql
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _lower_case(sql):
|
|
160
|
+
for t in tokenize_sql(sql):
|
|
161
|
+
if not t.in_quotes and not t.text.startswith('@'):
|
|
162
|
+
sql = substring(sql, 0, t.start) + t.text.lower() + substring(sql, t.end)
|
|
163
|
+
return sql
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def translate_bigquery(sql):
|
|
167
|
+
sql = _lower_case(sql)
|
|
168
|
+
sql = _alias_ctes(sql, 'with @@a (@@b) as (select @@c from @@d)')
|
|
169
|
+
sql = _alias_ctes(sql, 'with @@a (@@b) as (select @@c union @@d)')
|
|
170
|
+
sql = _alias_ctes(sql, 'with @@a (@@b) as (select @@c)')
|
|
171
|
+
sql = _alias_ctes(sql, ', @@a (@@b) as (select @@c from @@d)')
|
|
172
|
+
g = 'select @@s from @@b group by @@r'
|
|
173
|
+
for suffix in (';', ')', ' having', ' order by'):
|
|
174
|
+
sql = _convert_select_list_references(sql, g + suffix, GROUP_BY)
|
|
175
|
+
o = 'select @@s from @@b order by @@r'
|
|
176
|
+
for suffix in (';', ')'):
|
|
177
|
+
sql = _convert_select_list_references(sql, o + suffix, ORDER_BY)
|
|
178
|
+
return sql
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _spark_create_table(sql):
|
|
182
|
+
if not sql.endswith(';'):
|
|
183
|
+
sql += ';'
|
|
184
|
+
pp = parse_search_pattern('CREATE TABLE @@table (@@definition)')
|
|
185
|
+
sql = re.sub(' +', ' ', trim(sql).replace('\t', ' '))
|
|
186
|
+
m = search(sql, pp, 0)
|
|
187
|
+
table = m.variable_to_value.get('@@table')
|
|
188
|
+
definition = m.variable_to_value.get('@@definition')
|
|
189
|
+
if table is not None and definition is not None:
|
|
190
|
+
table = table.replace('\r\n', '')
|
|
191
|
+
definition = definition.lower().replace('\r\n', '').replace(' as ', ' ')
|
|
192
|
+
cols = []
|
|
193
|
+
for f in split(definition, ','):
|
|
194
|
+
parts = split(trim(f), ' ')
|
|
195
|
+
if len(parts) < 2:
|
|
196
|
+
raise JavaError('ArrayIndexOutOfBoundsException')
|
|
197
|
+
cols.append('\tCAST(NULL AS ' + parts[1] + ') AS ' + parts[0])
|
|
198
|
+
sql = substring(sql, 0, m.start) + 'SELECT ' + ',\r\n'.join(cols) + ' INTO ' + table + ' WHERE 1 = 0'
|
|
199
|
+
return sql.replace(';', '')
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def translate_spark(sql):
|
|
203
|
+
parts = [_spark_create_table(p) for p in split_sql(sql)]
|
|
204
|
+
joined = trim(';\r\n'.join(parts))
|
|
205
|
+
if len(parts) > 1 or trim(sql).endswith(';'):
|
|
206
|
+
return joined + ';'
|
|
207
|
+
return joined
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Java 동작을 그대로 흉내 내는 보조 함수들.
|
|
2
|
+
|
|
3
|
+
SqlRender(Java) 와 출력이 글자 단위로 같아야 하므로 Python 기본 동작과 다른 부분을 여기서 맞춘다:
|
|
4
|
+
String.trim/split, Character.isWhitespace/isLetterOrDigit, HashMap 순회 순서, 범위를 벗어난 substring 의 예외.
|
|
5
|
+
"""
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
|
|
9
|
+
__all__ = ['JavaError', 'trim', 'split', 'is_whitespace', 'is_letter_or_digit', 'is_letter', 'substring',
|
|
10
|
+
'char_at', 'hash_order', 'strip_trailing_ws_once',
|
|
11
|
+
'strip_ws_before_end', 'remove_blank_lines']
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class JavaError(RuntimeError):
|
|
15
|
+
"""Java 쪽이 예외(StringIndexOutOfBounds, RuntimeException 등)를 던지는 자리."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def trim(s):
|
|
19
|
+
"""String.trim(): 앞뒤의 코드포인트 ≤ U+0020 문자를 지운다(유니코드 공백은 건드리지 않음)."""
|
|
20
|
+
i, j = 0, len(s)
|
|
21
|
+
while i < j and s[i] <= ' ':
|
|
22
|
+
i += 1
|
|
23
|
+
while j > i and s[j - 1] <= ' ':
|
|
24
|
+
j -= 1
|
|
25
|
+
return s[i:j]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def split(s, regex):
|
|
29
|
+
"""String.split(regex): 끝쪽 빈 문자열을 버리고, 일치가 전혀 없으면 [s]."""
|
|
30
|
+
parts = re.split(regex, s)
|
|
31
|
+
if len(parts) == 1:
|
|
32
|
+
return [s]
|
|
33
|
+
while parts and parts[-1] == '':
|
|
34
|
+
parts.pop()
|
|
35
|
+
return parts
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
_JAVA_SPACE_EXCLUDED = {' ', ' ', ' '}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def is_whitespace(ch):
|
|
42
|
+
"""Character.isWhitespace."""
|
|
43
|
+
if ch in '\t\n\u000b\f\r\u001c\u001d\u001e\u001f':
|
|
44
|
+
return True
|
|
45
|
+
if ch in _JAVA_SPACE_EXCLUDED:
|
|
46
|
+
return False
|
|
47
|
+
return unicodedata.category(ch) in ('Zs', 'Zl', 'Zp')
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_letter(ch):
|
|
51
|
+
"""Character.isLetter: Lu Ll Lt Lm Lo."""
|
|
52
|
+
return unicodedata.category(ch) in ('Lu', 'Ll', 'Lt', 'Lm', 'Lo')
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def is_letter_or_digit(ch):
|
|
56
|
+
"""Character.isLetterOrDigit: 문자 또는 Nd(십진 숫자)."""
|
|
57
|
+
c = unicodedata.category(ch)
|
|
58
|
+
return c in ('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd')
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def substring(s, start, end=None):
|
|
62
|
+
"""String.substring: 범위를 벗어나면 예외."""
|
|
63
|
+
if end is None:
|
|
64
|
+
end = len(s)
|
|
65
|
+
if start < 0 or end > len(s) or start > end:
|
|
66
|
+
raise JavaError(f'StringIndexOutOfBoundsException: begin {start}, end {end}, length {len(s)}')
|
|
67
|
+
return s[start:end]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def char_at(s, i):
|
|
71
|
+
if i < 0 or i >= len(s):
|
|
72
|
+
raise JavaError(f'StringIndexOutOfBoundsException: index {i}, length {len(s)}')
|
|
73
|
+
return s[i]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _string_hash(s):
|
|
77
|
+
"""java.lang.String.hashCode (UTF-16 단위)."""
|
|
78
|
+
h = 0
|
|
79
|
+
data = s.encode('utf-16-be')
|
|
80
|
+
for k in range(0, len(data), 2):
|
|
81
|
+
h = (31 * h + ((data[k] << 8) | data[k + 1])) & 0xFFFFFFFF
|
|
82
|
+
return h
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def hash_order(keys):
|
|
86
|
+
"""java.util.HashMap 의 순회 순서로 키를 정렬한다(삽입 순서 목록을 받는다).
|
|
87
|
+
용량은 16에서 시작해 크기가 0.75배를 넘으면 두 배. 버킷 = (h ^ h>>>16) & (cap-1), 같은 버킷은 삽입 순서."""
|
|
88
|
+
n = len(keys)
|
|
89
|
+
cap = 16
|
|
90
|
+
while n > cap * 0.75:
|
|
91
|
+
cap *= 2
|
|
92
|
+
def bucket(k):
|
|
93
|
+
h = _string_hash(k)
|
|
94
|
+
h ^= h >> 16
|
|
95
|
+
return h & (cap - 1)
|
|
96
|
+
return [k for _, _, k in sorted((bucket(k), i, k) for i, k in enumerate(keys))]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def strip_trailing_ws_once(s):
|
|
100
|
+
"""s.replaceAll("\\s$", "") — Java 의 \\s 는 [ \\t\\n\\x0B\\f\\r], $ 는 끝 또는 마지막 줄바꿈 앞."""
|
|
101
|
+
return re.sub(r'[ \t\n\x0b\f\r]$', '', s)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
_LINE_TERMS = '\n\r\u0085
'
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def strip_ws_before_end(s):
|
|
108
|
+
"""s.replaceAll("\\\\s$", "") 를 Java 규칙 그대로: $ 는 입력 끝 또는 마지막 줄끝문자(\\r\\n 포함) 바로 앞."""
|
|
109
|
+
n = len(s)
|
|
110
|
+
ends = {n}
|
|
111
|
+
if s.endswith('\r\n'):
|
|
112
|
+
ends.add(n - 2)
|
|
113
|
+
elif n and s[-1] in _LINE_TERMS:
|
|
114
|
+
ends.add(n - 1)
|
|
115
|
+
drop = {p - 1 for p in ends if p > 0 and s[p - 1] in ' \t\n\x0b\f\r'}
|
|
116
|
+
return ''.join(ch for i, ch in enumerate(s) if i not in drop)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# Java (?m)^[ \t]*\r?\n : ^ 는 처음, [\n\u0085
] 뒤, 또는 \n 이 따르지 않는 \r 뒤
|
|
120
|
+
_BLANK_LINE_RE = re.compile('(?:\\A|(?<=[\n\u0085
])|(?<=\r)(?!\n))[ \t]*\r?\n')
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def remove_blank_lines(s):
|
|
124
|
+
return _BLANK_LINE_RE.sub('', s)
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""org.ohdsi.sql.SqlRender 포팅: @파라미터 치환, {DEFAULT …}, {조건} ? {참} : {거짓}."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from ._java import JavaError, hash_order, is_whitespace, split, substring, trim
|
|
5
|
+
from ._strings import replace as _str_replace
|
|
6
|
+
|
|
7
|
+
_JAVA_WS = '[ \\t\\n\\x0b\\f\\r]'
|
|
8
|
+
_DEFAULTS_RE = re.compile(r'\{DEFAULT[^}]*\}' + _JAVA_WS + '*\n?')
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _Span:
|
|
12
|
+
__slots__ = ('start', 'end', 'valid')
|
|
13
|
+
|
|
14
|
+
def __init__(self, start, end):
|
|
15
|
+
self.start, self.end, self.valid = start, end, True
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _IfThenElse:
|
|
19
|
+
__slots__ = ('condition', 'if_true', 'if_false', 'has_if_false')
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.condition = self.if_true = self.if_false = None
|
|
23
|
+
self.has_if_false = False
|
|
24
|
+
|
|
25
|
+
def start(self):
|
|
26
|
+
return self.condition.start
|
|
27
|
+
|
|
28
|
+
def end(self):
|
|
29
|
+
return self.if_false.end if self.has_if_false else self.if_true.end
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _find_spans(s, open_ch, close_ch):
|
|
33
|
+
starts, spans = [], []
|
|
34
|
+
for i, ch in enumerate(s):
|
|
35
|
+
if ch == open_ch:
|
|
36
|
+
starts.append(i)
|
|
37
|
+
elif ch == close_ch and starts:
|
|
38
|
+
spans.append(_Span(starts.pop(), i + 1))
|
|
39
|
+
return spans
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _link_if_then_elses(s, spans):
|
|
43
|
+
# Java 와 같이 break 없이 전부 훑는다: 한 조건에 '?' 짝이 여럿이면 여럿 만들고, ':' 는 마지막 것이 이긴다
|
|
44
|
+
out = []
|
|
45
|
+
n = len(spans)
|
|
46
|
+
if n > 1:
|
|
47
|
+
for i in range(n - 1):
|
|
48
|
+
for j in range(i + 1, n):
|
|
49
|
+
if spans[j].start > spans[i].end:
|
|
50
|
+
if trim(s[spans[i].end:spans[j].start]) == '?':
|
|
51
|
+
ite = _IfThenElse()
|
|
52
|
+
ite.condition = spans[i]
|
|
53
|
+
ite.if_true = spans[j]
|
|
54
|
+
for k in range(j + 1, n):
|
|
55
|
+
if spans[k].start > spans[j].end:
|
|
56
|
+
if trim(s[spans[j].end:spans[k].start]) == ':':
|
|
57
|
+
ite.if_false = spans[k]
|
|
58
|
+
ite.has_if_false = True
|
|
59
|
+
out.append(ite)
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _remove_parentheses(s):
|
|
64
|
+
if len(s) > 1 and ((s[0] == "'" and s[-1] == "'") or (s[0] == '"' and s[-1] == '"')):
|
|
65
|
+
return s[1:-1]
|
|
66
|
+
return s
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _preceded_by_in(start, s):
|
|
70
|
+
s = s.lower()
|
|
71
|
+
matched = 0
|
|
72
|
+
for i in range(start - 1, -1, -1):
|
|
73
|
+
ch = s[i]
|
|
74
|
+
if not is_whitespace(ch):
|
|
75
|
+
if matched == 0 and ch == 'n':
|
|
76
|
+
matched += 1
|
|
77
|
+
elif matched == 1 and ch == 'i':
|
|
78
|
+
matched += 1
|
|
79
|
+
else:
|
|
80
|
+
return False
|
|
81
|
+
elif matched == 2:
|
|
82
|
+
return True
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _evaluate_primitive(s):
|
|
87
|
+
s = trim(s)
|
|
88
|
+
lc = s.lower()
|
|
89
|
+
if lc in ('false', '0', '!true', '!1'):
|
|
90
|
+
return False
|
|
91
|
+
if lc in ('true', '1', '!false', '!0'):
|
|
92
|
+
return True
|
|
93
|
+
found = s.find('==')
|
|
94
|
+
if found != -1:
|
|
95
|
+
left = _remove_parentheses(trim(s[:found]))
|
|
96
|
+
right = _remove_parentheses(trim(s[found + 2:]))
|
|
97
|
+
return left == right
|
|
98
|
+
found = s.find('!=')
|
|
99
|
+
if found == -1:
|
|
100
|
+
found = s.find('<>')
|
|
101
|
+
if found != -1:
|
|
102
|
+
left = _remove_parentheses(trim(s[:found]))
|
|
103
|
+
right = _remove_parentheses(trim(s[found + 2:]))
|
|
104
|
+
return left != right
|
|
105
|
+
found = lc.find(' in ')
|
|
106
|
+
if found != -1:
|
|
107
|
+
left = _remove_parentheses(trim(substring(s, 0, found)))
|
|
108
|
+
right = trim(substring(s, found + 4))
|
|
109
|
+
if len(right) > 2 and right[0] == '(' and right[-1] == ')':
|
|
110
|
+
for part in split(right[1:-1], ','):
|
|
111
|
+
if left == _remove_parentheses(part): # Java 도 part 를 trim 하지 않는다
|
|
112
|
+
return True
|
|
113
|
+
return False
|
|
114
|
+
raise JavaError('Error parsing boolean condition: "' + s + '"')
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _evaluate_boolean(s):
|
|
118
|
+
s = trim(s)
|
|
119
|
+
if '&' in s:
|
|
120
|
+
return all(_evaluate_primitive(p) for p in split(s, '&'))
|
|
121
|
+
if '|' in s:
|
|
122
|
+
return any(_evaluate_primitive(p) for p in split(s, r'\|'))
|
|
123
|
+
return _evaluate_primitive(s)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _replace(s, spans, to_start, to_end, with_start, with_end):
|
|
127
|
+
with_str = substring(s, with_start, with_end + 1)
|
|
128
|
+
s = _str_replace(s, to_start, to_end, with_str)
|
|
129
|
+
for sp in spans:
|
|
130
|
+
if sp.valid:
|
|
131
|
+
if sp.start > to_start:
|
|
132
|
+
if with_start <= sp.start < with_end:
|
|
133
|
+
delta = to_start - with_start
|
|
134
|
+
sp.start += delta
|
|
135
|
+
sp.end += delta
|
|
136
|
+
elif sp.start > to_end:
|
|
137
|
+
delta = to_start - to_end + len(with_str)
|
|
138
|
+
sp.start += delta
|
|
139
|
+
sp.end += delta
|
|
140
|
+
else:
|
|
141
|
+
sp.valid = False
|
|
142
|
+
elif sp.end > to_end:
|
|
143
|
+
sp.end += to_start - to_end + len(with_str)
|
|
144
|
+
return s
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _evaluate_condition(s):
|
|
148
|
+
s = trim(s)
|
|
149
|
+
spans = _find_spans(s, '(', ')')
|
|
150
|
+
# 닫는 괄호 순서라 안쪽 괄호가 먼저 처리된다. Java 도 여기서는 valid 를 보지 않는다
|
|
151
|
+
for sp in spans:
|
|
152
|
+
if not _preceded_by_in(sp.start, s):
|
|
153
|
+
ev = _evaluate_boolean(substring(s, sp.start + 1, sp.end - 1))
|
|
154
|
+
s = substring(s, 0, sp.start) + ('1' if ev else '0') + substring(s, sp.start + 1)
|
|
155
|
+
s = _replace(s, spans, sp.start, sp.end, sp.start, sp.start)
|
|
156
|
+
return _evaluate_boolean(s)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _extract_defaults(s):
|
|
160
|
+
defaults = {}
|
|
161
|
+
pre, post = '{DEFAULT ', '}'
|
|
162
|
+
d_start = d_end = 0
|
|
163
|
+
while d_start != -1 and d_end != -1:
|
|
164
|
+
d_start = s.find(pre, d_end)
|
|
165
|
+
if d_start != -1:
|
|
166
|
+
d_end = s.find(post, d_start + len(pre))
|
|
167
|
+
if d_end != -1:
|
|
168
|
+
span = s[d_start + len(pre):d_end]
|
|
169
|
+
found = span.find('=')
|
|
170
|
+
if found != -1:
|
|
171
|
+
param = trim(span[:found])
|
|
172
|
+
if param and param[0] == '@':
|
|
173
|
+
param = param[1:]
|
|
174
|
+
defaults[param] = _remove_parentheses(trim(substring(span, found + 2)))
|
|
175
|
+
return defaults
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _substitute_parameters(s, params):
|
|
179
|
+
"""params: 삽입 순서를 지닌 dict (Java HashMap 에 넣은 순서)."""
|
|
180
|
+
defaults = _extract_defaults(s)
|
|
181
|
+
s = _DEFAULTS_RE.sub('', s)
|
|
182
|
+
params = dict(params)
|
|
183
|
+
for k in hash_order(list(defaults)):
|
|
184
|
+
if k not in params:
|
|
185
|
+
params[k] = defaults[k]
|
|
186
|
+
# HashMap 순회 순서에서 키 길이 내림차순 안정 정렬 → 값 속의 @다른키 도 Java 와 같은 순서로 치환된다
|
|
187
|
+
keys = sorted(hash_order(list(params)), key=len, reverse=True)
|
|
188
|
+
for k in keys:
|
|
189
|
+
v = params[k]
|
|
190
|
+
s = re.sub('@' + k, lambda m, v=v: v, s)
|
|
191
|
+
return s
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _parse_if_then_else(s):
|
|
195
|
+
spans = _find_spans(s, '{', '}')
|
|
196
|
+
ites = _link_if_then_elses(s, spans)
|
|
197
|
+
result = s
|
|
198
|
+
for ite in ites:
|
|
199
|
+
if ite.condition.valid:
|
|
200
|
+
cond = substring(result, ite.condition.start + 1, ite.condition.end - 1)
|
|
201
|
+
if _evaluate_condition(cond):
|
|
202
|
+
result = _replace(result, spans, ite.start(), ite.end(), ite.if_true.start + 1, ite.if_true.end - 2)
|
|
203
|
+
elif ite.has_if_false:
|
|
204
|
+
result = _replace(result, spans, ite.start(), ite.end(), ite.if_false.start + 1, ite.if_false.end - 2)
|
|
205
|
+
else:
|
|
206
|
+
result = _replace(result, spans, ite.start(), ite.end(), 0, -1)
|
|
207
|
+
return result
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def render_sql(sql, parameters=None, values=None):
|
|
211
|
+
"""SqlRender.renderSql(sql, String[] parameters, String[] values) 와 같은 결과."""
|
|
212
|
+
params = {}
|
|
213
|
+
if parameters is not None:
|
|
214
|
+
for p, v in zip(parameters, values):
|
|
215
|
+
params[p] = v
|
|
216
|
+
return _parse_if_then_else(_substitute_parameters(sql, params))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def check(sql, parameters=None, values=None):
|
|
220
|
+
return [f"Parameter '{p}' not found in SQL" for p in (parameters or []) if '@' + p not in sql]
|