sql2sqlx 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql2sqlx/__init__.py +83 -0
- sql2sqlx/__main__.py +14 -0
- sql2sqlx/cli.py +323 -0
- sql2sqlx/converter.py +1241 -0
- sql2sqlx/emitter.py +275 -0
- sql2sqlx/errors.py +93 -0
- sql2sqlx/keywords.py +169 -0
- sql2sqlx/lexer.py +494 -0
- sql2sqlx/model.py +487 -0
- sql2sqlx/parser.py +2193 -0
- sql2sqlx/py.typed +2 -0
- sql2sqlx/refs.py +902 -0
- sql2sqlx/splitter.py +360 -0
- sql2sqlx/version.py +15 -0
- sql2sqlx-0.1.0.dist-info/METADATA +733 -0
- sql2sqlx-0.1.0.dist-info/RECORD +20 -0
- sql2sqlx-0.1.0.dist-info/WHEEL +5 -0
- sql2sqlx-0.1.0.dist-info/entry_points.txt +2 -0
- sql2sqlx-0.1.0.dist-info/licenses/LICENSE +202 -0
- sql2sqlx-0.1.0.dist-info/top_level.txt +1 -0
sql2sqlx/parser.py
ADDED
|
@@ -0,0 +1,2193 @@
|
|
|
1
|
+
# Copyright (c) Soumyadip Sarkar.
|
|
2
|
+
# All rights reserved.
|
|
3
|
+
#
|
|
4
|
+
# This source code is licensed under the Apache-style license found in the
|
|
5
|
+
# LICENSE file in the root directory of this source tree.
|
|
6
|
+
|
|
7
|
+
"""Statement classification and Dataform metadata extraction.
|
|
8
|
+
|
|
9
|
+
This module decides, for every top-level statement, which Dataform action
|
|
10
|
+
type it becomes and extracts everything needed for the ``config { ... }``
|
|
11
|
+
block. The governing principle is:
|
|
12
|
+
|
|
13
|
+
**Convert when provably safe; otherwise fall back to a verbatim
|
|
14
|
+
``operations`` action and record a warning.**
|
|
15
|
+
|
|
16
|
+
An ``operations`` action executes the original SQL unchanged, so the
|
|
17
|
+
fallback path can never alter behavior - it only forgoes idiomatic
|
|
18
|
+
Dataform structure. Every fallback carries a stable warning code and a
|
|
19
|
+
human-readable reason in the conversion report.
|
|
20
|
+
|
|
21
|
+
Mapping summary
|
|
22
|
+
---------------
|
|
23
|
+
==================================== =======================================
|
|
24
|
+
Statement Result
|
|
25
|
+
==================================== =======================================
|
|
26
|
+
``CREATE [OR REPLACE] TABLE .. AS`` ``type: "table"`` (+ partition/cluster/
|
|
27
|
+
OPTIONS mapping)
|
|
28
|
+
``CREATE [MATERIALIZED] VIEW .. AS`` ``type: "view"`` (+ ``materialized``)
|
|
29
|
+
``INSERT INTO .. SELECT`` ``type: "incremental"`` (configurable)
|
|
30
|
+
``MERGE`` ``operations`` by default; provably
|
|
31
|
+
equivalent MERGEs can opt into
|
|
32
|
+
``incremental`` + ``uniqueKey``
|
|
33
|
+
``CREATE TABLE`` (no ``AS``) ``operations`` (or ``declaration``)
|
|
34
|
+
UPDATE/DELETE/TRUNCATE/DROP/ALTER/ ``operations`` with write-target
|
|
35
|
+
LOAD tracking for dependency chaining
|
|
36
|
+
everything else ``operations``
|
|
37
|
+
==================================== =======================================
|
|
38
|
+
|
|
39
|
+
Select-list aliasing
|
|
40
|
+
--------------------
|
|
41
|
+
``INSERT INTO t (a, b) SELECT x, y ...`` and ``CREATE VIEW v (a, b) AS
|
|
42
|
+
SELECT ...`` change output column names. To convert these faithfully the
|
|
43
|
+
select list is rewritten item-by-item (``x AS a, y AS b``) - but **only**
|
|
44
|
+
when every item's existing output name/alias can be determined exactly at
|
|
45
|
+
the token level. Constructs that make that ambiguous (``*`` expansion,
|
|
46
|
+
``INTERVAL 1 DAY`` implicit-alias traps, naked ``STRUCT<...>``/
|
|
47
|
+
``ARRAY<...>`` constructors, ``SELECT AS STRUCT``) trigger the operations
|
|
48
|
+
fallback instead of a guess.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import json
|
|
54
|
+
import math
|
|
55
|
+
import re
|
|
56
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
|
57
|
+
|
|
58
|
+
from sql2sqlx.keywords import RESERVED
|
|
59
|
+
from sql2sqlx.lexer import (
|
|
60
|
+
BACKTICK,
|
|
61
|
+
EOF,
|
|
62
|
+
IDENT,
|
|
63
|
+
NUMBER,
|
|
64
|
+
OP,
|
|
65
|
+
PARAM,
|
|
66
|
+
STRING,
|
|
67
|
+
Token,
|
|
68
|
+
unquote_identifier,
|
|
69
|
+
)
|
|
70
|
+
from sql2sqlx.model import (
|
|
71
|
+
ActionDraft,
|
|
72
|
+
ActionType,
|
|
73
|
+
ConversionOptions,
|
|
74
|
+
IfNotExistsStrategy,
|
|
75
|
+
InsertStrategy,
|
|
76
|
+
MergeStrategy,
|
|
77
|
+
PlainCreateStrategy,
|
|
78
|
+
RefSite,
|
|
79
|
+
TableName,
|
|
80
|
+
)
|
|
81
|
+
from sql2sqlx.refs import parse_table_path, scan_ref_sites
|
|
82
|
+
from sql2sqlx.splitter import RawStatement
|
|
83
|
+
|
|
84
|
+
#: Sentinel: an OPTIONS value kept as raw SQL text (-> ``additionalOptions``).
|
|
85
|
+
RAW = object()
|
|
86
|
+
|
|
87
|
+
_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z_0-9]*$")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _dataform_merge_column(name: str) -> bool:
|
|
91
|
+
"""Whether Dataform can emit ``name`` in an incremental MERGE.
|
|
92
|
+
|
|
93
|
+
Dataform Core 3.x quotes target/INSERT columns, but emits unique keys and
|
|
94
|
+
source-side UPDATE references without quoting. Those names must therefore
|
|
95
|
+
be simple, unreserved GoogleSQL identifiers.
|
|
96
|
+
"""
|
|
97
|
+
return bool(_IDENT_RE.fullmatch(name)) and name.upper() not in RESERVED
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _dataform_insert_column(name: str) -> bool:
|
|
101
|
+
"""Whether Dataform can safely backtick-quote an incremental column.
|
|
102
|
+
|
|
103
|
+
The BigQuery adapter wraps existing-table metadata names in raw backticks
|
|
104
|
+
without escaping identifier terminators or backslashes. Control and line
|
|
105
|
+
separator characters are rejected as well so the generated runtime SQL
|
|
106
|
+
cannot be split or reinterpreted.
|
|
107
|
+
"""
|
|
108
|
+
return bool(name) and all(
|
|
109
|
+
character not in ("`", "\\")
|
|
110
|
+
and ord(character) >= 0x20
|
|
111
|
+
and not (0x7F <= ord(character) <= 0x9F)
|
|
112
|
+
and ord(character) not in (0x2028, 0x2029)
|
|
113
|
+
for character in name
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
#: Reserved keywords that terminate an expression and may precede an
|
|
118
|
+
#: implicit alias (``SELECT x IS NULL flag`` -> alias ``flag``).
|
|
119
|
+
_EXPRESSION_END_KEYWORDS = frozenset({"TRUE", "FALSE", "NULL", "END"})
|
|
120
|
+
|
|
121
|
+
#: Keywords ending a select list at nesting depth 0.
|
|
122
|
+
_SELECT_LIST_ENDERS = frozenset(
|
|
123
|
+
{
|
|
124
|
+
"FROM",
|
|
125
|
+
"UNION",
|
|
126
|
+
"INTERSECT",
|
|
127
|
+
"EXCEPT",
|
|
128
|
+
"LIMIT",
|
|
129
|
+
"ORDER",
|
|
130
|
+
"WHERE",
|
|
131
|
+
"GROUP",
|
|
132
|
+
"HAVING",
|
|
133
|
+
"QUALIFY",
|
|
134
|
+
"WINDOW",
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# ---------------------------------------------------------------------------
|
|
140
|
+
# Small token utilities
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _with_eof(stmt: RawStatement) -> List[Token]:
|
|
145
|
+
"""Return the statement's tokens with a guaranteed trailing EOF token.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
stmt: The raw statement.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
The token list, EOF-terminated (a synthetic EOF is appended when
|
|
152
|
+
the splitter's slice did not include one).
|
|
153
|
+
"""
|
|
154
|
+
toks = stmt.tokens
|
|
155
|
+
if toks and toks[-1].kind == EOF:
|
|
156
|
+
return list(toks)
|
|
157
|
+
return list(toks) + [Token(EOF, "", stmt.end, stmt.end)]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _kw(toks: Sequence[Token], i: int) -> str:
|
|
161
|
+
"""Uppercase text of ``toks[i]`` if it is an identifier, else ``""``.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
toks: Token list.
|
|
165
|
+
i: Index (out-of-range indices are safe).
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
The uppercased identifier text, or the empty string.
|
|
169
|
+
"""
|
|
170
|
+
if 0 <= i < len(toks) and toks[i].kind == IDENT:
|
|
171
|
+
return toks[i].upper
|
|
172
|
+
return ""
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _is_op(toks: Sequence[Token], i: int, text: str) -> bool:
|
|
176
|
+
"""True when ``toks[i]`` is the operator/punctuation ``text``."""
|
|
177
|
+
return 0 <= i < len(toks) and toks[i].kind == OP and toks[i].text == text
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _skip_balanced(toks: Sequence[Token], i: int) -> int:
|
|
181
|
+
"""Skip a balanced parenthesis group starting at ``toks[i] == '('``.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
toks: Token list (EOF-terminated).
|
|
185
|
+
i: Index of the opening parenthesis.
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
Index of the first token after the matching ``)``. If the group is
|
|
189
|
+
unbalanced (invalid SQL), the EOF index is returned so callers
|
|
190
|
+
terminate gracefully.
|
|
191
|
+
"""
|
|
192
|
+
if not _is_op(toks, i, "("):
|
|
193
|
+
return i + 1
|
|
194
|
+
depth = 0
|
|
195
|
+
n = len(toks)
|
|
196
|
+
while i < n:
|
|
197
|
+
t = toks[i]
|
|
198
|
+
if t.kind == EOF:
|
|
199
|
+
break
|
|
200
|
+
if t.kind == OP:
|
|
201
|
+
if t.text == "(":
|
|
202
|
+
depth += 1
|
|
203
|
+
elif t.text == ")":
|
|
204
|
+
depth -= 1
|
|
205
|
+
if depth == 0:
|
|
206
|
+
return i + 1
|
|
207
|
+
i += 1
|
|
208
|
+
return n - 1
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _body_head_ok(toks: Sequence[Token], i: int) -> bool:
|
|
212
|
+
"""True when ``toks[i]`` can start a query body.
|
|
213
|
+
|
|
214
|
+
Accepts ``SELECT``, ``WITH``, a parenthesized query, and ``FROM``
|
|
215
|
+
(BigQuery pipe-syntax queries start with ``FROM``).
|
|
216
|
+
"""
|
|
217
|
+
if i >= len(toks):
|
|
218
|
+
return False
|
|
219
|
+
t = toks[i]
|
|
220
|
+
if t.kind == OP and t.text == "(":
|
|
221
|
+
return True
|
|
222
|
+
return t.kind == IDENT and t.upper in ("SELECT", "WITH", "FROM")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _quote_ident(name: str) -> str:
|
|
226
|
+
"""Render ``name`` as a safe SQL identifier (backticked if needed).
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
name: The identifier text (already decoded).
|
|
230
|
+
|
|
231
|
+
Returns:
|
|
232
|
+
``name`` unchanged when it is a plain unreserved identifier,
|
|
233
|
+
otherwise a backtick-quoted, escaped form.
|
|
234
|
+
"""
|
|
235
|
+
if _IDENT_RE.match(name) and name.upper() not in RESERVED:
|
|
236
|
+
return name
|
|
237
|
+
control_escapes = {
|
|
238
|
+
"\a": "\\a",
|
|
239
|
+
"\b": "\\b",
|
|
240
|
+
"\f": "\\f",
|
|
241
|
+
"\n": "\\n",
|
|
242
|
+
"\r": "\\r",
|
|
243
|
+
"\t": "\\t",
|
|
244
|
+
"\v": "\\v",
|
|
245
|
+
}
|
|
246
|
+
escaped = "".join(
|
|
247
|
+
(
|
|
248
|
+
"\\\\"
|
|
249
|
+
if char == "\\"
|
|
250
|
+
else (
|
|
251
|
+
"\\`"
|
|
252
|
+
if char == "`"
|
|
253
|
+
else control_escapes.get(
|
|
254
|
+
char,
|
|
255
|
+
(
|
|
256
|
+
f"\\x{ord(char):02X}"
|
|
257
|
+
if ord(char) < 0x20 or 0x7F <= ord(char) <= 0x9F
|
|
258
|
+
else f"\\u{ord(char):04X}" if ord(char) in (0x2028, 0x2029) else char
|
|
259
|
+
),
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
for char in name
|
|
264
|
+
)
|
|
265
|
+
quoted = "`" + escaped + "`"
|
|
266
|
+
# A SQLX placeholder cannot sit between BigQuery backticks. If the
|
|
267
|
+
# generated identifier contains `${`, emit the entire token as a constant
|
|
268
|
+
# JavaScript expression so compilation reconstructs it exactly.
|
|
269
|
+
if "${" in quoted:
|
|
270
|
+
return "${" + json.dumps(quoted, ensure_ascii=False) + "}"
|
|
271
|
+
return quoted
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
# ---------------------------------------------------------------------------
|
|
275
|
+
# String / OPTIONS parsing
|
|
276
|
+
# ---------------------------------------------------------------------------
|
|
277
|
+
|
|
278
|
+
_SIMPLE_ESCAPES = {
|
|
279
|
+
"a": "\a",
|
|
280
|
+
"b": "\b",
|
|
281
|
+
"f": "\f",
|
|
282
|
+
"n": "\n",
|
|
283
|
+
"r": "\r",
|
|
284
|
+
"t": "\t",
|
|
285
|
+
"v": "\v",
|
|
286
|
+
"\\": "\\",
|
|
287
|
+
"?": "?",
|
|
288
|
+
'"': '"',
|
|
289
|
+
"'": "'",
|
|
290
|
+
"`": "`",
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
_STRING_HEAD_RE = re.compile(r"([rRbB]{0,2})('''|\"\"\"|'|\")")
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _decode_string(literal: str) -> Optional[str]:
|
|
297
|
+
"""Decode a GoogleSQL string literal token to a Python string.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
literal: The exact token text, including quotes and any prefix.
|
|
301
|
+
|
|
302
|
+
Returns:
|
|
303
|
+
The decoded text, or ``None`` for bytes literals and malformed
|
|
304
|
+
input (callers then treat the value as raw SQL).
|
|
305
|
+
"""
|
|
306
|
+
m = _STRING_HEAD_RE.match(literal)
|
|
307
|
+
if not m:
|
|
308
|
+
return None
|
|
309
|
+
prefix = m.group(1).lower()
|
|
310
|
+
if "b" in prefix:
|
|
311
|
+
return None
|
|
312
|
+
quote = m.group(2)
|
|
313
|
+
body = literal[m.end() : -len(quote)]
|
|
314
|
+
if "r" in prefix or "\\" not in body:
|
|
315
|
+
return body
|
|
316
|
+
out: List[str] = []
|
|
317
|
+
i, size = 0, len(body)
|
|
318
|
+
while i < size:
|
|
319
|
+
c = body[i]
|
|
320
|
+
if c != "\\" or i + 1 >= size:
|
|
321
|
+
out.append(c)
|
|
322
|
+
i += 1
|
|
323
|
+
continue
|
|
324
|
+
nxt = body[i + 1]
|
|
325
|
+
if nxt in _SIMPLE_ESCAPES:
|
|
326
|
+
out.append(_SIMPLE_ESCAPES[nxt])
|
|
327
|
+
i += 2
|
|
328
|
+
elif nxt in ("x", "X"):
|
|
329
|
+
try:
|
|
330
|
+
out.append(chr(int(body[i + 2 : i + 4], 16)))
|
|
331
|
+
i += 4
|
|
332
|
+
except ValueError:
|
|
333
|
+
out.append(nxt)
|
|
334
|
+
i += 2
|
|
335
|
+
elif nxt == "u":
|
|
336
|
+
try:
|
|
337
|
+
out.append(chr(int(body[i + 2 : i + 6], 16)))
|
|
338
|
+
i += 6
|
|
339
|
+
except ValueError:
|
|
340
|
+
out.append(nxt)
|
|
341
|
+
i += 2
|
|
342
|
+
elif nxt == "U":
|
|
343
|
+
try:
|
|
344
|
+
out.append(chr(int(body[i + 2 : i + 10], 16)))
|
|
345
|
+
i += 10
|
|
346
|
+
except ValueError:
|
|
347
|
+
out.append(nxt)
|
|
348
|
+
i += 2
|
|
349
|
+
elif nxt in "01234567":
|
|
350
|
+
j = i + 1
|
|
351
|
+
while j < min(i + 4, size) and body[j] in "01234567":
|
|
352
|
+
j += 1
|
|
353
|
+
out.append(chr(int(body[i + 1 : j], 8)))
|
|
354
|
+
i = j
|
|
355
|
+
else:
|
|
356
|
+
out.append(nxt)
|
|
357
|
+
i += 2
|
|
358
|
+
return "".join(out)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _num(s: str) -> Any:
|
|
362
|
+
"""Parse a NUMBER token's text into ``int``/``float`` (RAW on failure)."""
|
|
363
|
+
try:
|
|
364
|
+
if re.fullmatch(r"0[xX][0-9A-Fa-f]+", s):
|
|
365
|
+
return int(s, 16)
|
|
366
|
+
if re.fullmatch(r"\d+", s):
|
|
367
|
+
return int(s)
|
|
368
|
+
value = float(s)
|
|
369
|
+
return value if math.isfinite(value) else RAW
|
|
370
|
+
except ValueError: # pragma: no cover - lexer guarantees numeric shape
|
|
371
|
+
return RAW
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _parse_label_array(vtoks: Sequence[Token]) -> Optional[Dict[str, str]]:
|
|
375
|
+
"""Strictly parse a labels value ``[("k", "v"), ...]`` into a dict.
|
|
376
|
+
|
|
377
|
+
Args:
|
|
378
|
+
vtoks: The value's tokens.
|
|
379
|
+
|
|
380
|
+
Returns:
|
|
381
|
+
The label mapping, or ``None`` if the value has any other shape
|
|
382
|
+
(the caller then keeps the raw SQL in ``additionalOptions``).
|
|
383
|
+
"""
|
|
384
|
+
if len(vtoks) < 2:
|
|
385
|
+
return None
|
|
386
|
+
if not (vtoks[0].kind == OP and vtoks[0].text == "["):
|
|
387
|
+
return None
|
|
388
|
+
if not (vtoks[-1].kind == OP and vtoks[-1].text == "]"):
|
|
389
|
+
return None
|
|
390
|
+
out: Dict[str, str] = {}
|
|
391
|
+
j, end = 1, len(vtoks) - 1
|
|
392
|
+
while j < end:
|
|
393
|
+
if j + 5 > end or not (vtoks[j].kind == OP and vtoks[j].text == "("):
|
|
394
|
+
return None
|
|
395
|
+
k, comma, v, close = vtoks[j + 1], vtoks[j + 2], vtoks[j + 3], vtoks[j + 4]
|
|
396
|
+
if k.kind != STRING or v.kind != STRING:
|
|
397
|
+
return None
|
|
398
|
+
if not (comma.kind == OP and comma.text == ","):
|
|
399
|
+
return None
|
|
400
|
+
if not (close.kind == OP and close.text == ")"):
|
|
401
|
+
return None
|
|
402
|
+
ks, vs = _decode_string(k.text), _decode_string(v.text)
|
|
403
|
+
if ks is None or vs is None:
|
|
404
|
+
return None
|
|
405
|
+
out[ks] = vs
|
|
406
|
+
j += 5
|
|
407
|
+
if j < end:
|
|
408
|
+
if not (vtoks[j].kind == OP and vtoks[j].text == ","):
|
|
409
|
+
return None
|
|
410
|
+
j += 1
|
|
411
|
+
return out
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _parse_option_value(vtoks: Sequence[Token]) -> Any:
|
|
415
|
+
"""Interpret an OPTIONS value's tokens as a simple literal if possible.
|
|
416
|
+
|
|
417
|
+
Args:
|
|
418
|
+
vtoks: The value's tokens (never empty).
|
|
419
|
+
|
|
420
|
+
Returns:
|
|
421
|
+
``str``/``int``/``float``/``bool``/``dict`` for recognized literal
|
|
422
|
+
shapes, otherwise the :data:`RAW` sentinel.
|
|
423
|
+
"""
|
|
424
|
+
if len(vtoks) == 1:
|
|
425
|
+
t = vtoks[0]
|
|
426
|
+
if t.kind == STRING:
|
|
427
|
+
s = _decode_string(t.text)
|
|
428
|
+
return s if s is not None else RAW
|
|
429
|
+
if t.kind == NUMBER:
|
|
430
|
+
return _num(t.text)
|
|
431
|
+
if t.kind == IDENT:
|
|
432
|
+
if t.upper == "TRUE":
|
|
433
|
+
return True
|
|
434
|
+
if t.upper == "FALSE":
|
|
435
|
+
return False
|
|
436
|
+
return RAW
|
|
437
|
+
if len(vtoks) == 2 and vtoks[0].kind == OP and vtoks[0].text == "-" and vtoks[1].kind == NUMBER:
|
|
438
|
+
v = _num(vtoks[1].text)
|
|
439
|
+
return -v if isinstance(v, (int, float)) else RAW
|
|
440
|
+
labels = _parse_label_array(vtoks)
|
|
441
|
+
return labels if labels is not None else RAW
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _parse_options(
|
|
445
|
+
toks: Sequence[Token], i: int, text: str
|
|
446
|
+
) -> Optional[Tuple[List[Tuple[str, str, Any]], int]]:
|
|
447
|
+
"""Parse an ``OPTIONS(key = value, ...)`` clause.
|
|
448
|
+
|
|
449
|
+
Args:
|
|
450
|
+
toks: Token list; ``toks[i]`` must be the ``OPTIONS`` identifier.
|
|
451
|
+
i: Index of ``OPTIONS``.
|
|
452
|
+
text: Full source text (for raw value capture).
|
|
453
|
+
|
|
454
|
+
Returns:
|
|
455
|
+
``(entries, next_index)`` where each entry is
|
|
456
|
+
``(key, raw_sql_text, parsed_value_or_RAW)``, or ``None`` when the
|
|
457
|
+
clause cannot be parsed (caller falls back to operations).
|
|
458
|
+
"""
|
|
459
|
+
j = i + 1
|
|
460
|
+
if not _is_op(toks, j, "("):
|
|
461
|
+
return None
|
|
462
|
+
j += 1
|
|
463
|
+
entries: List[Tuple[str, str, Any]] = []
|
|
464
|
+
n = len(toks)
|
|
465
|
+
while j < n:
|
|
466
|
+
if _is_op(toks, j, ")"):
|
|
467
|
+
return entries, j + 1
|
|
468
|
+
kt = toks[j]
|
|
469
|
+
if kt.kind not in (IDENT, BACKTICK):
|
|
470
|
+
return None
|
|
471
|
+
key = unquote_identifier(kt.text)
|
|
472
|
+
j += 1
|
|
473
|
+
if not _is_op(toks, j, "="):
|
|
474
|
+
return None
|
|
475
|
+
j += 1
|
|
476
|
+
vstart = j
|
|
477
|
+
depth = 0
|
|
478
|
+
while j < n and toks[j].kind != EOF:
|
|
479
|
+
t = toks[j]
|
|
480
|
+
if t.kind == OP:
|
|
481
|
+
if t.text in ("(", "["):
|
|
482
|
+
depth += 1
|
|
483
|
+
elif t.text in (")", "]"):
|
|
484
|
+
if depth == 0 and t.text == ")":
|
|
485
|
+
break
|
|
486
|
+
depth -= 1
|
|
487
|
+
elif t.text == "," and depth == 0:
|
|
488
|
+
break
|
|
489
|
+
j += 1
|
|
490
|
+
if j >= n or toks[j].kind == EOF or j == vstart:
|
|
491
|
+
return None
|
|
492
|
+
vtoks = toks[vstart:j]
|
|
493
|
+
raw = text[vtoks[0].start : vtoks[-1].end]
|
|
494
|
+
entries.append((key, raw, _parse_option_value(vtoks)))
|
|
495
|
+
if _is_op(toks, j, ","):
|
|
496
|
+
j += 1
|
|
497
|
+
return None
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
# ---------------------------------------------------------------------------
|
|
501
|
+
# Select-list machinery (shared by INSERT column lists, VIEW column lists
|
|
502
|
+
# and the safe-MERGE analyzer)
|
|
503
|
+
# ---------------------------------------------------------------------------
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
class _SelectItem:
|
|
507
|
+
"""One top-level select-list item and its safety flags.
|
|
508
|
+
|
|
509
|
+
Attributes:
|
|
510
|
+
toks: The item's tokens.
|
|
511
|
+
interval: True when a top-level ``INTERVAL`` keyword makes
|
|
512
|
+
implicit-alias detection ambiguous.
|
|
513
|
+
generic: True when a naked ``STRUCT<``/``ARRAY<`` constructor makes
|
|
514
|
+
comma splitting ambiguous.
|
|
515
|
+
star: True when the item is a ``*`` / ``t.*`` expansion.
|
|
516
|
+
"""
|
|
517
|
+
|
|
518
|
+
__slots__ = ("toks", "interval", "generic", "star")
|
|
519
|
+
|
|
520
|
+
def __init__(self) -> None:
|
|
521
|
+
"""Create an empty item."""
|
|
522
|
+
self.toks: List[Token] = []
|
|
523
|
+
self.interval = False
|
|
524
|
+
self.generic = False
|
|
525
|
+
self.star = False
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _split_select_items(toks: Sequence[Token], ts: int, te: int) -> Optional[List[_SelectItem]]:
|
|
529
|
+
"""Split the select list of the query spanning ``toks[ts:te)``.
|
|
530
|
+
|
|
531
|
+
Handles wrapper parentheses around the whole query, a leading ``WITH``
|
|
532
|
+
clause (CTEs are skipped, not inspected), and ``DISTINCT``/``ALL``
|
|
533
|
+
modifiers. BigQuery's trailing comma before ``FROM`` is accepted.
|
|
534
|
+
|
|
535
|
+
Args:
|
|
536
|
+
toks: Token list (EOF-terminated).
|
|
537
|
+
ts: Index of the query's first token.
|
|
538
|
+
te: Exclusive end index of the query (usually the EOF index).
|
|
539
|
+
|
|
540
|
+
Returns:
|
|
541
|
+
The list of items, or ``None`` for shapes that cannot be handled
|
|
542
|
+
(missing ``SELECT``, ``SELECT AS STRUCT/VALUE``,
|
|
543
|
+
``SELECT WITH DIFFERENTIAL_PRIVACY``, malformed ``WITH``...).
|
|
544
|
+
"""
|
|
545
|
+
n = len(toks)
|
|
546
|
+
te = min(te, n)
|
|
547
|
+
while (
|
|
548
|
+
ts < te
|
|
549
|
+
and _is_op(toks, ts, "(")
|
|
550
|
+
and _skip_balanced(toks, ts) == te
|
|
551
|
+
and _is_op(toks, te - 1, ")")
|
|
552
|
+
):
|
|
553
|
+
ts += 1
|
|
554
|
+
te -= 1
|
|
555
|
+
j = ts
|
|
556
|
+
if _kw(toks, j) == "WITH":
|
|
557
|
+
j += 1
|
|
558
|
+
if _kw(toks, j) == "RECURSIVE":
|
|
559
|
+
j += 1
|
|
560
|
+
while True:
|
|
561
|
+
if j >= te or toks[j].kind not in (IDENT, BACKTICK):
|
|
562
|
+
return None
|
|
563
|
+
if toks[j].kind == IDENT and toks[j].upper in RESERVED:
|
|
564
|
+
return None
|
|
565
|
+
j += 1
|
|
566
|
+
if _is_op(toks, j, "("):
|
|
567
|
+
j = _skip_balanced(toks, j)
|
|
568
|
+
if _kw(toks, j) != "AS":
|
|
569
|
+
return None
|
|
570
|
+
j += 1
|
|
571
|
+
if not _is_op(toks, j, "("):
|
|
572
|
+
return None
|
|
573
|
+
j = _skip_balanced(toks, j)
|
|
574
|
+
if _is_op(toks, j, ","):
|
|
575
|
+
j += 1
|
|
576
|
+
continue
|
|
577
|
+
break
|
|
578
|
+
if _kw(toks, j) != "SELECT":
|
|
579
|
+
return None
|
|
580
|
+
j += 1
|
|
581
|
+
if _kw(toks, j) in ("DISTINCT", "ALL"):
|
|
582
|
+
j += 1
|
|
583
|
+
if _kw(toks, j) in ("AS", "WITH"):
|
|
584
|
+
return None
|
|
585
|
+
items: List[_SelectItem] = []
|
|
586
|
+
cur = _SelectItem()
|
|
587
|
+
depth = 0
|
|
588
|
+
k = j
|
|
589
|
+
while k < te:
|
|
590
|
+
t = toks[k]
|
|
591
|
+
if t.kind == EOF:
|
|
592
|
+
break
|
|
593
|
+
if t.kind == OP:
|
|
594
|
+
x = t.text
|
|
595
|
+
if x in ("(", "["):
|
|
596
|
+
depth += 1
|
|
597
|
+
elif x in (")", "]"):
|
|
598
|
+
depth -= 1
|
|
599
|
+
if depth < 0:
|
|
600
|
+
break
|
|
601
|
+
elif depth == 0:
|
|
602
|
+
if x == ",":
|
|
603
|
+
if not cur.toks:
|
|
604
|
+
return None
|
|
605
|
+
items.append(cur)
|
|
606
|
+
cur = _SelectItem()
|
|
607
|
+
k += 1
|
|
608
|
+
continue
|
|
609
|
+
if x == "|>":
|
|
610
|
+
break
|
|
611
|
+
if x == "*":
|
|
612
|
+
prev = cur.toks[-1] if cur.toks else None
|
|
613
|
+
if prev is None or (prev.kind == OP and prev.text == "."):
|
|
614
|
+
cur.star = True
|
|
615
|
+
elif t.kind == IDENT and depth == 0:
|
|
616
|
+
u = t.upper
|
|
617
|
+
if u in _SELECT_LIST_ENDERS:
|
|
618
|
+
break
|
|
619
|
+
if u == "INTERVAL":
|
|
620
|
+
cur.interval = True
|
|
621
|
+
elif u in ("STRUCT", "ARRAY") and _is_op(toks, k + 1, "<"):
|
|
622
|
+
cur.generic = True
|
|
623
|
+
cur.toks.append(t)
|
|
624
|
+
k += 1
|
|
625
|
+
if cur.toks:
|
|
626
|
+
items.append(cur)
|
|
627
|
+
if not items:
|
|
628
|
+
return None
|
|
629
|
+
return items
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def _item_output_info(item: _SelectItem) -> Optional[Tuple[str, Optional[str], Optional[Token]]]:
|
|
633
|
+
"""Determine a select-list item's output name and alias structure.
|
|
634
|
+
|
|
635
|
+
Args:
|
|
636
|
+
item: The item to analyze.
|
|
637
|
+
|
|
638
|
+
Returns:
|
|
639
|
+
``(mode, name, alias_token)`` where mode is one of:
|
|
640
|
+
|
|
641
|
+
* ``"explicit"`` - ``expr AS alias`` (name = alias);
|
|
642
|
+
* ``"implicit"`` - ``expr alias`` (name = alias);
|
|
643
|
+
* ``"bare"`` - a plain column/path (name = last segment);
|
|
644
|
+
* ``"anon"`` - an expression with no derivable output name.
|
|
645
|
+
|
|
646
|
+
Returns ``None`` when the item is *unsafe* to reason about
|
|
647
|
+
(star expansion, ambiguous ``INTERVAL``/generic constructors).
|
|
648
|
+
"""
|
|
649
|
+
if item.star:
|
|
650
|
+
return None
|
|
651
|
+
t = item.toks
|
|
652
|
+
last = t[-1]
|
|
653
|
+
if len(t) == 1:
|
|
654
|
+
if last.kind == BACKTICK:
|
|
655
|
+
return ("bare", unquote_identifier(last.text), None)
|
|
656
|
+
if last.kind == IDENT and last.upper not in RESERVED:
|
|
657
|
+
return ("bare", last.text, None)
|
|
658
|
+
return ("anon", None, None)
|
|
659
|
+
penult = t[-2]
|
|
660
|
+
last_is_name = last.kind == BACKTICK or (last.kind == IDENT and last.upper not in RESERVED)
|
|
661
|
+
if penult.kind == IDENT and penult.upper == "AS" and last_is_name:
|
|
662
|
+
return ("explicit", unquote_identifier(last.text), last)
|
|
663
|
+
if last_is_name:
|
|
664
|
+
if penult.kind == OP and penult.text == ".":
|
|
665
|
+
return ("bare", unquote_identifier(last.text), None)
|
|
666
|
+
if item.interval or item.generic:
|
|
667
|
+
return None
|
|
668
|
+
terminal = (
|
|
669
|
+
penult.kind in (BACKTICK, NUMBER, STRING, PARAM)
|
|
670
|
+
or (penult.kind == OP and penult.text in (")", "]"))
|
|
671
|
+
or (
|
|
672
|
+
penult.kind == IDENT
|
|
673
|
+
and (penult.upper not in RESERVED or penult.upper in _EXPRESSION_END_KEYWORDS)
|
|
674
|
+
)
|
|
675
|
+
)
|
|
676
|
+
if terminal:
|
|
677
|
+
return ("implicit", unquote_identifier(last.text), last)
|
|
678
|
+
return ("anon", None, None)
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _alias_select_list_edits(
|
|
682
|
+
toks: Sequence[Token], ts: int, te: int, cols: Sequence[str]
|
|
683
|
+
) -> Optional[List[Tuple[int, int, str]]]:
|
|
684
|
+
"""Compute span edits forcing the select list to output ``cols``.
|
|
685
|
+
|
|
686
|
+
Args:
|
|
687
|
+
toks: Token list (EOF-terminated).
|
|
688
|
+
ts: Index of the query's first token.
|
|
689
|
+
te: Exclusive end index of the query.
|
|
690
|
+
cols: Required output column names, in order.
|
|
691
|
+
|
|
692
|
+
Returns:
|
|
693
|
+
Absolute-offset span edits, or ``None`` when the rewrite is not
|
|
694
|
+
provably safe (arity mismatch, star expansion, ambiguous items).
|
|
695
|
+
"""
|
|
696
|
+
items = _split_select_items(toks, ts, te)
|
|
697
|
+
if items is None or len(items) != len(cols):
|
|
698
|
+
return None
|
|
699
|
+
infos = []
|
|
700
|
+
for it in items:
|
|
701
|
+
info = _item_output_info(it)
|
|
702
|
+
if info is None:
|
|
703
|
+
return None
|
|
704
|
+
infos.append(info)
|
|
705
|
+
changed_aliases = set()
|
|
706
|
+
for (mode, name, _alias), col in zip(infos, cols):
|
|
707
|
+
if name is not None and name.upper() == col.upper():
|
|
708
|
+
continue # no edit for this item
|
|
709
|
+
if mode in ("explicit", "implicit") and name is not None:
|
|
710
|
+
# The old alias disappears; later references to it would
|
|
711
|
+
# re-resolve (or break).
|
|
712
|
+
changed_aliases.add(name.upper())
|
|
713
|
+
# The new alias appears; BigQuery resolves SELECT aliases in
|
|
714
|
+
# preference to FROM columns inside GROUP BY/HAVING/QUALIFY/
|
|
715
|
+
# ORDER BY, so a same-named column reference there would silently
|
|
716
|
+
# start meaning this item's expression instead.
|
|
717
|
+
changed_aliases.add(col.upper())
|
|
718
|
+
if changed_aliases:
|
|
719
|
+
# SELECT aliases are visible to GROUP BY, HAVING, QUALIFY, ORDER BY
|
|
720
|
+
# and subsequent pipe operators. Adding, removing or replacing an
|
|
721
|
+
# alias without rewriting such references would either change which
|
|
722
|
+
# expression is used or make the query invalid. Keep the original
|
|
723
|
+
# statement as operations instead of guessing.
|
|
724
|
+
list_end = max(item.toks[-1].end for item in items)
|
|
725
|
+
depth = 0
|
|
726
|
+
alias_visible = False
|
|
727
|
+
for token in toks:
|
|
728
|
+
if token.start < list_end or token.kind == EOF or token.start >= toks[te].start:
|
|
729
|
+
continue
|
|
730
|
+
if token.kind == OP:
|
|
731
|
+
if token.text in ("(", "["):
|
|
732
|
+
depth += 1
|
|
733
|
+
elif token.text in (")", "]"):
|
|
734
|
+
depth = max(0, depth - 1)
|
|
735
|
+
elif token.text == "|>" and depth == 0:
|
|
736
|
+
alias_visible = True
|
|
737
|
+
continue
|
|
738
|
+
if token.kind not in (IDENT, BACKTICK):
|
|
739
|
+
continue
|
|
740
|
+
word = token.upper if token.kind == IDENT else unquote_identifier(token.text).upper()
|
|
741
|
+
if (
|
|
742
|
+
depth == 0
|
|
743
|
+
and token.kind == IDENT
|
|
744
|
+
and word in ("GROUP", "HAVING", "QUALIFY", "ORDER")
|
|
745
|
+
):
|
|
746
|
+
alias_visible = True
|
|
747
|
+
continue
|
|
748
|
+
if (
|
|
749
|
+
depth == 0
|
|
750
|
+
and token.kind == IDENT
|
|
751
|
+
and word in ("LIMIT", "UNION", "INTERSECT", "EXCEPT")
|
|
752
|
+
):
|
|
753
|
+
alias_visible = False
|
|
754
|
+
continue
|
|
755
|
+
if alias_visible and word in changed_aliases:
|
|
756
|
+
return None
|
|
757
|
+
edits: List[Tuple[int, int, str]] = []
|
|
758
|
+
for it, (_mode, name, alias_tok), col in zip(items, infos, cols):
|
|
759
|
+
if name is not None and name.upper() == col.upper():
|
|
760
|
+
continue
|
|
761
|
+
quoted = _quote_ident(col)
|
|
762
|
+
if alias_tok is not None:
|
|
763
|
+
edits.append((alias_tok.start, alias_tok.end, quoted))
|
|
764
|
+
else:
|
|
765
|
+
end = it.toks[-1].end
|
|
766
|
+
edits.append((end, end, " AS " + quoted))
|
|
767
|
+
return edits
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _select_output_names(toks: Sequence[Token], ts: int, te: int) -> Optional[List[str]]:
|
|
771
|
+
"""Derive the output column names of the query in ``toks[ts:te)``.
|
|
772
|
+
|
|
773
|
+
Args:
|
|
774
|
+
toks: Token list.
|
|
775
|
+
ts: Index of the query's first token.
|
|
776
|
+
te: Exclusive end index.
|
|
777
|
+
|
|
778
|
+
Returns:
|
|
779
|
+
The names, or ``None`` when any item's name cannot be determined
|
|
780
|
+
exactly.
|
|
781
|
+
"""
|
|
782
|
+
items = _split_select_items(toks, ts, te)
|
|
783
|
+
if items is None:
|
|
784
|
+
return None
|
|
785
|
+
names: List[str] = []
|
|
786
|
+
for it in items:
|
|
787
|
+
info = _item_output_info(it)
|
|
788
|
+
if info is None or info[1] is None:
|
|
789
|
+
return None
|
|
790
|
+
names.append(info[1])
|
|
791
|
+
return names
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def _parse_ident_list(toks: Sequence[Token], i: int) -> Tuple[Optional[List[str]], int]:
|
|
795
|
+
"""Parse a plain parenthesized name list ``(a, b, ...)``.
|
|
796
|
+
|
|
797
|
+
Args:
|
|
798
|
+
toks: Token list; ``toks[i]`` must be ``(``.
|
|
799
|
+
i: Index of the opening parenthesis.
|
|
800
|
+
|
|
801
|
+
Returns:
|
|
802
|
+
``(names, next_index)`` on success, ``(None, i)`` otherwise.
|
|
803
|
+
"""
|
|
804
|
+
j = i + 1
|
|
805
|
+
names: List[str] = []
|
|
806
|
+
n = len(toks)
|
|
807
|
+
while j < n:
|
|
808
|
+
t = toks[j]
|
|
809
|
+
if t.kind == BACKTICK:
|
|
810
|
+
names.append(unquote_identifier(t.text))
|
|
811
|
+
elif t.kind == IDENT and t.upper not in RESERVED:
|
|
812
|
+
names.append(t.text)
|
|
813
|
+
else:
|
|
814
|
+
return None, i
|
|
815
|
+
j += 1
|
|
816
|
+
if _is_op(toks, j, ","):
|
|
817
|
+
j += 1
|
|
818
|
+
continue
|
|
819
|
+
if _is_op(toks, j, ")"):
|
|
820
|
+
return names, j + 1
|
|
821
|
+
return None, i
|
|
822
|
+
return None, i
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
# ---------------------------------------------------------------------------
|
|
826
|
+
# Draft helpers
|
|
827
|
+
# ---------------------------------------------------------------------------
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _ops_draft(
|
|
831
|
+
stmt: RawStatement,
|
|
832
|
+
kind: str,
|
|
833
|
+
*,
|
|
834
|
+
target: Optional[TableName] = None,
|
|
835
|
+
writes: bool = False,
|
|
836
|
+
tspan: Optional[Tuple[int, int]] = None,
|
|
837
|
+
warnings: Optional[List[Tuple[str, str, int]]] = None,
|
|
838
|
+
) -> ActionDraft:
|
|
839
|
+
"""Build a verbatim ``operations`` draft for ``stmt``.
|
|
840
|
+
|
|
841
|
+
Args:
|
|
842
|
+
stmt: The raw statement.
|
|
843
|
+
kind: Uppercase original statement kind (for annotation/report).
|
|
844
|
+
target: Table written by the statement, if identifiable.
|
|
845
|
+
writes: True when the statement mutates ``target``.
|
|
846
|
+
tspan: Span of the target path (enables ``${self()}`` substitution
|
|
847
|
+
if the linker elects this action as the target's producer).
|
|
848
|
+
warnings: ``(code, message, offset)`` findings.
|
|
849
|
+
|
|
850
|
+
Returns:
|
|
851
|
+
The draft.
|
|
852
|
+
"""
|
|
853
|
+
return ActionDraft(
|
|
854
|
+
action_type=ActionType.OPERATIONS,
|
|
855
|
+
target=target,
|
|
856
|
+
writes_target=writes,
|
|
857
|
+
creates_target=False,
|
|
858
|
+
body_start=stmt.start,
|
|
859
|
+
body_end=stmt.end,
|
|
860
|
+
stmt_start=stmt.start,
|
|
861
|
+
stmt_end=stmt.terminator_end,
|
|
862
|
+
primary_target_span=tspan,
|
|
863
|
+
original_kind=kind,
|
|
864
|
+
warnings=list(warnings or []),
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def _finish(
|
|
869
|
+
draft: ActionDraft, toks: Sequence[Token], head: str, skip_spans: Sequence[Tuple[int, int]] = ()
|
|
870
|
+
) -> ActionDraft:
|
|
871
|
+
"""Attach reference sites to a draft and return it.
|
|
872
|
+
|
|
873
|
+
Args:
|
|
874
|
+
draft: The draft under construction.
|
|
875
|
+
toks: The statement's tokens.
|
|
876
|
+
head: Uppercase leading keyword (``"MERGE"`` enables the USING
|
|
877
|
+
table introducer in the scanner).
|
|
878
|
+
skip_spans: Spans that must not yield reference sites (e.g. the
|
|
879
|
+
``DELETE FROM`` target).
|
|
880
|
+
|
|
881
|
+
Returns:
|
|
882
|
+
The same draft, with ``ref_sites`` populated.
|
|
883
|
+
"""
|
|
884
|
+
draft.ref_sites = scan_ref_sites(toks, head, skip_spans)
|
|
885
|
+
return draft
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _append_table_ref(draft: ActionDraft, toks: Sequence[Token], i: int) -> None:
|
|
889
|
+
"""Append a table reference introduced by a DDL-specific keyword."""
|
|
890
|
+
match = parse_table_path(toks, i)
|
|
891
|
+
if match is None or not (1 <= len(match.parts) <= 3) or not all(match.parts):
|
|
892
|
+
return
|
|
893
|
+
draft.ref_sites.append(
|
|
894
|
+
RefSite(
|
|
895
|
+
match.start,
|
|
896
|
+
match.end,
|
|
897
|
+
TableName.from_parts(list(match.parts)),
|
|
898
|
+
)
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
# ---------------------------------------------------------------------------
|
|
903
|
+
# Top-level dispatcher
|
|
904
|
+
# ---------------------------------------------------------------------------
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def classify_statement(stmt: RawStatement, text: str, opts: ConversionOptions) -> ActionDraft:
|
|
908
|
+
"""Classify one top-level statement into a Dataform action draft.
|
|
909
|
+
|
|
910
|
+
Args:
|
|
911
|
+
stmt: The raw statement from the splitter.
|
|
912
|
+
text: Full source text of the file (for raw span capture).
|
|
913
|
+
opts: Conversion options.
|
|
914
|
+
|
|
915
|
+
Returns:
|
|
916
|
+
An :class:`~sql2sqlx.model.ActionDraft`. This function never
|
|
917
|
+
raises for valid-but-unsupported SQL - such statements become
|
|
918
|
+
``operations`` drafts with explanatory warnings.
|
|
919
|
+
"""
|
|
920
|
+
toks = _with_eof(stmt)
|
|
921
|
+
first = toks[0]
|
|
922
|
+
head = first.upper if first.kind == IDENT else ""
|
|
923
|
+
if head == "CREATE":
|
|
924
|
+
return _classify_create(stmt, toks, text, opts)
|
|
925
|
+
if head == "INSERT":
|
|
926
|
+
return _classify_insert(stmt, toks, text, opts)
|
|
927
|
+
if head == "MERGE":
|
|
928
|
+
return _classify_merge(stmt, toks, text, opts)
|
|
929
|
+
if head in ("UPDATE", "DELETE", "TRUNCATE", "DROP", "ALTER", "LOAD"):
|
|
930
|
+
return _classify_dml(stmt, toks, head)
|
|
931
|
+
if head == "CALL" or (head == "EXECUTE" and _kw(toks, 1) == "IMMEDIATE"):
|
|
932
|
+
draft = _ops_draft(
|
|
933
|
+
stmt,
|
|
934
|
+
head,
|
|
935
|
+
warnings=[
|
|
936
|
+
(
|
|
937
|
+
"DYNAMIC_SIDE_EFFECTS",
|
|
938
|
+
"Called procedures and dynamic SQL can hide table reads or "
|
|
939
|
+
"writes; kept verbatim, but review manual dependencies.",
|
|
940
|
+
stmt.start,
|
|
941
|
+
)
|
|
942
|
+
],
|
|
943
|
+
)
|
|
944
|
+
return _finish(draft, toks, head)
|
|
945
|
+
if head in ("SELECT", "WITH", "FROM") or (first.kind == OP and first.text == "("):
|
|
946
|
+
draft = _ops_draft(
|
|
947
|
+
stmt,
|
|
948
|
+
head or "SELECT",
|
|
949
|
+
warnings=[
|
|
950
|
+
(
|
|
951
|
+
"ORPHAN_SELECT",
|
|
952
|
+
"Standalone query converted to an operations action; review "
|
|
953
|
+
"whether it should be a table, view or assertion instead.",
|
|
954
|
+
stmt.start,
|
|
955
|
+
)
|
|
956
|
+
],
|
|
957
|
+
)
|
|
958
|
+
return _finish(draft, toks, head or "SELECT")
|
|
959
|
+
return _finish(_ops_draft(stmt, head or "STATEMENT"), toks, head)
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
# ---------------------------------------------------------------------------
|
|
963
|
+
# CREATE handling
|
|
964
|
+
# ---------------------------------------------------------------------------
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def _classify_create(
|
|
968
|
+
stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
|
|
969
|
+
) -> ActionDraft:
|
|
970
|
+
"""Classify a ``CREATE ...`` statement (dispatch on entity/modifiers)."""
|
|
971
|
+
i = 1
|
|
972
|
+
or_replace = False
|
|
973
|
+
if _kw(toks, i) == "OR" and _kw(toks, i + 1) == "REPLACE":
|
|
974
|
+
or_replace = True
|
|
975
|
+
i += 2
|
|
976
|
+
temp = _kw(toks, i) in ("TEMP", "TEMPORARY")
|
|
977
|
+
if temp:
|
|
978
|
+
i += 1
|
|
979
|
+
materialized = _kw(toks, i) == "MATERIALIZED"
|
|
980
|
+
if materialized:
|
|
981
|
+
i += 1
|
|
982
|
+
external = _kw(toks, i) == "EXTERNAL"
|
|
983
|
+
if external:
|
|
984
|
+
i += 1
|
|
985
|
+
snapshot = _kw(toks, i) == "SNAPSHOT"
|
|
986
|
+
if snapshot:
|
|
987
|
+
i += 1
|
|
988
|
+
entity = _kw(toks, i)
|
|
989
|
+
|
|
990
|
+
if entity == "TABLE" and _kw(toks, i + 1) == "FUNCTION":
|
|
991
|
+
return _finish(_ops_draft(stmt, "CREATE TABLE FUNCTION"), toks, "CREATE")
|
|
992
|
+
if entity == "PROCEDURE":
|
|
993
|
+
# The body is stored for later execution; DML within it is not an
|
|
994
|
+
# immediate read/write dependency of the CREATE action. Keeping the
|
|
995
|
+
# definition source-faithful also preserves the procedure owner's
|
|
996
|
+
# default-project resolution rules.
|
|
997
|
+
return _ops_draft(
|
|
998
|
+
stmt,
|
|
999
|
+
"CREATE PROCEDURE",
|
|
1000
|
+
warnings=[
|
|
1001
|
+
(
|
|
1002
|
+
"PROCEDURE_PRESERVED",
|
|
1003
|
+
"Stored procedure definition kept verbatim; its body is not "
|
|
1004
|
+
"linked as an immediately executed workflow dependency.",
|
|
1005
|
+
stmt.start,
|
|
1006
|
+
)
|
|
1007
|
+
],
|
|
1008
|
+
)
|
|
1009
|
+
if entity not in ("TABLE", "VIEW"):
|
|
1010
|
+
kind = f"CREATE {entity}" if entity else "CREATE"
|
|
1011
|
+
return _finish(_ops_draft(stmt, kind), toks, "CREATE")
|
|
1012
|
+
i += 1
|
|
1013
|
+
ine = False
|
|
1014
|
+
if _kw(toks, i) == "IF" and _kw(toks, i + 1) == "NOT" and _kw(toks, i + 2) == "EXISTS":
|
|
1015
|
+
ine = True
|
|
1016
|
+
i += 3
|
|
1017
|
+
pm = parse_table_path(toks, i)
|
|
1018
|
+
if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
|
|
1019
|
+
draft = _ops_draft(
|
|
1020
|
+
stmt,
|
|
1021
|
+
f"CREATE {entity}",
|
|
1022
|
+
warnings=[
|
|
1023
|
+
(
|
|
1024
|
+
"FALLBACK_OPERATIONS",
|
|
1025
|
+
f"Could not parse the CREATE {entity} target path; " "statement kept verbatim.",
|
|
1026
|
+
stmt.start,
|
|
1027
|
+
)
|
|
1028
|
+
],
|
|
1029
|
+
)
|
|
1030
|
+
return _finish(draft, toks, "CREATE")
|
|
1031
|
+
target = TableName.from_parts(list(pm.parts))
|
|
1032
|
+
tspan = (pm.start, pm.end)
|
|
1033
|
+
i = pm.next_index
|
|
1034
|
+
|
|
1035
|
+
if temp:
|
|
1036
|
+
draft = _ops_draft(
|
|
1037
|
+
stmt,
|
|
1038
|
+
f"CREATE TEMP {entity}",
|
|
1039
|
+
warnings=[
|
|
1040
|
+
(
|
|
1041
|
+
"TEMP_TABLE",
|
|
1042
|
+
"Temporary objects have no Dataform action equivalent; " "kept as operations.",
|
|
1043
|
+
stmt.start,
|
|
1044
|
+
)
|
|
1045
|
+
],
|
|
1046
|
+
)
|
|
1047
|
+
return _finish(draft, toks, "CREATE")
|
|
1048
|
+
if external:
|
|
1049
|
+
draft = _ops_draft(
|
|
1050
|
+
stmt,
|
|
1051
|
+
"CREATE EXTERNAL TABLE",
|
|
1052
|
+
target=target,
|
|
1053
|
+
writes=True,
|
|
1054
|
+
tspan=tspan,
|
|
1055
|
+
warnings=[
|
|
1056
|
+
(
|
|
1057
|
+
"EXTERNAL_TABLE",
|
|
1058
|
+
f"External table DDL for {target.display()} kept as "
|
|
1059
|
+
"operations; consider replacing it with a Dataform "
|
|
1060
|
+
"declaration if the table is managed elsewhere.",
|
|
1061
|
+
stmt.start,
|
|
1062
|
+
)
|
|
1063
|
+
],
|
|
1064
|
+
)
|
|
1065
|
+
return _finish(draft, toks, "CREATE")
|
|
1066
|
+
if snapshot:
|
|
1067
|
+
draft = _ops_draft(
|
|
1068
|
+
stmt,
|
|
1069
|
+
"CREATE SNAPSHOT TABLE",
|
|
1070
|
+
target=target,
|
|
1071
|
+
writes=True,
|
|
1072
|
+
tspan=tspan,
|
|
1073
|
+
warnings=[("SNAPSHOT_TABLE", "Snapshot DDL kept as operations.", stmt.start)],
|
|
1074
|
+
)
|
|
1075
|
+
_finish(draft, toks, "CREATE")
|
|
1076
|
+
if _kw(toks, i) == "CLONE":
|
|
1077
|
+
_append_table_ref(draft, toks, i + 1)
|
|
1078
|
+
return draft
|
|
1079
|
+
|
|
1080
|
+
if entity == "VIEW":
|
|
1081
|
+
return _classify_view_body(
|
|
1082
|
+
stmt, toks, text, opts, i, target, tspan, materialized, or_replace, ine
|
|
1083
|
+
)
|
|
1084
|
+
return _classify_table_body(stmt, toks, text, opts, i, target, tspan, or_replace, ine)
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def _scan_pre_as_clauses(
|
|
1088
|
+
toks: List[Token], i: int, text: str, allow_columns: bool
|
|
1089
|
+
) -> Union[Tuple[str, str], Dict[str, Any]]:
|
|
1090
|
+
"""Scan the clauses between a CREATE target and ``AS``.
|
|
1091
|
+
|
|
1092
|
+
Recognizes an optional column definition list (position-gated to
|
|
1093
|
+
directly after the target), ``PARTITION BY``, ``CLUSTER BY`` and
|
|
1094
|
+
``OPTIONS(...)``. Anything else is a strict fallback: pretending to
|
|
1095
|
+
understand an unknown clause would risk silently changing semantics.
|
|
1096
|
+
|
|
1097
|
+
Args:
|
|
1098
|
+
toks: Token list (EOF-terminated).
|
|
1099
|
+
i: Index of the first token after the target path.
|
|
1100
|
+
text: Full source text.
|
|
1101
|
+
allow_columns: Whether a column list may appear (tables only).
|
|
1102
|
+
|
|
1103
|
+
Returns:
|
|
1104
|
+
On success, a dict with keys ``as_index`` (int or ``None``),
|
|
1105
|
+
``has_columns`` (bool), ``partition`` (raw text or ``None``),
|
|
1106
|
+
``cluster`` (list of raw texts) and ``options`` (entries or
|
|
1107
|
+
``None``). On failure, ``("fallback", reason)``.
|
|
1108
|
+
"""
|
|
1109
|
+
info: Dict[str, Any] = {
|
|
1110
|
+
"as_index": None,
|
|
1111
|
+
"has_columns": False,
|
|
1112
|
+
"partition": None,
|
|
1113
|
+
"cluster": [],
|
|
1114
|
+
"options": None,
|
|
1115
|
+
}
|
|
1116
|
+
first = True
|
|
1117
|
+
n = len(toks)
|
|
1118
|
+
while i < n:
|
|
1119
|
+
t = toks[i]
|
|
1120
|
+
if t.kind == EOF:
|
|
1121
|
+
break
|
|
1122
|
+
if t.kind == OP and t.text == "(":
|
|
1123
|
+
if allow_columns and first:
|
|
1124
|
+
info["has_columns"] = True
|
|
1125
|
+
i = _skip_balanced(toks, i)
|
|
1126
|
+
first = False
|
|
1127
|
+
continue
|
|
1128
|
+
return ("fallback", "unexpected '(' before AS")
|
|
1129
|
+
kw = _kw(toks, i)
|
|
1130
|
+
if kw == "AS":
|
|
1131
|
+
info["as_index"] = i
|
|
1132
|
+
break
|
|
1133
|
+
if kw == "PARTITION" and _kw(toks, i + 1) == "BY":
|
|
1134
|
+
j = i + 2
|
|
1135
|
+
if j >= n or toks[j].kind == EOF:
|
|
1136
|
+
return ("fallback", "empty PARTITION BY expression")
|
|
1137
|
+
start = toks[j].start
|
|
1138
|
+
endpos = start
|
|
1139
|
+
depth = 0
|
|
1140
|
+
while j < n and toks[j].kind != EOF:
|
|
1141
|
+
tj = toks[j]
|
|
1142
|
+
if tj.kind == OP:
|
|
1143
|
+
if tj.text in ("(", "["):
|
|
1144
|
+
depth += 1
|
|
1145
|
+
elif tj.text in (")", "]"):
|
|
1146
|
+
depth -= 1
|
|
1147
|
+
elif (
|
|
1148
|
+
tj.kind == IDENT
|
|
1149
|
+
and depth == 0
|
|
1150
|
+
and tj.upper in ("CLUSTER", "OPTIONS", "AS", "DEFAULT")
|
|
1151
|
+
):
|
|
1152
|
+
break
|
|
1153
|
+
endpos = tj.end
|
|
1154
|
+
j += 1
|
|
1155
|
+
if endpos <= start:
|
|
1156
|
+
return ("fallback", "empty PARTITION BY expression")
|
|
1157
|
+
info["partition"] = text[start:endpos].strip()
|
|
1158
|
+
i = j
|
|
1159
|
+
first = False
|
|
1160
|
+
continue
|
|
1161
|
+
if kw == "CLUSTER" and _kw(toks, i + 1) == "BY":
|
|
1162
|
+
j = i + 2
|
|
1163
|
+
if j >= n or toks[j].kind == EOF:
|
|
1164
|
+
return ("fallback", "empty CLUSTER BY list")
|
|
1165
|
+
depth = 0
|
|
1166
|
+
item_start = toks[j].start
|
|
1167
|
+
last_end = item_start
|
|
1168
|
+
items: List[str] = []
|
|
1169
|
+
while j < n and toks[j].kind != EOF:
|
|
1170
|
+
tj = toks[j]
|
|
1171
|
+
if tj.kind == OP:
|
|
1172
|
+
if tj.text in ("(", "["):
|
|
1173
|
+
depth += 1
|
|
1174
|
+
elif tj.text in (")", "]"):
|
|
1175
|
+
depth -= 1
|
|
1176
|
+
elif tj.text == "," and depth == 0:
|
|
1177
|
+
items.append(text[item_start:last_end].strip())
|
|
1178
|
+
j += 1
|
|
1179
|
+
if j < n and toks[j].kind != EOF:
|
|
1180
|
+
item_start = toks[j].start
|
|
1181
|
+
last_end = item_start
|
|
1182
|
+
continue
|
|
1183
|
+
elif (
|
|
1184
|
+
tj.kind == IDENT
|
|
1185
|
+
and depth == 0
|
|
1186
|
+
and tj.upper in ("OPTIONS", "AS", "DEFAULT", "PARTITION")
|
|
1187
|
+
):
|
|
1188
|
+
break
|
|
1189
|
+
last_end = tj.end
|
|
1190
|
+
j += 1
|
|
1191
|
+
if last_end > item_start:
|
|
1192
|
+
items.append(text[item_start:last_end].strip())
|
|
1193
|
+
if not items or not all(items):
|
|
1194
|
+
return ("fallback", "empty CLUSTER BY list")
|
|
1195
|
+
info["cluster"] = items
|
|
1196
|
+
i = j
|
|
1197
|
+
first = False
|
|
1198
|
+
continue
|
|
1199
|
+
if kw == "OPTIONS":
|
|
1200
|
+
parsed = _parse_options(toks, i, text)
|
|
1201
|
+
if parsed is None:
|
|
1202
|
+
return ("fallback", "unparseable OPTIONS(...) clause")
|
|
1203
|
+
info["options"], i = parsed
|
|
1204
|
+
first = False
|
|
1205
|
+
continue
|
|
1206
|
+
if kw == "DEFAULT":
|
|
1207
|
+
return ("fallback", "DEFAULT COLLATE has no Dataform config equivalent")
|
|
1208
|
+
if kw in ("LIKE", "CLONE", "COPY"):
|
|
1209
|
+
return ("fallback", f"CREATE TABLE ... {kw} has no typed Dataform equivalent")
|
|
1210
|
+
return ("fallback", f"unrecognized clause {toks[i].text!r} before AS")
|
|
1211
|
+
return info
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def _apply_table_metadata(config: Dict[str, Any], info: Dict[str, Any]) -> None:
|
|
1215
|
+
"""Map scanned PARTITION/CLUSTER/OPTIONS metadata into a config dict.
|
|
1216
|
+
|
|
1217
|
+
``description``, ``labels``, ``partition_expiration_days`` and
|
|
1218
|
+
``require_partition_filter`` map onto first-class Dataform fields when
|
|
1219
|
+
their values are simple literals; every other option (or a complex
|
|
1220
|
+
value) is preserved verbatim in ``bigquery.additionalOptions`` so the
|
|
1221
|
+
compiled DDL keeps it.
|
|
1222
|
+
|
|
1223
|
+
Args:
|
|
1224
|
+
config: Draft config dict to mutate.
|
|
1225
|
+
info: Result of :func:`_scan_pre_as_clauses`.
|
|
1226
|
+
"""
|
|
1227
|
+
bq: Dict[str, Any] = {}
|
|
1228
|
+
if info.get("partition"):
|
|
1229
|
+
bq["partitionBy"] = info["partition"]
|
|
1230
|
+
if info.get("cluster"):
|
|
1231
|
+
bq["clusterBy"] = list(info["cluster"])
|
|
1232
|
+
addl: Dict[str, str] = {}
|
|
1233
|
+
for key, raw, val in info.get("options") or []:
|
|
1234
|
+
lk = key.lower()
|
|
1235
|
+
if lk == "description" and isinstance(val, str):
|
|
1236
|
+
config["description"] = val
|
|
1237
|
+
elif lk == "labels" and isinstance(val, dict):
|
|
1238
|
+
bq["labels"] = val
|
|
1239
|
+
elif (
|
|
1240
|
+
lk == "partition_expiration_days"
|
|
1241
|
+
and isinstance(val, (int, float))
|
|
1242
|
+
and not isinstance(val, bool)
|
|
1243
|
+
):
|
|
1244
|
+
bq["partitionExpirationDays"] = val
|
|
1245
|
+
elif lk == "require_partition_filter" and isinstance(val, bool):
|
|
1246
|
+
bq["requirePartitionFilter"] = val
|
|
1247
|
+
else:
|
|
1248
|
+
addl[key] = raw
|
|
1249
|
+
if addl:
|
|
1250
|
+
bq["additionalOptions"] = addl
|
|
1251
|
+
if bq:
|
|
1252
|
+
config["bigquery"] = bq
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def _classify_table_body(
|
|
1256
|
+
stmt: RawStatement,
|
|
1257
|
+
toks: List[Token],
|
|
1258
|
+
text: str,
|
|
1259
|
+
opts: ConversionOptions,
|
|
1260
|
+
i: int,
|
|
1261
|
+
target: TableName,
|
|
1262
|
+
tspan: Tuple[int, int],
|
|
1263
|
+
or_replace: bool,
|
|
1264
|
+
ine: bool,
|
|
1265
|
+
) -> ActionDraft:
|
|
1266
|
+
"""Classify ``CREATE [OR REPLACE] TABLE`` after its target path."""
|
|
1267
|
+
source_keyword = _kw(toks, i)
|
|
1268
|
+
if source_keyword in ("LIKE", "CLONE", "COPY"):
|
|
1269
|
+
draft = _ops_draft(
|
|
1270
|
+
stmt,
|
|
1271
|
+
"CREATE TABLE",
|
|
1272
|
+
target=target,
|
|
1273
|
+
writes=True,
|
|
1274
|
+
tspan=tspan,
|
|
1275
|
+
warnings=[
|
|
1276
|
+
(
|
|
1277
|
+
"FALLBACK_OPERATIONS",
|
|
1278
|
+
f"CREATE TABLE ... {source_keyword} has no typed Dataform "
|
|
1279
|
+
"equivalent; kept as operations.",
|
|
1280
|
+
stmt.start,
|
|
1281
|
+
)
|
|
1282
|
+
],
|
|
1283
|
+
)
|
|
1284
|
+
_finish(draft, toks, "CREATE")
|
|
1285
|
+
_append_table_ref(draft, toks, i + 1)
|
|
1286
|
+
return draft
|
|
1287
|
+
info = _scan_pre_as_clauses(toks, i, text, allow_columns=True)
|
|
1288
|
+
if isinstance(info, tuple):
|
|
1289
|
+
draft = _ops_draft(
|
|
1290
|
+
stmt,
|
|
1291
|
+
"CREATE TABLE",
|
|
1292
|
+
target=target,
|
|
1293
|
+
writes=True,
|
|
1294
|
+
tspan=tspan,
|
|
1295
|
+
warnings=[
|
|
1296
|
+
("FALLBACK_OPERATIONS", f"CREATE TABLE kept as operations: {info[1]}.", stmt.start)
|
|
1297
|
+
],
|
|
1298
|
+
)
|
|
1299
|
+
return _finish(draft, toks, "CREATE")
|
|
1300
|
+
if info["as_index"] is None:
|
|
1301
|
+
return _plain_create(stmt, toks, opts, target, tspan, or_replace, ine, info)
|
|
1302
|
+
if info["has_columns"]:
|
|
1303
|
+
draft = _ops_draft(
|
|
1304
|
+
stmt,
|
|
1305
|
+
"CREATE TABLE",
|
|
1306
|
+
target=target,
|
|
1307
|
+
writes=True,
|
|
1308
|
+
tspan=tspan,
|
|
1309
|
+
warnings=[
|
|
1310
|
+
(
|
|
1311
|
+
"COLUMN_DDL",
|
|
1312
|
+
"CREATE TABLE ... AS with an explicit column "
|
|
1313
|
+
"list can carry types or constraints that "
|
|
1314
|
+
"Dataform's table type cannot express; kept "
|
|
1315
|
+
"as operations.",
|
|
1316
|
+
stmt.start,
|
|
1317
|
+
)
|
|
1318
|
+
],
|
|
1319
|
+
)
|
|
1320
|
+
return _finish(draft, toks, "CREATE")
|
|
1321
|
+
if ine and opts.if_not_exists_strategy is IfNotExistsStrategy.OPERATIONS:
|
|
1322
|
+
draft = _ops_draft(
|
|
1323
|
+
stmt,
|
|
1324
|
+
"CREATE TABLE",
|
|
1325
|
+
target=target,
|
|
1326
|
+
writes=True,
|
|
1327
|
+
tspan=tspan,
|
|
1328
|
+
warnings=[
|
|
1329
|
+
(
|
|
1330
|
+
"IF_NOT_EXISTS",
|
|
1331
|
+
"IF NOT EXISTS preserved verbatim as " "operations per options.",
|
|
1332
|
+
stmt.start,
|
|
1333
|
+
)
|
|
1334
|
+
],
|
|
1335
|
+
)
|
|
1336
|
+
return _finish(draft, toks, "CREATE")
|
|
1337
|
+
bi = info["as_index"] + 1
|
|
1338
|
+
if not _body_head_ok(toks, bi):
|
|
1339
|
+
draft = _ops_draft(
|
|
1340
|
+
stmt,
|
|
1341
|
+
"CREATE TABLE",
|
|
1342
|
+
target=target,
|
|
1343
|
+
writes=True,
|
|
1344
|
+
tspan=tspan,
|
|
1345
|
+
warnings=[
|
|
1346
|
+
(
|
|
1347
|
+
"FALLBACK_OPERATIONS",
|
|
1348
|
+
"CREATE TABLE AS is not followed by a query; " "kept as operations.",
|
|
1349
|
+
stmt.start,
|
|
1350
|
+
)
|
|
1351
|
+
],
|
|
1352
|
+
)
|
|
1353
|
+
return _finish(draft, toks, "CREATE")
|
|
1354
|
+
warnings: List[Tuple[str, str, int]] = []
|
|
1355
|
+
if ine:
|
|
1356
|
+
warnings.append(
|
|
1357
|
+
(
|
|
1358
|
+
"IF_NOT_EXISTS",
|
|
1359
|
+
f"CREATE TABLE IF NOT EXISTS {target.display()} converted to "
|
|
1360
|
+
'type "table": Dataform always creates or replaces, so the '
|
|
1361
|
+
"'only if absent' guard is lost. Use "
|
|
1362
|
+
"--if-not-exists operations to preserve it verbatim.",
|
|
1363
|
+
stmt.start,
|
|
1364
|
+
)
|
|
1365
|
+
)
|
|
1366
|
+
elif not or_replace:
|
|
1367
|
+
warnings.append(
|
|
1368
|
+
(
|
|
1369
|
+
"CREATE_REPLACE_SEMANTICS",
|
|
1370
|
+
f"CREATE TABLE {target.display()} converted to a Dataform table; "
|
|
1371
|
+
"subsequent Dataform runs rebuild the target instead of failing "
|
|
1372
|
+
"when it already exists.",
|
|
1373
|
+
stmt.start,
|
|
1374
|
+
)
|
|
1375
|
+
)
|
|
1376
|
+
config: Dict[str, Any] = {}
|
|
1377
|
+
_apply_table_metadata(config, info)
|
|
1378
|
+
draft = ActionDraft(
|
|
1379
|
+
action_type=ActionType.TABLE,
|
|
1380
|
+
target=target,
|
|
1381
|
+
writes_target=False,
|
|
1382
|
+
creates_target=True,
|
|
1383
|
+
body_start=toks[bi].start,
|
|
1384
|
+
body_end=stmt.end,
|
|
1385
|
+
stmt_start=stmt.start,
|
|
1386
|
+
stmt_end=stmt.terminator_end,
|
|
1387
|
+
config=config,
|
|
1388
|
+
original_kind="CREATE TABLE",
|
|
1389
|
+
warnings=warnings,
|
|
1390
|
+
)
|
|
1391
|
+
return _finish(draft, toks, "CREATE")
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _plain_create(
|
|
1395
|
+
stmt: RawStatement,
|
|
1396
|
+
toks: List[Token],
|
|
1397
|
+
opts: ConversionOptions,
|
|
1398
|
+
target: TableName,
|
|
1399
|
+
tspan: Tuple[int, int],
|
|
1400
|
+
or_replace: bool,
|
|
1401
|
+
ine: bool,
|
|
1402
|
+
info: Dict[str, Any],
|
|
1403
|
+
) -> ActionDraft:
|
|
1404
|
+
"""Handle ``CREATE TABLE`` without ``AS`` (schema-only DDL)."""
|
|
1405
|
+
if opts.plain_create_strategy is PlainCreateStrategy.DECLARATION:
|
|
1406
|
+
config: Dict[str, Any] = {}
|
|
1407
|
+
_apply_table_metadata(config, info)
|
|
1408
|
+
config.pop("bigquery", None)
|
|
1409
|
+
draft = ActionDraft(
|
|
1410
|
+
action_type=ActionType.DECLARATION,
|
|
1411
|
+
target=target,
|
|
1412
|
+
writes_target=False,
|
|
1413
|
+
creates_target=True,
|
|
1414
|
+
body_start=stmt.end,
|
|
1415
|
+
body_end=stmt.end,
|
|
1416
|
+
stmt_start=stmt.start,
|
|
1417
|
+
stmt_end=stmt.terminator_end,
|
|
1418
|
+
config=config,
|
|
1419
|
+
original_kind="CREATE TABLE",
|
|
1420
|
+
warnings=[
|
|
1421
|
+
(
|
|
1422
|
+
"DECLARATION_DROPPED_DDL",
|
|
1423
|
+
f"Plain CREATE TABLE {target.display()} emitted as a "
|
|
1424
|
+
"declaration; the DDL itself was dropped (schema "
|
|
1425
|
+
"assumed to be managed outside Dataform).",
|
|
1426
|
+
stmt.start,
|
|
1427
|
+
)
|
|
1428
|
+
],
|
|
1429
|
+
)
|
|
1430
|
+
return _finish(draft, toks, "CREATE")
|
|
1431
|
+
warns: List[Tuple[str, str, int]] = [
|
|
1432
|
+
(
|
|
1433
|
+
"CREATE_NO_AS",
|
|
1434
|
+
f"Plain CREATE TABLE {target.display()} (no AS query) kept as "
|
|
1435
|
+
"operations; use --plain-create declaration to emit a source "
|
|
1436
|
+
"declaration instead.",
|
|
1437
|
+
stmt.start,
|
|
1438
|
+
)
|
|
1439
|
+
]
|
|
1440
|
+
if not ine and not or_replace:
|
|
1441
|
+
warns.append(
|
|
1442
|
+
(
|
|
1443
|
+
"RERUN_RISK",
|
|
1444
|
+
"This CREATE TABLE has neither OR REPLACE nor IF NOT EXISTS; "
|
|
1445
|
+
"as a repeated Dataform operation it will fail on the second "
|
|
1446
|
+
"run.",
|
|
1447
|
+
stmt.start,
|
|
1448
|
+
)
|
|
1449
|
+
)
|
|
1450
|
+
draft = _ops_draft(
|
|
1451
|
+
stmt, "CREATE TABLE", target=target, writes=True, tspan=tspan, warnings=warns
|
|
1452
|
+
)
|
|
1453
|
+
return _finish(draft, toks, "CREATE")
|
|
1454
|
+
|
|
1455
|
+
|
|
1456
|
+
def _parse_view_columns(
|
|
1457
|
+
toks: List[Token], i: int, text: str
|
|
1458
|
+
) -> Optional[Tuple[List[Tuple[str, Optional[str]]], int]]:
|
|
1459
|
+
"""Parse a view column list ``(name [OPTIONS(description='..')], ...)``.
|
|
1460
|
+
|
|
1461
|
+
Args:
|
|
1462
|
+
toks: Token list; ``toks[i]`` must be ``(``.
|
|
1463
|
+
i: Index of the opening parenthesis.
|
|
1464
|
+
text: Full source text.
|
|
1465
|
+
|
|
1466
|
+
Returns:
|
|
1467
|
+
``([(name, description_or_None), ...], next_index)`` or ``None``
|
|
1468
|
+
for any other shape (caller falls back to operations).
|
|
1469
|
+
"""
|
|
1470
|
+
j = i + 1
|
|
1471
|
+
out: List[Tuple[str, Optional[str]]] = []
|
|
1472
|
+
n = len(toks)
|
|
1473
|
+
while j < n:
|
|
1474
|
+
t = toks[j]
|
|
1475
|
+
if t.kind == BACKTICK:
|
|
1476
|
+
name = unquote_identifier(t.text)
|
|
1477
|
+
elif t.kind == IDENT and t.upper not in RESERVED:
|
|
1478
|
+
name = t.text
|
|
1479
|
+
else:
|
|
1480
|
+
return None
|
|
1481
|
+
j += 1
|
|
1482
|
+
desc: Optional[str] = None
|
|
1483
|
+
if _kw(toks, j) == "OPTIONS":
|
|
1484
|
+
parsed = _parse_options(toks, j, text)
|
|
1485
|
+
if parsed is None:
|
|
1486
|
+
return None
|
|
1487
|
+
entries, j = parsed
|
|
1488
|
+
if (
|
|
1489
|
+
len(entries) != 1
|
|
1490
|
+
or entries[0][0].lower() != "description"
|
|
1491
|
+
or not isinstance(entries[0][2], str)
|
|
1492
|
+
):
|
|
1493
|
+
return None
|
|
1494
|
+
desc = entries[0][2]
|
|
1495
|
+
out.append((name, desc))
|
|
1496
|
+
if _is_op(toks, j, ","):
|
|
1497
|
+
j += 1
|
|
1498
|
+
continue
|
|
1499
|
+
if _is_op(toks, j, ")"):
|
|
1500
|
+
return out, j + 1
|
|
1501
|
+
return None
|
|
1502
|
+
return None
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
def _classify_view_body(
|
|
1506
|
+
stmt: RawStatement,
|
|
1507
|
+
toks: List[Token],
|
|
1508
|
+
text: str,
|
|
1509
|
+
opts: ConversionOptions,
|
|
1510
|
+
i: int,
|
|
1511
|
+
target: TableName,
|
|
1512
|
+
tspan: Tuple[int, int],
|
|
1513
|
+
materialized: bool,
|
|
1514
|
+
or_replace: bool,
|
|
1515
|
+
ine: bool,
|
|
1516
|
+
) -> ActionDraft:
|
|
1517
|
+
"""Classify ``CREATE [MATERIALIZED] VIEW`` after its target path."""
|
|
1518
|
+
kind = "CREATE MATERIALIZED VIEW" if materialized else "CREATE VIEW"
|
|
1519
|
+
|
|
1520
|
+
def fallback(code: str, msg: str, source_index: Optional[int] = None) -> ActionDraft:
|
|
1521
|
+
draft = _ops_draft(
|
|
1522
|
+
stmt, kind, target=target, writes=True, tspan=tspan, warnings=[(code, msg, stmt.start)]
|
|
1523
|
+
)
|
|
1524
|
+
_finish(draft, toks, "CREATE")
|
|
1525
|
+
if source_index is not None:
|
|
1526
|
+
_append_table_ref(draft, toks, source_index)
|
|
1527
|
+
return draft
|
|
1528
|
+
|
|
1529
|
+
columns: Optional[List[Tuple[str, Optional[str]]]] = None
|
|
1530
|
+
if _is_op(toks, i, "("):
|
|
1531
|
+
parsed_cols = _parse_view_columns(toks, i, text)
|
|
1532
|
+
if parsed_cols is None:
|
|
1533
|
+
return fallback(
|
|
1534
|
+
"FALLBACK_OPERATIONS",
|
|
1535
|
+
"View column list uses a form that cannot be "
|
|
1536
|
+
"safely converted; kept as operations.",
|
|
1537
|
+
)
|
|
1538
|
+
columns, i = parsed_cols
|
|
1539
|
+
info = _scan_pre_as_clauses(toks, i, text, allow_columns=False)
|
|
1540
|
+
if isinstance(info, tuple):
|
|
1541
|
+
return fallback("FALLBACK_OPERATIONS", f"{kind} kept as operations: {info[1]}.")
|
|
1542
|
+
if info["as_index"] is None:
|
|
1543
|
+
return fallback("FALLBACK_OPERATIONS", f"{kind} has no AS query; kept as operations.")
|
|
1544
|
+
bi = info["as_index"] + 1
|
|
1545
|
+
if _kw(toks, bi) == "REPLICA":
|
|
1546
|
+
return fallback(
|
|
1547
|
+
"FALLBACK_OPERATIONS",
|
|
1548
|
+
"MATERIALIZED VIEW ... AS REPLICA OF has no typed " "equivalent; kept as operations.",
|
|
1549
|
+
bi + 2 if _kw(toks, bi + 1) == "OF" else None,
|
|
1550
|
+
)
|
|
1551
|
+
if not _body_head_ok(toks, bi):
|
|
1552
|
+
return fallback(
|
|
1553
|
+
"FALLBACK_OPERATIONS", f"{kind} AS is not followed by a query; kept as " "operations."
|
|
1554
|
+
)
|
|
1555
|
+
if ine and opts.if_not_exists_strategy is IfNotExistsStrategy.OPERATIONS:
|
|
1556
|
+
return fallback(
|
|
1557
|
+
"IF_NOT_EXISTS", "IF NOT EXISTS preserved verbatim as operations per " "options."
|
|
1558
|
+
)
|
|
1559
|
+
edits: List[Tuple[int, int, str]] = []
|
|
1560
|
+
if columns:
|
|
1561
|
+
maybe = _alias_select_list_edits(toks, bi, len(toks) - 1, [c[0] for c in columns])
|
|
1562
|
+
if maybe is None:
|
|
1563
|
+
return fallback(
|
|
1564
|
+
"FALLBACK_SELECT_ALIAS",
|
|
1565
|
+
"The view column list could not be safely pushed "
|
|
1566
|
+
"into the select list; kept as operations.",
|
|
1567
|
+
)
|
|
1568
|
+
edits = maybe
|
|
1569
|
+
warnings: List[Tuple[str, str, int]] = []
|
|
1570
|
+
if ine:
|
|
1571
|
+
warnings.append(
|
|
1572
|
+
(
|
|
1573
|
+
"IF_NOT_EXISTS",
|
|
1574
|
+
f"{kind} IF NOT EXISTS {target.display()} converted to type "
|
|
1575
|
+
'"view": Dataform always creates or replaces, so the guard is '
|
|
1576
|
+
"lost.",
|
|
1577
|
+
stmt.start,
|
|
1578
|
+
)
|
|
1579
|
+
)
|
|
1580
|
+
elif not or_replace:
|
|
1581
|
+
warnings.append(
|
|
1582
|
+
(
|
|
1583
|
+
"CREATE_REPLACE_SEMANTICS",
|
|
1584
|
+
f"{kind} {target.display()} converted to a Dataform view; "
|
|
1585
|
+
"subsequent Dataform runs rebuild the target instead of failing "
|
|
1586
|
+
"when it already exists.",
|
|
1587
|
+
stmt.start,
|
|
1588
|
+
)
|
|
1589
|
+
)
|
|
1590
|
+
config: Dict[str, Any] = {}
|
|
1591
|
+
if materialized:
|
|
1592
|
+
config["materialized"] = True
|
|
1593
|
+
_apply_table_metadata(config, info)
|
|
1594
|
+
if columns:
|
|
1595
|
+
described = {name: desc for name, desc in columns if desc}
|
|
1596
|
+
if described:
|
|
1597
|
+
config["columns"] = described
|
|
1598
|
+
draft = ActionDraft(
|
|
1599
|
+
action_type=ActionType.VIEW,
|
|
1600
|
+
target=target,
|
|
1601
|
+
writes_target=False,
|
|
1602
|
+
creates_target=True,
|
|
1603
|
+
body_start=toks[bi].start,
|
|
1604
|
+
body_end=stmt.end,
|
|
1605
|
+
stmt_start=stmt.start,
|
|
1606
|
+
stmt_end=stmt.terminator_end,
|
|
1607
|
+
edits=edits,
|
|
1608
|
+
config=config,
|
|
1609
|
+
original_kind=kind,
|
|
1610
|
+
warnings=warnings,
|
|
1611
|
+
)
|
|
1612
|
+
return _finish(draft, toks, "CREATE")
|
|
1613
|
+
|
|
1614
|
+
|
|
1615
|
+
# ---------------------------------------------------------------------------
|
|
1616
|
+
# INSERT handling
|
|
1617
|
+
# ---------------------------------------------------------------------------
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def _classify_insert(
|
|
1621
|
+
stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
|
|
1622
|
+
) -> ActionDraft:
|
|
1623
|
+
"""Classify ``INSERT [INTO] target [(cols)] <query|VALUES ...>``."""
|
|
1624
|
+
i = 1
|
|
1625
|
+
if _kw(toks, i) == "INTO":
|
|
1626
|
+
i += 1
|
|
1627
|
+
pm = parse_table_path(toks, i)
|
|
1628
|
+
if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
|
|
1629
|
+
draft = _ops_draft(
|
|
1630
|
+
stmt,
|
|
1631
|
+
"INSERT",
|
|
1632
|
+
warnings=[
|
|
1633
|
+
(
|
|
1634
|
+
"FALLBACK_OPERATIONS",
|
|
1635
|
+
"Could not parse the INSERT target path; kept verbatim.",
|
|
1636
|
+
stmt.start,
|
|
1637
|
+
)
|
|
1638
|
+
],
|
|
1639
|
+
)
|
|
1640
|
+
return _finish(draft, toks, "INSERT")
|
|
1641
|
+
target = TableName.from_parts(list(pm.parts))
|
|
1642
|
+
tspan = (pm.start, pm.end)
|
|
1643
|
+
i = pm.next_index
|
|
1644
|
+
cols: Optional[List[str]] = None
|
|
1645
|
+
if _is_op(toks, i, "("):
|
|
1646
|
+
cols, ni = _parse_ident_list(toks, i)
|
|
1647
|
+
if cols is None:
|
|
1648
|
+
draft = _ops_draft(
|
|
1649
|
+
stmt,
|
|
1650
|
+
"INSERT",
|
|
1651
|
+
target=target,
|
|
1652
|
+
writes=True,
|
|
1653
|
+
tspan=tspan,
|
|
1654
|
+
warnings=[
|
|
1655
|
+
(
|
|
1656
|
+
"FALLBACK_OPERATIONS",
|
|
1657
|
+
"Unrecognized INSERT column list; kept " "verbatim.",
|
|
1658
|
+
stmt.start,
|
|
1659
|
+
)
|
|
1660
|
+
],
|
|
1661
|
+
)
|
|
1662
|
+
return _finish(draft, toks, "INSERT")
|
|
1663
|
+
i = ni
|
|
1664
|
+
if _kw(toks, i) == "VALUES":
|
|
1665
|
+
draft = _ops_draft(
|
|
1666
|
+
stmt,
|
|
1667
|
+
"INSERT",
|
|
1668
|
+
target=target,
|
|
1669
|
+
writes=True,
|
|
1670
|
+
tspan=tspan,
|
|
1671
|
+
warnings=[
|
|
1672
|
+
(
|
|
1673
|
+
"INSERT_VALUES",
|
|
1674
|
+
"INSERT ... VALUES has no query body; kept as " "operations.",
|
|
1675
|
+
stmt.start,
|
|
1676
|
+
)
|
|
1677
|
+
],
|
|
1678
|
+
)
|
|
1679
|
+
return _finish(draft, toks, "INSERT")
|
|
1680
|
+
if opts.insert_strategy is InsertStrategy.OPERATIONS:
|
|
1681
|
+
draft = _ops_draft(stmt, "INSERT", target=target, writes=True, tspan=tspan)
|
|
1682
|
+
return _finish(draft, toks, "INSERT")
|
|
1683
|
+
if not _body_head_ok(toks, i):
|
|
1684
|
+
draft = _ops_draft(
|
|
1685
|
+
stmt,
|
|
1686
|
+
"INSERT",
|
|
1687
|
+
target=target,
|
|
1688
|
+
writes=True,
|
|
1689
|
+
tspan=tspan,
|
|
1690
|
+
warnings=[
|
|
1691
|
+
(
|
|
1692
|
+
"FALLBACK_OPERATIONS",
|
|
1693
|
+
"INSERT body is not a plain query; kept " "verbatim.",
|
|
1694
|
+
stmt.start,
|
|
1695
|
+
)
|
|
1696
|
+
],
|
|
1697
|
+
)
|
|
1698
|
+
return _finish(draft, toks, "INSERT")
|
|
1699
|
+
output_names = cols or _select_output_names(toks, i, len(toks) - 1)
|
|
1700
|
+
if (
|
|
1701
|
+
output_names is None
|
|
1702
|
+
or len({name.upper() for name in output_names}) != len(output_names)
|
|
1703
|
+
or any(not _dataform_insert_column(name) for name in output_names)
|
|
1704
|
+
):
|
|
1705
|
+
draft = _ops_draft(
|
|
1706
|
+
stmt,
|
|
1707
|
+
"INSERT",
|
|
1708
|
+
target=target,
|
|
1709
|
+
writes=True,
|
|
1710
|
+
tspan=tspan,
|
|
1711
|
+
warnings=[
|
|
1712
|
+
(
|
|
1713
|
+
"FALLBACK_SELECT_ALIAS",
|
|
1714
|
+
"The incremental query's output column names "
|
|
1715
|
+
"are not all exactly derivable, unique, and "
|
|
1716
|
+
"safe for Dataform Core's runtime backtick "
|
|
1717
|
+
"quoting; the statement was kept verbatim.",
|
|
1718
|
+
stmt.start,
|
|
1719
|
+
)
|
|
1720
|
+
],
|
|
1721
|
+
)
|
|
1722
|
+
return _finish(draft, toks, "INSERT")
|
|
1723
|
+
edits: List[Tuple[int, int, str]] = []
|
|
1724
|
+
if cols:
|
|
1725
|
+
maybe = _alias_select_list_edits(toks, i, len(toks) - 1, cols)
|
|
1726
|
+
if maybe is None:
|
|
1727
|
+
draft = _ops_draft(
|
|
1728
|
+
stmt,
|
|
1729
|
+
"INSERT",
|
|
1730
|
+
target=target,
|
|
1731
|
+
writes=True,
|
|
1732
|
+
tspan=tspan,
|
|
1733
|
+
warnings=[
|
|
1734
|
+
(
|
|
1735
|
+
"FALLBACK_SELECT_ALIAS",
|
|
1736
|
+
"The INSERT column list could not be "
|
|
1737
|
+
"safely pushed into the select list (star "
|
|
1738
|
+
"expansion, INTERVAL/typed constructors, "
|
|
1739
|
+
"or arity mismatch); kept verbatim.",
|
|
1740
|
+
stmt.start,
|
|
1741
|
+
)
|
|
1742
|
+
],
|
|
1743
|
+
)
|
|
1744
|
+
return _finish(draft, toks, "INSERT")
|
|
1745
|
+
edits = maybe
|
|
1746
|
+
config: Dict[str, Any] = {}
|
|
1747
|
+
if opts.protect_incrementals:
|
|
1748
|
+
config["protected"] = True
|
|
1749
|
+
draft = ActionDraft(
|
|
1750
|
+
action_type=ActionType.INCREMENTAL,
|
|
1751
|
+
target=target,
|
|
1752
|
+
writes_target=False,
|
|
1753
|
+
creates_target=True,
|
|
1754
|
+
body_start=toks[i].start,
|
|
1755
|
+
body_end=stmt.end,
|
|
1756
|
+
stmt_start=stmt.start,
|
|
1757
|
+
stmt_end=stmt.terminator_end,
|
|
1758
|
+
edits=edits,
|
|
1759
|
+
config=config,
|
|
1760
|
+
original_kind="INSERT",
|
|
1761
|
+
warnings=[
|
|
1762
|
+
(
|
|
1763
|
+
"INSERT_INCREMENTAL",
|
|
1764
|
+
f"INSERT INTO {target.display()} converted to type "
|
|
1765
|
+
'"incremental": on first run or --full-refresh Dataform will '
|
|
1766
|
+
"(re)create the table from this query. Wrap date filters in "
|
|
1767
|
+
"${when(incremental(), ...)} if they should apply only to "
|
|
1768
|
+
"incremental runs. On later runs Dataform projects every "
|
|
1769
|
+
"existing target column by name, so the query output must match "
|
|
1770
|
+
"that target schema.",
|
|
1771
|
+
stmt.start,
|
|
1772
|
+
)
|
|
1773
|
+
],
|
|
1774
|
+
)
|
|
1775
|
+
return _finish(draft, toks, "INSERT")
|
|
1776
|
+
|
|
1777
|
+
|
|
1778
|
+
# ---------------------------------------------------------------------------
|
|
1779
|
+
# MERGE handling
|
|
1780
|
+
# ---------------------------------------------------------------------------
|
|
1781
|
+
|
|
1782
|
+
|
|
1783
|
+
def _read_qualified_col(toks: Sequence[Token], j: int) -> Optional[Tuple[str, str, int]]:
|
|
1784
|
+
"""Read a strictly qualified column ``qual.col`` at index ``j``.
|
|
1785
|
+
|
|
1786
|
+
Returns:
|
|
1787
|
+
``(qualifier, column, next_index)`` or ``None``.
|
|
1788
|
+
"""
|
|
1789
|
+
if j + 2 >= len(toks):
|
|
1790
|
+
return None
|
|
1791
|
+
a, dot, b = toks[j], toks[j + 1], toks[j + 2]
|
|
1792
|
+
if a.kind not in (IDENT, BACKTICK) or (a.kind == IDENT and a.upper in RESERVED):
|
|
1793
|
+
return None
|
|
1794
|
+
if not (dot.kind == OP and dot.text == "."):
|
|
1795
|
+
return None
|
|
1796
|
+
if b.kind not in (IDENT, BACKTICK) or (b.kind == IDENT and b.upper in RESERVED):
|
|
1797
|
+
return None
|
|
1798
|
+
return (unquote_identifier(a.text), unquote_identifier(b.text), j + 3)
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def _read_set_target_col(toks: Sequence[Token], j: int, tq: str) -> Optional[Tuple[str, int]]:
|
|
1802
|
+
"""Read an UPDATE SET target column, optionally target-qualified.
|
|
1803
|
+
|
|
1804
|
+
Args:
|
|
1805
|
+
toks: Token list.
|
|
1806
|
+
j: Start index.
|
|
1807
|
+
tq: Uppercased target qualifier (alias or table name).
|
|
1808
|
+
|
|
1809
|
+
Returns:
|
|
1810
|
+
``(column, next_index)`` or ``None``.
|
|
1811
|
+
"""
|
|
1812
|
+
q = _read_qualified_col(toks, j)
|
|
1813
|
+
if q is not None:
|
|
1814
|
+
if q[0].upper() != tq:
|
|
1815
|
+
return None
|
|
1816
|
+
return q[1], q[2]
|
|
1817
|
+
if j < len(toks):
|
|
1818
|
+
t = toks[j]
|
|
1819
|
+
if t.kind == BACKTICK or (t.kind == IDENT and t.upper not in RESERVED):
|
|
1820
|
+
return unquote_identifier(t.text), j + 1
|
|
1821
|
+
return None
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
def _merge_safe_extract(
|
|
1825
|
+
toks: List[Token], i: int, target: TableName, talias: Optional[str]
|
|
1826
|
+
) -> Union[str, Tuple[int, int, List[str]]]:
|
|
1827
|
+
"""Prove a MERGE equivalent to a Dataform ``uniqueKey`` incremental.
|
|
1828
|
+
|
|
1829
|
+
The proof requires *all* of: a subquery source with an alias; an ``ON``
|
|
1830
|
+
clause that is a pure conjunction of same-named target/source column
|
|
1831
|
+
equalities; exactly ``WHEN MATCHED THEN UPDATE SET`` with only direct
|
|
1832
|
+
same-named source assignments; exactly ``WHEN NOT MATCHED [BY TARGET]
|
|
1833
|
+
THEN INSERT`` with explicit matching source columns; and a
|
|
1834
|
+
source select list whose derivable output names are exactly the key
|
|
1835
|
+
columns plus the updated columns. Under those conditions the MERGE
|
|
1836
|
+
Dataform generates for ``type: "incremental"`` + ``uniqueKey`` has
|
|
1837
|
+
identical row-level effects.
|
|
1838
|
+
|
|
1839
|
+
Args:
|
|
1840
|
+
toks: Statement tokens (EOF-terminated).
|
|
1841
|
+
i: Index of the expected ``USING`` keyword.
|
|
1842
|
+
target: The MERGE target table.
|
|
1843
|
+
talias: The target alias, if any.
|
|
1844
|
+
|
|
1845
|
+
Returns:
|
|
1846
|
+
``(body_start_offset, body_end_offset, unique_keys)`` on success,
|
|
1847
|
+
or a human-readable failure reason string.
|
|
1848
|
+
"""
|
|
1849
|
+
if _kw(toks, i) != "USING":
|
|
1850
|
+
return "unrecognized MERGE shape"
|
|
1851
|
+
i += 1
|
|
1852
|
+
if not _is_op(toks, i, "("):
|
|
1853
|
+
return "USING source is a table, not a subquery"
|
|
1854
|
+
open_i = i
|
|
1855
|
+
after = _skip_balanced(toks, i)
|
|
1856
|
+
close_i = after - 1
|
|
1857
|
+
if not _is_op(toks, close_i, ")"):
|
|
1858
|
+
return "unterminated USING subquery"
|
|
1859
|
+
inner_ts, inner_te = open_i + 1, close_i
|
|
1860
|
+
if inner_ts >= inner_te:
|
|
1861
|
+
return "empty USING subquery"
|
|
1862
|
+
j = after
|
|
1863
|
+
if _kw(toks, j) == "AS":
|
|
1864
|
+
j += 1
|
|
1865
|
+
salias: Optional[str] = None
|
|
1866
|
+
if j < len(toks):
|
|
1867
|
+
t = toks[j]
|
|
1868
|
+
if t.kind == BACKTICK or (t.kind == IDENT and t.upper not in RESERVED):
|
|
1869
|
+
salias = unquote_identifier(t.text)
|
|
1870
|
+
j += 1
|
|
1871
|
+
if salias is None:
|
|
1872
|
+
return "the USING subquery has no alias"
|
|
1873
|
+
if _kw(toks, j) != "ON":
|
|
1874
|
+
return "unrecognized MERGE shape after USING"
|
|
1875
|
+
j += 1
|
|
1876
|
+
tq = (talias or target.table).upper()
|
|
1877
|
+
sq = salias.upper()
|
|
1878
|
+
if tq == sq:
|
|
1879
|
+
return "target and source aliases are not distinct"
|
|
1880
|
+
keys: List[str] = []
|
|
1881
|
+
while True:
|
|
1882
|
+
left = _read_qualified_col(toks, j)
|
|
1883
|
+
if left is None:
|
|
1884
|
+
return "ON is not a conjunction of simple qualified equalities"
|
|
1885
|
+
lq, lc, j = left
|
|
1886
|
+
if not _is_op(toks, j, "="):
|
|
1887
|
+
return "ON is not a conjunction of simple qualified equalities"
|
|
1888
|
+
j += 1
|
|
1889
|
+
right = _read_qualified_col(toks, j)
|
|
1890
|
+
if right is None:
|
|
1891
|
+
return "ON is not a conjunction of simple qualified equalities"
|
|
1892
|
+
rq, rc, j = right
|
|
1893
|
+
if lc.upper() != rc.upper():
|
|
1894
|
+
return "ON joins differently named columns"
|
|
1895
|
+
if {lq.upper(), rq.upper()} != {tq, sq}:
|
|
1896
|
+
return "ON does not compare target columns to source columns"
|
|
1897
|
+
keys.append(lc if lq.upper() == tq else rc)
|
|
1898
|
+
nxt = _kw(toks, j)
|
|
1899
|
+
if nxt == "AND":
|
|
1900
|
+
j += 1
|
|
1901
|
+
continue
|
|
1902
|
+
if nxt == "WHEN":
|
|
1903
|
+
break
|
|
1904
|
+
return "unsupported ON clause"
|
|
1905
|
+
if not (_kw(toks, j) == "WHEN" and _kw(toks, j + 1) == "MATCHED"):
|
|
1906
|
+
return "first WHEN clause is not WHEN MATCHED"
|
|
1907
|
+
j += 2
|
|
1908
|
+
if _kw(toks, j) == "AND":
|
|
1909
|
+
return "conditional WHEN MATCHED clause"
|
|
1910
|
+
if not (_kw(toks, j) == "THEN" and _kw(toks, j + 1) == "UPDATE" and _kw(toks, j + 2) == "SET"):
|
|
1911
|
+
return "WHEN MATCHED does not perform a plain UPDATE SET"
|
|
1912
|
+
j += 3
|
|
1913
|
+
set_cols: List[str] = []
|
|
1914
|
+
while True:
|
|
1915
|
+
left_set = _read_set_target_col(toks, j, tq)
|
|
1916
|
+
if left_set is None:
|
|
1917
|
+
return "UPDATE SET has a non-trivial assignment target"
|
|
1918
|
+
lc, j = left_set
|
|
1919
|
+
if not _is_op(toks, j, "="):
|
|
1920
|
+
return "UPDATE SET has a non-trivial assignment"
|
|
1921
|
+
j += 1
|
|
1922
|
+
right = _read_qualified_col(toks, j)
|
|
1923
|
+
if right is None or right[0].upper() != sq:
|
|
1924
|
+
return "UPDATE SET assigns something other than a source column"
|
|
1925
|
+
if right[1].upper() != lc.upper():
|
|
1926
|
+
return "UPDATE SET renames a column"
|
|
1927
|
+
j = right[2]
|
|
1928
|
+
set_cols.append(lc)
|
|
1929
|
+
if _is_op(toks, j, ","):
|
|
1930
|
+
j += 1
|
|
1931
|
+
continue
|
|
1932
|
+
if _kw(toks, j) == "WHEN":
|
|
1933
|
+
break
|
|
1934
|
+
return "unsupported UPDATE SET clause"
|
|
1935
|
+
if not (_kw(toks, j) == "WHEN" and _kw(toks, j + 1) == "NOT" and _kw(toks, j + 2) == "MATCHED"):
|
|
1936
|
+
return "second WHEN clause is not WHEN NOT MATCHED"
|
|
1937
|
+
j += 3
|
|
1938
|
+
if _kw(toks, j) == "BY":
|
|
1939
|
+
if _kw(toks, j + 1) != "TARGET":
|
|
1940
|
+
return "WHEN NOT MATCHED BY SOURCE is not expressible"
|
|
1941
|
+
j += 2
|
|
1942
|
+
if _kw(toks, j) == "AND":
|
|
1943
|
+
return "conditional WHEN NOT MATCHED clause"
|
|
1944
|
+
if not (_kw(toks, j) == "THEN" and _kw(toks, j + 1) == "INSERT"):
|
|
1945
|
+
return "WHEN NOT MATCHED does not perform a plain INSERT"
|
|
1946
|
+
j += 2
|
|
1947
|
+
outputs = _select_output_names(toks, inner_ts, inner_te)
|
|
1948
|
+
if outputs is None:
|
|
1949
|
+
return "cannot derive the source subquery's output column names"
|
|
1950
|
+
if len({output.upper() for output in outputs}) != len(outputs):
|
|
1951
|
+
return "the source subquery has duplicate output column names"
|
|
1952
|
+
if len({key.upper() for key in keys}) != len(keys):
|
|
1953
|
+
return "the MERGE ON clause repeats a unique-key column"
|
|
1954
|
+
if len({column.upper() for column in set_cols}) != len(set_cols):
|
|
1955
|
+
return "UPDATE SET assigns the same target column more than once"
|
|
1956
|
+
out_set = {o.upper() for o in outputs}
|
|
1957
|
+
key_set = {k.upper() for k in keys}
|
|
1958
|
+
set_col_set = {c.upper() for c in set_cols}
|
|
1959
|
+
if not key_set.issubset(set_col_set):
|
|
1960
|
+
return (
|
|
1961
|
+
"Dataform updates unique-key columns during its generated "
|
|
1962
|
+
"MERGE, but the original UPDATE SET does not"
|
|
1963
|
+
)
|
|
1964
|
+
if any(not _dataform_merge_column(output) for output in outputs):
|
|
1965
|
+
return (
|
|
1966
|
+
"Dataform cannot safely emit a quoted or reserved source "
|
|
1967
|
+
"column name in its generated MERGE"
|
|
1968
|
+
)
|
|
1969
|
+
need = key_set | set_col_set
|
|
1970
|
+
if out_set != need:
|
|
1971
|
+
return "source columns do not exactly cover the key + updated columns"
|
|
1972
|
+
if _kw(toks, j) == "ROW":
|
|
1973
|
+
return (
|
|
1974
|
+
"INSERT ROW depends on the target table's physical column "
|
|
1975
|
+
"order, which cannot be proven from SQL text alone"
|
|
1976
|
+
)
|
|
1977
|
+
else:
|
|
1978
|
+
if not _is_op(toks, j, "("):
|
|
1979
|
+
return "WHEN NOT MATCHED INSERT is not (columns) VALUES or ROW"
|
|
1980
|
+
cols, j = _parse_ident_list(toks, j)
|
|
1981
|
+
if cols is None:
|
|
1982
|
+
return "WHEN NOT MATCHED INSERT is not (columns) VALUES or ROW"
|
|
1983
|
+
if len({column.upper() for column in cols}) != len(cols):
|
|
1984
|
+
return "INSERT column list contains duplicates"
|
|
1985
|
+
if _kw(toks, j) != "VALUES" or not _is_op(toks, j + 1, "("):
|
|
1986
|
+
return "WHEN NOT MATCHED INSERT is not (columns) VALUES (...)"
|
|
1987
|
+
j += 2
|
|
1988
|
+
for idx, col in enumerate(cols):
|
|
1989
|
+
right = _read_qualified_col(toks, j)
|
|
1990
|
+
if right is None or right[0].upper() != sq or right[1].upper() != col.upper():
|
|
1991
|
+
return "INSERT VALUES are not the matching source columns"
|
|
1992
|
+
j = right[2]
|
|
1993
|
+
if idx < len(cols) - 1:
|
|
1994
|
+
if not _is_op(toks, j, ","):
|
|
1995
|
+
return "INSERT VALUES are not the matching source columns"
|
|
1996
|
+
j += 1
|
|
1997
|
+
if not _is_op(toks, j, ")"):
|
|
1998
|
+
return "INSERT VALUES are not the matching source columns"
|
|
1999
|
+
j += 1
|
|
2000
|
+
if {c.upper() for c in cols} != out_set:
|
|
2001
|
+
return "INSERT column list does not match the source columns"
|
|
2002
|
+
if [c.upper() for c in cols] != [o.upper() for o in outputs]:
|
|
2003
|
+
return "source output order does not match the explicit INSERT " "column order"
|
|
2004
|
+
if toks[j].kind != EOF:
|
|
2005
|
+
return "unsupported trailing MERGE clauses"
|
|
2006
|
+
return (toks[inner_ts].start, toks[inner_te - 1].end, keys)
|
|
2007
|
+
|
|
2008
|
+
|
|
2009
|
+
def _classify_merge(
|
|
2010
|
+
stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
|
|
2011
|
+
) -> ActionDraft:
|
|
2012
|
+
"""Classify a ``MERGE`` statement per the configured strategy."""
|
|
2013
|
+
i = 1
|
|
2014
|
+
if _kw(toks, i) == "INTO":
|
|
2015
|
+
i += 1
|
|
2016
|
+
pm = parse_table_path(toks, i)
|
|
2017
|
+
if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
|
|
2018
|
+
draft = _ops_draft(
|
|
2019
|
+
stmt,
|
|
2020
|
+
"MERGE",
|
|
2021
|
+
warnings=[
|
|
2022
|
+
(
|
|
2023
|
+
"FALLBACK_OPERATIONS",
|
|
2024
|
+
"Could not parse the MERGE target path; kept verbatim.",
|
|
2025
|
+
stmt.start,
|
|
2026
|
+
)
|
|
2027
|
+
],
|
|
2028
|
+
)
|
|
2029
|
+
return _finish(draft, toks, "MERGE")
|
|
2030
|
+
target = TableName.from_parts(list(pm.parts))
|
|
2031
|
+
tspan = (pm.start, pm.end)
|
|
2032
|
+
i = pm.next_index
|
|
2033
|
+
talias: Optional[str] = None
|
|
2034
|
+
if _kw(toks, i) == "AS":
|
|
2035
|
+
if toks[i + 1].kind in (IDENT, BACKTICK):
|
|
2036
|
+
talias = unquote_identifier(toks[i + 1].text)
|
|
2037
|
+
i += 2
|
|
2038
|
+
elif toks[i].kind == BACKTICK or (toks[i].kind == IDENT and toks[i].upper not in RESERVED):
|
|
2039
|
+
talias = unquote_identifier(toks[i].text)
|
|
2040
|
+
i += 1
|
|
2041
|
+
if opts.merge_strategy is MergeStrategy.INCREMENTAL_WHEN_SAFE:
|
|
2042
|
+
result = _merge_safe_extract(toks, i, target, talias)
|
|
2043
|
+
if not isinstance(result, str):
|
|
2044
|
+
body_start, body_end, keys = result
|
|
2045
|
+
config: Dict[str, Any] = {"uniqueKey": keys}
|
|
2046
|
+
if opts.protect_incrementals:
|
|
2047
|
+
config["protected"] = True
|
|
2048
|
+
draft = ActionDraft(
|
|
2049
|
+
action_type=ActionType.INCREMENTAL,
|
|
2050
|
+
target=target,
|
|
2051
|
+
writes_target=False,
|
|
2052
|
+
creates_target=True,
|
|
2053
|
+
body_start=body_start,
|
|
2054
|
+
body_end=body_end,
|
|
2055
|
+
stmt_start=stmt.start,
|
|
2056
|
+
stmt_end=stmt.terminator_end,
|
|
2057
|
+
config=config,
|
|
2058
|
+
original_kind="MERGE",
|
|
2059
|
+
warnings=[
|
|
2060
|
+
(
|
|
2061
|
+
"MERGE_INCREMENTAL",
|
|
2062
|
+
f"MERGE into {target.display()} proven equivalent to an "
|
|
2063
|
+
f"incremental table with uniqueKey {keys}, provided the "
|
|
2064
|
+
"existing target schema exactly matches the source "
|
|
2065
|
+
"output; converted.",
|
|
2066
|
+
stmt.start,
|
|
2067
|
+
),
|
|
2068
|
+
(
|
|
2069
|
+
"TARGET_SCHEMA_REQUIRED",
|
|
2070
|
+
"Dataform builds incremental MERGE assignments from "
|
|
2071
|
+
"existing target metadata. Verify that its columns "
|
|
2072
|
+
"exactly match this query before the first incremental "
|
|
2073
|
+
"run.",
|
|
2074
|
+
stmt.start,
|
|
2075
|
+
),
|
|
2076
|
+
],
|
|
2077
|
+
)
|
|
2078
|
+
return _finish(draft, toks, "MERGE")
|
|
2079
|
+
draft = _ops_draft(
|
|
2080
|
+
stmt,
|
|
2081
|
+
"MERGE",
|
|
2082
|
+
target=target,
|
|
2083
|
+
writes=True,
|
|
2084
|
+
tspan=tspan,
|
|
2085
|
+
warnings=[("MERGE_FALLBACK", f"MERGE kept as operations ({result}).", stmt.start)],
|
|
2086
|
+
)
|
|
2087
|
+
return _finish(draft, toks, "MERGE")
|
|
2088
|
+
draft = _ops_draft(stmt, "MERGE", target=target, writes=True, tspan=tspan)
|
|
2089
|
+
return _finish(draft, toks, "MERGE")
|
|
2090
|
+
|
|
2091
|
+
|
|
2092
|
+
# ---------------------------------------------------------------------------
|
|
2093
|
+
# Other DML / DDL with identifiable write targets
|
|
2094
|
+
# ---------------------------------------------------------------------------
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _dml_common(
|
|
2098
|
+
stmt: RawStatement, toks: List[Token], pm: Optional[Any], kind: str, skip_target: bool
|
|
2099
|
+
) -> ActionDraft:
|
|
2100
|
+
"""Finalize a DML/DDL operations draft with an optional write target.
|
|
2101
|
+
|
|
2102
|
+
Args:
|
|
2103
|
+
stmt: The raw statement.
|
|
2104
|
+
toks: Statement tokens.
|
|
2105
|
+
pm: Parsed target path (or ``None``).
|
|
2106
|
+
kind: Uppercase statement kind.
|
|
2107
|
+
skip_target: Exclude the target span from reference scanning
|
|
2108
|
+
(needed for ``DELETE FROM target``, where the scanner would
|
|
2109
|
+
otherwise see the target as a FROM-clause table read).
|
|
2110
|
+
|
|
2111
|
+
Returns:
|
|
2112
|
+
The draft.
|
|
2113
|
+
"""
|
|
2114
|
+
if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
|
|
2115
|
+
return _finish(_ops_draft(stmt, kind), toks, kind)
|
|
2116
|
+
target = TableName.from_parts(list(pm.parts))
|
|
2117
|
+
tspan = (pm.start, pm.end)
|
|
2118
|
+
draft = _ops_draft(stmt, kind, target=target, writes=True, tspan=tspan)
|
|
2119
|
+
return _finish(draft, toks, kind, skip_spans=[tspan] if skip_target else ())
|
|
2120
|
+
|
|
2121
|
+
|
|
2122
|
+
def _classify_drop_alter(stmt: RawStatement, toks: List[Token], head: str) -> ActionDraft:
|
|
2123
|
+
"""Classify DROP/ALTER, extracting the table/view target when present."""
|
|
2124
|
+
i = 1
|
|
2125
|
+
ent = _kw(toks, i)
|
|
2126
|
+
if ent in ("MATERIALIZED", "EXTERNAL", "SNAPSHOT"):
|
|
2127
|
+
nxt = _kw(toks, i + 1)
|
|
2128
|
+
if (ent == "MATERIALIZED" and nxt == "VIEW") or (
|
|
2129
|
+
ent in ("EXTERNAL", "SNAPSHOT") and nxt == "TABLE"
|
|
2130
|
+
):
|
|
2131
|
+
i += 2
|
|
2132
|
+
else:
|
|
2133
|
+
return _finish(_ops_draft(stmt, head), toks, head)
|
|
2134
|
+
elif ent in ("TABLE", "VIEW"):
|
|
2135
|
+
i += 1
|
|
2136
|
+
else:
|
|
2137
|
+
return _finish(_ops_draft(stmt, head), toks, head)
|
|
2138
|
+
if _kw(toks, i) == "IF" and _kw(toks, i + 1) == "EXISTS":
|
|
2139
|
+
i += 2
|
|
2140
|
+
pm = parse_table_path(toks, i)
|
|
2141
|
+
draft = _dml_common(stmt, toks, pm, head, skip_target=False)
|
|
2142
|
+
if (
|
|
2143
|
+
head == "ALTER"
|
|
2144
|
+
and pm is not None
|
|
2145
|
+
and 1 <= len(pm.parts) <= 3
|
|
2146
|
+
and all(pm.parts)
|
|
2147
|
+
and _kw(toks, pm.next_index) == "RENAME"
|
|
2148
|
+
and _kw(toks, pm.next_index + 1) == "TO"
|
|
2149
|
+
):
|
|
2150
|
+
renamed_match = parse_table_path(toks, pm.next_index + 2)
|
|
2151
|
+
if (
|
|
2152
|
+
renamed_match is not None
|
|
2153
|
+
and 1 <= len(renamed_match.parts) <= 3
|
|
2154
|
+
and all(renamed_match.parts)
|
|
2155
|
+
):
|
|
2156
|
+
original = TableName.from_parts(list(pm.parts))
|
|
2157
|
+
renamed = TableName.from_parts(list(renamed_match.parts))
|
|
2158
|
+
if renamed.project is None and renamed.dataset is None:
|
|
2159
|
+
renamed = TableName(
|
|
2160
|
+
original.project,
|
|
2161
|
+
original.dataset,
|
|
2162
|
+
renamed.table,
|
|
2163
|
+
)
|
|
2164
|
+
elif renamed.project is None and original.project is not None:
|
|
2165
|
+
renamed = TableName(
|
|
2166
|
+
original.project,
|
|
2167
|
+
renamed.dataset,
|
|
2168
|
+
renamed.table,
|
|
2169
|
+
)
|
|
2170
|
+
draft.extra_write_targets.append(renamed)
|
|
2171
|
+
return draft
|
|
2172
|
+
|
|
2173
|
+
|
|
2174
|
+
def _classify_dml(stmt: RawStatement, toks: List[Token], head: str) -> ActionDraft:
|
|
2175
|
+
"""Classify UPDATE/DELETE/TRUNCATE/DROP/ALTER/LOAD statements."""
|
|
2176
|
+
if head == "UPDATE":
|
|
2177
|
+
return _dml_common(stmt, toks, parse_table_path(toks, 1), "UPDATE", skip_target=False)
|
|
2178
|
+
if head == "DELETE":
|
|
2179
|
+
i = 2 if _kw(toks, 1) == "FROM" else 1
|
|
2180
|
+
return _dml_common(stmt, toks, parse_table_path(toks, i), "DELETE", skip_target=True)
|
|
2181
|
+
if head == "TRUNCATE":
|
|
2182
|
+
i = 2 if _kw(toks, 1) == "TABLE" else 1
|
|
2183
|
+
return _dml_common(stmt, toks, parse_table_path(toks, i), "TRUNCATE", skip_target=False)
|
|
2184
|
+
if head in ("DROP", "ALTER"):
|
|
2185
|
+
return _classify_drop_alter(stmt, toks, head)
|
|
2186
|
+
if head == "LOAD":
|
|
2187
|
+
i = 1
|
|
2188
|
+
if _kw(toks, i) == "DATA":
|
|
2189
|
+
i += 1
|
|
2190
|
+
if _kw(toks, i) in ("INTO", "OVERWRITE"):
|
|
2191
|
+
i += 1
|
|
2192
|
+
return _dml_common(stmt, toks, parse_table_path(toks, i), "LOAD", skip_target=False)
|
|
2193
|
+
return _finish(_ops_draft(stmt, head), toks, head) # pragma: no cover
|