sql2sqlx 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sql2sqlx/parser.py ADDED
@@ -0,0 +1,2193 @@
1
+ # Copyright (c) Soumyadip Sarkar.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the Apache-style license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Statement classification and Dataform metadata extraction.
8
+
9
+ This module decides, for every top-level statement, which Dataform action
10
+ type it becomes and extracts everything needed for the ``config { ... }``
11
+ block. The governing principle is:
12
+
13
+ **Convert when provably safe; otherwise fall back to a verbatim
14
+ ``operations`` action and record a warning.**
15
+
16
+ An ``operations`` action executes the original SQL unchanged, so the
17
+ fallback path can never alter behavior - it only forgoes idiomatic
18
+ Dataform structure. Every fallback carries a stable warning code and a
19
+ human-readable reason in the conversion report.
20
+
21
+ Mapping summary
22
+ ---------------
23
+ ==================================== =======================================
24
+ Statement Result
25
+ ==================================== =======================================
26
+ ``CREATE [OR REPLACE] TABLE .. AS`` ``type: "table"`` (+ partition/cluster/
27
+ OPTIONS mapping)
28
+ ``CREATE [MATERIALIZED] VIEW .. AS`` ``type: "view"`` (+ ``materialized``)
29
+ ``INSERT INTO .. SELECT`` ``type: "incremental"`` (configurable)
30
+ ``MERGE`` ``operations`` by default; provably
31
+ equivalent MERGEs can opt into
32
+ ``incremental`` + ``uniqueKey``
33
+ ``CREATE TABLE`` (no ``AS``) ``operations`` (or ``declaration``)
34
+ UPDATE/DELETE/TRUNCATE/DROP/ALTER/ ``operations`` with write-target
35
+ LOAD tracking for dependency chaining
36
+ everything else ``operations``
37
+ ==================================== =======================================
38
+
39
+ Select-list aliasing
40
+ --------------------
41
+ ``INSERT INTO t (a, b) SELECT x, y ...`` and ``CREATE VIEW v (a, b) AS
42
+ SELECT ...`` change output column names. To convert these faithfully the
43
+ select list is rewritten item-by-item (``x AS a, y AS b``) - but **only**
44
+ when every item's existing output name/alias can be determined exactly at
45
+ the token level. Constructs that make that ambiguous (``*`` expansion,
46
+ ``INTERVAL 1 DAY`` implicit-alias traps, naked ``STRUCT<...>``/
47
+ ``ARRAY<...>`` constructors, ``SELECT AS STRUCT``) trigger the operations
48
+ fallback instead of a guess.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import json
54
+ import math
55
+ import re
56
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
57
+
58
+ from sql2sqlx.keywords import RESERVED
59
+ from sql2sqlx.lexer import (
60
+ BACKTICK,
61
+ EOF,
62
+ IDENT,
63
+ NUMBER,
64
+ OP,
65
+ PARAM,
66
+ STRING,
67
+ Token,
68
+ unquote_identifier,
69
+ )
70
+ from sql2sqlx.model import (
71
+ ActionDraft,
72
+ ActionType,
73
+ ConversionOptions,
74
+ IfNotExistsStrategy,
75
+ InsertStrategy,
76
+ MergeStrategy,
77
+ PlainCreateStrategy,
78
+ RefSite,
79
+ TableName,
80
+ )
81
+ from sql2sqlx.refs import parse_table_path, scan_ref_sites
82
+ from sql2sqlx.splitter import RawStatement
83
+
84
+ #: Sentinel: an OPTIONS value kept as raw SQL text (-> ``additionalOptions``).
85
+ RAW = object()
86
+
87
+ _IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z_0-9]*$")
88
+
89
+
90
+ def _dataform_merge_column(name: str) -> bool:
91
+ """Whether Dataform can emit ``name`` in an incremental MERGE.
92
+
93
+ Dataform Core 3.x quotes target/INSERT columns, but emits unique keys and
94
+ source-side UPDATE references without quoting. Those names must therefore
95
+ be simple, unreserved GoogleSQL identifiers.
96
+ """
97
+ return bool(_IDENT_RE.fullmatch(name)) and name.upper() not in RESERVED
98
+
99
+
100
+ def _dataform_insert_column(name: str) -> bool:
101
+ """Whether Dataform can safely backtick-quote an incremental column.
102
+
103
+ The BigQuery adapter wraps existing-table metadata names in raw backticks
104
+ without escaping identifier terminators or backslashes. Control and line
105
+ separator characters are rejected as well so the generated runtime SQL
106
+ cannot be split or reinterpreted.
107
+ """
108
+ return bool(name) and all(
109
+ character not in ("`", "\\")
110
+ and ord(character) >= 0x20
111
+ and not (0x7F <= ord(character) <= 0x9F)
112
+ and ord(character) not in (0x2028, 0x2029)
113
+ for character in name
114
+ )
115
+
116
+
117
+ #: Reserved keywords that terminate an expression and may precede an
118
+ #: implicit alias (``SELECT x IS NULL flag`` -> alias ``flag``).
119
+ _EXPRESSION_END_KEYWORDS = frozenset({"TRUE", "FALSE", "NULL", "END"})
120
+
121
+ #: Keywords ending a select list at nesting depth 0.
122
+ _SELECT_LIST_ENDERS = frozenset(
123
+ {
124
+ "FROM",
125
+ "UNION",
126
+ "INTERSECT",
127
+ "EXCEPT",
128
+ "LIMIT",
129
+ "ORDER",
130
+ "WHERE",
131
+ "GROUP",
132
+ "HAVING",
133
+ "QUALIFY",
134
+ "WINDOW",
135
+ }
136
+ )
137
+
138
+
139
+ # ---------------------------------------------------------------------------
140
+ # Small token utilities
141
+ # ---------------------------------------------------------------------------
142
+
143
+
144
+ def _with_eof(stmt: RawStatement) -> List[Token]:
145
+ """Return the statement's tokens with a guaranteed trailing EOF token.
146
+
147
+ Args:
148
+ stmt: The raw statement.
149
+
150
+ Returns:
151
+ The token list, EOF-terminated (a synthetic EOF is appended when
152
+ the splitter's slice did not include one).
153
+ """
154
+ toks = stmt.tokens
155
+ if toks and toks[-1].kind == EOF:
156
+ return list(toks)
157
+ return list(toks) + [Token(EOF, "", stmt.end, stmt.end)]
158
+
159
+
160
+ def _kw(toks: Sequence[Token], i: int) -> str:
161
+ """Uppercase text of ``toks[i]`` if it is an identifier, else ``""``.
162
+
163
+ Args:
164
+ toks: Token list.
165
+ i: Index (out-of-range indices are safe).
166
+
167
+ Returns:
168
+ The uppercased identifier text, or the empty string.
169
+ """
170
+ if 0 <= i < len(toks) and toks[i].kind == IDENT:
171
+ return toks[i].upper
172
+ return ""
173
+
174
+
175
+ def _is_op(toks: Sequence[Token], i: int, text: str) -> bool:
176
+ """True when ``toks[i]`` is the operator/punctuation ``text``."""
177
+ return 0 <= i < len(toks) and toks[i].kind == OP and toks[i].text == text
178
+
179
+
180
+ def _skip_balanced(toks: Sequence[Token], i: int) -> int:
181
+ """Skip a balanced parenthesis group starting at ``toks[i] == '('``.
182
+
183
+ Args:
184
+ toks: Token list (EOF-terminated).
185
+ i: Index of the opening parenthesis.
186
+
187
+ Returns:
188
+ Index of the first token after the matching ``)``. If the group is
189
+ unbalanced (invalid SQL), the EOF index is returned so callers
190
+ terminate gracefully.
191
+ """
192
+ if not _is_op(toks, i, "("):
193
+ return i + 1
194
+ depth = 0
195
+ n = len(toks)
196
+ while i < n:
197
+ t = toks[i]
198
+ if t.kind == EOF:
199
+ break
200
+ if t.kind == OP:
201
+ if t.text == "(":
202
+ depth += 1
203
+ elif t.text == ")":
204
+ depth -= 1
205
+ if depth == 0:
206
+ return i + 1
207
+ i += 1
208
+ return n - 1
209
+
210
+
211
+ def _body_head_ok(toks: Sequence[Token], i: int) -> bool:
212
+ """True when ``toks[i]`` can start a query body.
213
+
214
+ Accepts ``SELECT``, ``WITH``, a parenthesized query, and ``FROM``
215
+ (BigQuery pipe-syntax queries start with ``FROM``).
216
+ """
217
+ if i >= len(toks):
218
+ return False
219
+ t = toks[i]
220
+ if t.kind == OP and t.text == "(":
221
+ return True
222
+ return t.kind == IDENT and t.upper in ("SELECT", "WITH", "FROM")
223
+
224
+
225
+ def _quote_ident(name: str) -> str:
226
+ """Render ``name`` as a safe SQL identifier (backticked if needed).
227
+
228
+ Args:
229
+ name: The identifier text (already decoded).
230
+
231
+ Returns:
232
+ ``name`` unchanged when it is a plain unreserved identifier,
233
+ otherwise a backtick-quoted, escaped form.
234
+ """
235
+ if _IDENT_RE.match(name) and name.upper() not in RESERVED:
236
+ return name
237
+ control_escapes = {
238
+ "\a": "\\a",
239
+ "\b": "\\b",
240
+ "\f": "\\f",
241
+ "\n": "\\n",
242
+ "\r": "\\r",
243
+ "\t": "\\t",
244
+ "\v": "\\v",
245
+ }
246
+ escaped = "".join(
247
+ (
248
+ "\\\\"
249
+ if char == "\\"
250
+ else (
251
+ "\\`"
252
+ if char == "`"
253
+ else control_escapes.get(
254
+ char,
255
+ (
256
+ f"\\x{ord(char):02X}"
257
+ if ord(char) < 0x20 or 0x7F <= ord(char) <= 0x9F
258
+ else f"\\u{ord(char):04X}" if ord(char) in (0x2028, 0x2029) else char
259
+ ),
260
+ )
261
+ )
262
+ )
263
+ for char in name
264
+ )
265
+ quoted = "`" + escaped + "`"
266
+ # A SQLX placeholder cannot sit between BigQuery backticks. If the
267
+ # generated identifier contains `${`, emit the entire token as a constant
268
+ # JavaScript expression so compilation reconstructs it exactly.
269
+ if "${" in quoted:
270
+ return "${" + json.dumps(quoted, ensure_ascii=False) + "}"
271
+ return quoted
272
+
273
+
274
+ # ---------------------------------------------------------------------------
275
+ # String / OPTIONS parsing
276
+ # ---------------------------------------------------------------------------
277
+
278
+ _SIMPLE_ESCAPES = {
279
+ "a": "\a",
280
+ "b": "\b",
281
+ "f": "\f",
282
+ "n": "\n",
283
+ "r": "\r",
284
+ "t": "\t",
285
+ "v": "\v",
286
+ "\\": "\\",
287
+ "?": "?",
288
+ '"': '"',
289
+ "'": "'",
290
+ "`": "`",
291
+ }
292
+
293
+ _STRING_HEAD_RE = re.compile(r"([rRbB]{0,2})('''|\"\"\"|'|\")")
294
+
295
+
296
+ def _decode_string(literal: str) -> Optional[str]:
297
+ """Decode a GoogleSQL string literal token to a Python string.
298
+
299
+ Args:
300
+ literal: The exact token text, including quotes and any prefix.
301
+
302
+ Returns:
303
+ The decoded text, or ``None`` for bytes literals and malformed
304
+ input (callers then treat the value as raw SQL).
305
+ """
306
+ m = _STRING_HEAD_RE.match(literal)
307
+ if not m:
308
+ return None
309
+ prefix = m.group(1).lower()
310
+ if "b" in prefix:
311
+ return None
312
+ quote = m.group(2)
313
+ body = literal[m.end() : -len(quote)]
314
+ if "r" in prefix or "\\" not in body:
315
+ return body
316
+ out: List[str] = []
317
+ i, size = 0, len(body)
318
+ while i < size:
319
+ c = body[i]
320
+ if c != "\\" or i + 1 >= size:
321
+ out.append(c)
322
+ i += 1
323
+ continue
324
+ nxt = body[i + 1]
325
+ if nxt in _SIMPLE_ESCAPES:
326
+ out.append(_SIMPLE_ESCAPES[nxt])
327
+ i += 2
328
+ elif nxt in ("x", "X"):
329
+ try:
330
+ out.append(chr(int(body[i + 2 : i + 4], 16)))
331
+ i += 4
332
+ except ValueError:
333
+ out.append(nxt)
334
+ i += 2
335
+ elif nxt == "u":
336
+ try:
337
+ out.append(chr(int(body[i + 2 : i + 6], 16)))
338
+ i += 6
339
+ except ValueError:
340
+ out.append(nxt)
341
+ i += 2
342
+ elif nxt == "U":
343
+ try:
344
+ out.append(chr(int(body[i + 2 : i + 10], 16)))
345
+ i += 10
346
+ except ValueError:
347
+ out.append(nxt)
348
+ i += 2
349
+ elif nxt in "01234567":
350
+ j = i + 1
351
+ while j < min(i + 4, size) and body[j] in "01234567":
352
+ j += 1
353
+ out.append(chr(int(body[i + 1 : j], 8)))
354
+ i = j
355
+ else:
356
+ out.append(nxt)
357
+ i += 2
358
+ return "".join(out)
359
+
360
+
361
+ def _num(s: str) -> Any:
362
+ """Parse a NUMBER token's text into ``int``/``float`` (RAW on failure)."""
363
+ try:
364
+ if re.fullmatch(r"0[xX][0-9A-Fa-f]+", s):
365
+ return int(s, 16)
366
+ if re.fullmatch(r"\d+", s):
367
+ return int(s)
368
+ value = float(s)
369
+ return value if math.isfinite(value) else RAW
370
+ except ValueError: # pragma: no cover - lexer guarantees numeric shape
371
+ return RAW
372
+
373
+
374
+ def _parse_label_array(vtoks: Sequence[Token]) -> Optional[Dict[str, str]]:
375
+ """Strictly parse a labels value ``[("k", "v"), ...]`` into a dict.
376
+
377
+ Args:
378
+ vtoks: The value's tokens.
379
+
380
+ Returns:
381
+ The label mapping, or ``None`` if the value has any other shape
382
+ (the caller then keeps the raw SQL in ``additionalOptions``).
383
+ """
384
+ if len(vtoks) < 2:
385
+ return None
386
+ if not (vtoks[0].kind == OP and vtoks[0].text == "["):
387
+ return None
388
+ if not (vtoks[-1].kind == OP and vtoks[-1].text == "]"):
389
+ return None
390
+ out: Dict[str, str] = {}
391
+ j, end = 1, len(vtoks) - 1
392
+ while j < end:
393
+ if j + 5 > end or not (vtoks[j].kind == OP and vtoks[j].text == "("):
394
+ return None
395
+ k, comma, v, close = vtoks[j + 1], vtoks[j + 2], vtoks[j + 3], vtoks[j + 4]
396
+ if k.kind != STRING or v.kind != STRING:
397
+ return None
398
+ if not (comma.kind == OP and comma.text == ","):
399
+ return None
400
+ if not (close.kind == OP and close.text == ")"):
401
+ return None
402
+ ks, vs = _decode_string(k.text), _decode_string(v.text)
403
+ if ks is None or vs is None:
404
+ return None
405
+ out[ks] = vs
406
+ j += 5
407
+ if j < end:
408
+ if not (vtoks[j].kind == OP and vtoks[j].text == ","):
409
+ return None
410
+ j += 1
411
+ return out
412
+
413
+
414
+ def _parse_option_value(vtoks: Sequence[Token]) -> Any:
415
+ """Interpret an OPTIONS value's tokens as a simple literal if possible.
416
+
417
+ Args:
418
+ vtoks: The value's tokens (never empty).
419
+
420
+ Returns:
421
+ ``str``/``int``/``float``/``bool``/``dict`` for recognized literal
422
+ shapes, otherwise the :data:`RAW` sentinel.
423
+ """
424
+ if len(vtoks) == 1:
425
+ t = vtoks[0]
426
+ if t.kind == STRING:
427
+ s = _decode_string(t.text)
428
+ return s if s is not None else RAW
429
+ if t.kind == NUMBER:
430
+ return _num(t.text)
431
+ if t.kind == IDENT:
432
+ if t.upper == "TRUE":
433
+ return True
434
+ if t.upper == "FALSE":
435
+ return False
436
+ return RAW
437
+ if len(vtoks) == 2 and vtoks[0].kind == OP and vtoks[0].text == "-" and vtoks[1].kind == NUMBER:
438
+ v = _num(vtoks[1].text)
439
+ return -v if isinstance(v, (int, float)) else RAW
440
+ labels = _parse_label_array(vtoks)
441
+ return labels if labels is not None else RAW
442
+
443
+
444
+ def _parse_options(
445
+ toks: Sequence[Token], i: int, text: str
446
+ ) -> Optional[Tuple[List[Tuple[str, str, Any]], int]]:
447
+ """Parse an ``OPTIONS(key = value, ...)`` clause.
448
+
449
+ Args:
450
+ toks: Token list; ``toks[i]`` must be the ``OPTIONS`` identifier.
451
+ i: Index of ``OPTIONS``.
452
+ text: Full source text (for raw value capture).
453
+
454
+ Returns:
455
+ ``(entries, next_index)`` where each entry is
456
+ ``(key, raw_sql_text, parsed_value_or_RAW)``, or ``None`` when the
457
+ clause cannot be parsed (caller falls back to operations).
458
+ """
459
+ j = i + 1
460
+ if not _is_op(toks, j, "("):
461
+ return None
462
+ j += 1
463
+ entries: List[Tuple[str, str, Any]] = []
464
+ n = len(toks)
465
+ while j < n:
466
+ if _is_op(toks, j, ")"):
467
+ return entries, j + 1
468
+ kt = toks[j]
469
+ if kt.kind not in (IDENT, BACKTICK):
470
+ return None
471
+ key = unquote_identifier(kt.text)
472
+ j += 1
473
+ if not _is_op(toks, j, "="):
474
+ return None
475
+ j += 1
476
+ vstart = j
477
+ depth = 0
478
+ while j < n and toks[j].kind != EOF:
479
+ t = toks[j]
480
+ if t.kind == OP:
481
+ if t.text in ("(", "["):
482
+ depth += 1
483
+ elif t.text in (")", "]"):
484
+ if depth == 0 and t.text == ")":
485
+ break
486
+ depth -= 1
487
+ elif t.text == "," and depth == 0:
488
+ break
489
+ j += 1
490
+ if j >= n or toks[j].kind == EOF or j == vstart:
491
+ return None
492
+ vtoks = toks[vstart:j]
493
+ raw = text[vtoks[0].start : vtoks[-1].end]
494
+ entries.append((key, raw, _parse_option_value(vtoks)))
495
+ if _is_op(toks, j, ","):
496
+ j += 1
497
+ return None
498
+
499
+
500
+ # ---------------------------------------------------------------------------
501
+ # Select-list machinery (shared by INSERT column lists, VIEW column lists
502
+ # and the safe-MERGE analyzer)
503
+ # ---------------------------------------------------------------------------
504
+
505
+
506
+ class _SelectItem:
507
+ """One top-level select-list item and its safety flags.
508
+
509
+ Attributes:
510
+ toks: The item's tokens.
511
+ interval: True when a top-level ``INTERVAL`` keyword makes
512
+ implicit-alias detection ambiguous.
513
+ generic: True when a naked ``STRUCT<``/``ARRAY<`` constructor makes
514
+ comma splitting ambiguous.
515
+ star: True when the item is a ``*`` / ``t.*`` expansion.
516
+ """
517
+
518
+ __slots__ = ("toks", "interval", "generic", "star")
519
+
520
+ def __init__(self) -> None:
521
+ """Create an empty item."""
522
+ self.toks: List[Token] = []
523
+ self.interval = False
524
+ self.generic = False
525
+ self.star = False
526
+
527
+
528
+ def _split_select_items(toks: Sequence[Token], ts: int, te: int) -> Optional[List[_SelectItem]]:
529
+ """Split the select list of the query spanning ``toks[ts:te)``.
530
+
531
+ Handles wrapper parentheses around the whole query, a leading ``WITH``
532
+ clause (CTEs are skipped, not inspected), and ``DISTINCT``/``ALL``
533
+ modifiers. BigQuery's trailing comma before ``FROM`` is accepted.
534
+
535
+ Args:
536
+ toks: Token list (EOF-terminated).
537
+ ts: Index of the query's first token.
538
+ te: Exclusive end index of the query (usually the EOF index).
539
+
540
+ Returns:
541
+ The list of items, or ``None`` for shapes that cannot be handled
542
+ (missing ``SELECT``, ``SELECT AS STRUCT/VALUE``,
543
+ ``SELECT WITH DIFFERENTIAL_PRIVACY``, malformed ``WITH``...).
544
+ """
545
+ n = len(toks)
546
+ te = min(te, n)
547
+ while (
548
+ ts < te
549
+ and _is_op(toks, ts, "(")
550
+ and _skip_balanced(toks, ts) == te
551
+ and _is_op(toks, te - 1, ")")
552
+ ):
553
+ ts += 1
554
+ te -= 1
555
+ j = ts
556
+ if _kw(toks, j) == "WITH":
557
+ j += 1
558
+ if _kw(toks, j) == "RECURSIVE":
559
+ j += 1
560
+ while True:
561
+ if j >= te or toks[j].kind not in (IDENT, BACKTICK):
562
+ return None
563
+ if toks[j].kind == IDENT and toks[j].upper in RESERVED:
564
+ return None
565
+ j += 1
566
+ if _is_op(toks, j, "("):
567
+ j = _skip_balanced(toks, j)
568
+ if _kw(toks, j) != "AS":
569
+ return None
570
+ j += 1
571
+ if not _is_op(toks, j, "("):
572
+ return None
573
+ j = _skip_balanced(toks, j)
574
+ if _is_op(toks, j, ","):
575
+ j += 1
576
+ continue
577
+ break
578
+ if _kw(toks, j) != "SELECT":
579
+ return None
580
+ j += 1
581
+ if _kw(toks, j) in ("DISTINCT", "ALL"):
582
+ j += 1
583
+ if _kw(toks, j) in ("AS", "WITH"):
584
+ return None
585
+ items: List[_SelectItem] = []
586
+ cur = _SelectItem()
587
+ depth = 0
588
+ k = j
589
+ while k < te:
590
+ t = toks[k]
591
+ if t.kind == EOF:
592
+ break
593
+ if t.kind == OP:
594
+ x = t.text
595
+ if x in ("(", "["):
596
+ depth += 1
597
+ elif x in (")", "]"):
598
+ depth -= 1
599
+ if depth < 0:
600
+ break
601
+ elif depth == 0:
602
+ if x == ",":
603
+ if not cur.toks:
604
+ return None
605
+ items.append(cur)
606
+ cur = _SelectItem()
607
+ k += 1
608
+ continue
609
+ if x == "|>":
610
+ break
611
+ if x == "*":
612
+ prev = cur.toks[-1] if cur.toks else None
613
+ if prev is None or (prev.kind == OP and prev.text == "."):
614
+ cur.star = True
615
+ elif t.kind == IDENT and depth == 0:
616
+ u = t.upper
617
+ if u in _SELECT_LIST_ENDERS:
618
+ break
619
+ if u == "INTERVAL":
620
+ cur.interval = True
621
+ elif u in ("STRUCT", "ARRAY") and _is_op(toks, k + 1, "<"):
622
+ cur.generic = True
623
+ cur.toks.append(t)
624
+ k += 1
625
+ if cur.toks:
626
+ items.append(cur)
627
+ if not items:
628
+ return None
629
+ return items
630
+
631
+
632
+ def _item_output_info(item: _SelectItem) -> Optional[Tuple[str, Optional[str], Optional[Token]]]:
633
+ """Determine a select-list item's output name and alias structure.
634
+
635
+ Args:
636
+ item: The item to analyze.
637
+
638
+ Returns:
639
+ ``(mode, name, alias_token)`` where mode is one of:
640
+
641
+ * ``"explicit"`` - ``expr AS alias`` (name = alias);
642
+ * ``"implicit"`` - ``expr alias`` (name = alias);
643
+ * ``"bare"`` - a plain column/path (name = last segment);
644
+ * ``"anon"`` - an expression with no derivable output name.
645
+
646
+ Returns ``None`` when the item is *unsafe* to reason about
647
+ (star expansion, ambiguous ``INTERVAL``/generic constructors).
648
+ """
649
+ if item.star:
650
+ return None
651
+ t = item.toks
652
+ last = t[-1]
653
+ if len(t) == 1:
654
+ if last.kind == BACKTICK:
655
+ return ("bare", unquote_identifier(last.text), None)
656
+ if last.kind == IDENT and last.upper not in RESERVED:
657
+ return ("bare", last.text, None)
658
+ return ("anon", None, None)
659
+ penult = t[-2]
660
+ last_is_name = last.kind == BACKTICK or (last.kind == IDENT and last.upper not in RESERVED)
661
+ if penult.kind == IDENT and penult.upper == "AS" and last_is_name:
662
+ return ("explicit", unquote_identifier(last.text), last)
663
+ if last_is_name:
664
+ if penult.kind == OP and penult.text == ".":
665
+ return ("bare", unquote_identifier(last.text), None)
666
+ if item.interval or item.generic:
667
+ return None
668
+ terminal = (
669
+ penult.kind in (BACKTICK, NUMBER, STRING, PARAM)
670
+ or (penult.kind == OP and penult.text in (")", "]"))
671
+ or (
672
+ penult.kind == IDENT
673
+ and (penult.upper not in RESERVED or penult.upper in _EXPRESSION_END_KEYWORDS)
674
+ )
675
+ )
676
+ if terminal:
677
+ return ("implicit", unquote_identifier(last.text), last)
678
+ return ("anon", None, None)
679
+
680
+
681
+ def _alias_select_list_edits(
682
+ toks: Sequence[Token], ts: int, te: int, cols: Sequence[str]
683
+ ) -> Optional[List[Tuple[int, int, str]]]:
684
+ """Compute span edits forcing the select list to output ``cols``.
685
+
686
+ Args:
687
+ toks: Token list (EOF-terminated).
688
+ ts: Index of the query's first token.
689
+ te: Exclusive end index of the query.
690
+ cols: Required output column names, in order.
691
+
692
+ Returns:
693
+ Absolute-offset span edits, or ``None`` when the rewrite is not
694
+ provably safe (arity mismatch, star expansion, ambiguous items).
695
+ """
696
+ items = _split_select_items(toks, ts, te)
697
+ if items is None or len(items) != len(cols):
698
+ return None
699
+ infos = []
700
+ for it in items:
701
+ info = _item_output_info(it)
702
+ if info is None:
703
+ return None
704
+ infos.append(info)
705
+ changed_aliases = set()
706
+ for (mode, name, _alias), col in zip(infos, cols):
707
+ if name is not None and name.upper() == col.upper():
708
+ continue # no edit for this item
709
+ if mode in ("explicit", "implicit") and name is not None:
710
+ # The old alias disappears; later references to it would
711
+ # re-resolve (or break).
712
+ changed_aliases.add(name.upper())
713
+ # The new alias appears; BigQuery resolves SELECT aliases in
714
+ # preference to FROM columns inside GROUP BY/HAVING/QUALIFY/
715
+ # ORDER BY, so a same-named column reference there would silently
716
+ # start meaning this item's expression instead.
717
+ changed_aliases.add(col.upper())
718
+ if changed_aliases:
719
+ # SELECT aliases are visible to GROUP BY, HAVING, QUALIFY, ORDER BY
720
+ # and subsequent pipe operators. Adding, removing or replacing an
721
+ # alias without rewriting such references would either change which
722
+ # expression is used or make the query invalid. Keep the original
723
+ # statement as operations instead of guessing.
724
+ list_end = max(item.toks[-1].end for item in items)
725
+ depth = 0
726
+ alias_visible = False
727
+ for token in toks:
728
+ if token.start < list_end or token.kind == EOF or token.start >= toks[te].start:
729
+ continue
730
+ if token.kind == OP:
731
+ if token.text in ("(", "["):
732
+ depth += 1
733
+ elif token.text in (")", "]"):
734
+ depth = max(0, depth - 1)
735
+ elif token.text == "|>" and depth == 0:
736
+ alias_visible = True
737
+ continue
738
+ if token.kind not in (IDENT, BACKTICK):
739
+ continue
740
+ word = token.upper if token.kind == IDENT else unquote_identifier(token.text).upper()
741
+ if (
742
+ depth == 0
743
+ and token.kind == IDENT
744
+ and word in ("GROUP", "HAVING", "QUALIFY", "ORDER")
745
+ ):
746
+ alias_visible = True
747
+ continue
748
+ if (
749
+ depth == 0
750
+ and token.kind == IDENT
751
+ and word in ("LIMIT", "UNION", "INTERSECT", "EXCEPT")
752
+ ):
753
+ alias_visible = False
754
+ continue
755
+ if alias_visible and word in changed_aliases:
756
+ return None
757
+ edits: List[Tuple[int, int, str]] = []
758
+ for it, (_mode, name, alias_tok), col in zip(items, infos, cols):
759
+ if name is not None and name.upper() == col.upper():
760
+ continue
761
+ quoted = _quote_ident(col)
762
+ if alias_tok is not None:
763
+ edits.append((alias_tok.start, alias_tok.end, quoted))
764
+ else:
765
+ end = it.toks[-1].end
766
+ edits.append((end, end, " AS " + quoted))
767
+ return edits
768
+
769
+
770
+ def _select_output_names(toks: Sequence[Token], ts: int, te: int) -> Optional[List[str]]:
771
+ """Derive the output column names of the query in ``toks[ts:te)``.
772
+
773
+ Args:
774
+ toks: Token list.
775
+ ts: Index of the query's first token.
776
+ te: Exclusive end index.
777
+
778
+ Returns:
779
+ The names, or ``None`` when any item's name cannot be determined
780
+ exactly.
781
+ """
782
+ items = _split_select_items(toks, ts, te)
783
+ if items is None:
784
+ return None
785
+ names: List[str] = []
786
+ for it in items:
787
+ info = _item_output_info(it)
788
+ if info is None or info[1] is None:
789
+ return None
790
+ names.append(info[1])
791
+ return names
792
+
793
+
794
+ def _parse_ident_list(toks: Sequence[Token], i: int) -> Tuple[Optional[List[str]], int]:
795
+ """Parse a plain parenthesized name list ``(a, b, ...)``.
796
+
797
+ Args:
798
+ toks: Token list; ``toks[i]`` must be ``(``.
799
+ i: Index of the opening parenthesis.
800
+
801
+ Returns:
802
+ ``(names, next_index)`` on success, ``(None, i)`` otherwise.
803
+ """
804
+ j = i + 1
805
+ names: List[str] = []
806
+ n = len(toks)
807
+ while j < n:
808
+ t = toks[j]
809
+ if t.kind == BACKTICK:
810
+ names.append(unquote_identifier(t.text))
811
+ elif t.kind == IDENT and t.upper not in RESERVED:
812
+ names.append(t.text)
813
+ else:
814
+ return None, i
815
+ j += 1
816
+ if _is_op(toks, j, ","):
817
+ j += 1
818
+ continue
819
+ if _is_op(toks, j, ")"):
820
+ return names, j + 1
821
+ return None, i
822
+ return None, i
823
+
824
+
825
+ # ---------------------------------------------------------------------------
826
+ # Draft helpers
827
+ # ---------------------------------------------------------------------------
828
+
829
+
830
+ def _ops_draft(
831
+ stmt: RawStatement,
832
+ kind: str,
833
+ *,
834
+ target: Optional[TableName] = None,
835
+ writes: bool = False,
836
+ tspan: Optional[Tuple[int, int]] = None,
837
+ warnings: Optional[List[Tuple[str, str, int]]] = None,
838
+ ) -> ActionDraft:
839
+ """Build a verbatim ``operations`` draft for ``stmt``.
840
+
841
+ Args:
842
+ stmt: The raw statement.
843
+ kind: Uppercase original statement kind (for annotation/report).
844
+ target: Table written by the statement, if identifiable.
845
+ writes: True when the statement mutates ``target``.
846
+ tspan: Span of the target path (enables ``${self()}`` substitution
847
+ if the linker elects this action as the target's producer).
848
+ warnings: ``(code, message, offset)`` findings.
849
+
850
+ Returns:
851
+ The draft.
852
+ """
853
+ return ActionDraft(
854
+ action_type=ActionType.OPERATIONS,
855
+ target=target,
856
+ writes_target=writes,
857
+ creates_target=False,
858
+ body_start=stmt.start,
859
+ body_end=stmt.end,
860
+ stmt_start=stmt.start,
861
+ stmt_end=stmt.terminator_end,
862
+ primary_target_span=tspan,
863
+ original_kind=kind,
864
+ warnings=list(warnings or []),
865
+ )
866
+
867
+
868
+ def _finish(
869
+ draft: ActionDraft, toks: Sequence[Token], head: str, skip_spans: Sequence[Tuple[int, int]] = ()
870
+ ) -> ActionDraft:
871
+ """Attach reference sites to a draft and return it.
872
+
873
+ Args:
874
+ draft: The draft under construction.
875
+ toks: The statement's tokens.
876
+ head: Uppercase leading keyword (``"MERGE"`` enables the USING
877
+ table introducer in the scanner).
878
+ skip_spans: Spans that must not yield reference sites (e.g. the
879
+ ``DELETE FROM`` target).
880
+
881
+ Returns:
882
+ The same draft, with ``ref_sites`` populated.
883
+ """
884
+ draft.ref_sites = scan_ref_sites(toks, head, skip_spans)
885
+ return draft
886
+
887
+
888
+ def _append_table_ref(draft: ActionDraft, toks: Sequence[Token], i: int) -> None:
889
+ """Append a table reference introduced by a DDL-specific keyword."""
890
+ match = parse_table_path(toks, i)
891
+ if match is None or not (1 <= len(match.parts) <= 3) or not all(match.parts):
892
+ return
893
+ draft.ref_sites.append(
894
+ RefSite(
895
+ match.start,
896
+ match.end,
897
+ TableName.from_parts(list(match.parts)),
898
+ )
899
+ )
900
+
901
+
902
+ # ---------------------------------------------------------------------------
903
+ # Top-level dispatcher
904
+ # ---------------------------------------------------------------------------
905
+
906
+
907
+ def classify_statement(stmt: RawStatement, text: str, opts: ConversionOptions) -> ActionDraft:
908
+ """Classify one top-level statement into a Dataform action draft.
909
+
910
+ Args:
911
+ stmt: The raw statement from the splitter.
912
+ text: Full source text of the file (for raw span capture).
913
+ opts: Conversion options.
914
+
915
+ Returns:
916
+ An :class:`~sql2sqlx.model.ActionDraft`. This function never
917
+ raises for valid-but-unsupported SQL - such statements become
918
+ ``operations`` drafts with explanatory warnings.
919
+ """
920
+ toks = _with_eof(stmt)
921
+ first = toks[0]
922
+ head = first.upper if first.kind == IDENT else ""
923
+ if head == "CREATE":
924
+ return _classify_create(stmt, toks, text, opts)
925
+ if head == "INSERT":
926
+ return _classify_insert(stmt, toks, text, opts)
927
+ if head == "MERGE":
928
+ return _classify_merge(stmt, toks, text, opts)
929
+ if head in ("UPDATE", "DELETE", "TRUNCATE", "DROP", "ALTER", "LOAD"):
930
+ return _classify_dml(stmt, toks, head)
931
+ if head == "CALL" or (head == "EXECUTE" and _kw(toks, 1) == "IMMEDIATE"):
932
+ draft = _ops_draft(
933
+ stmt,
934
+ head,
935
+ warnings=[
936
+ (
937
+ "DYNAMIC_SIDE_EFFECTS",
938
+ "Called procedures and dynamic SQL can hide table reads or "
939
+ "writes; kept verbatim, but review manual dependencies.",
940
+ stmt.start,
941
+ )
942
+ ],
943
+ )
944
+ return _finish(draft, toks, head)
945
+ if head in ("SELECT", "WITH", "FROM") or (first.kind == OP and first.text == "("):
946
+ draft = _ops_draft(
947
+ stmt,
948
+ head or "SELECT",
949
+ warnings=[
950
+ (
951
+ "ORPHAN_SELECT",
952
+ "Standalone query converted to an operations action; review "
953
+ "whether it should be a table, view or assertion instead.",
954
+ stmt.start,
955
+ )
956
+ ],
957
+ )
958
+ return _finish(draft, toks, head or "SELECT")
959
+ return _finish(_ops_draft(stmt, head or "STATEMENT"), toks, head)
960
+
961
+
962
+ # ---------------------------------------------------------------------------
963
+ # CREATE handling
964
+ # ---------------------------------------------------------------------------
965
+
966
+
967
+ def _classify_create(
968
+ stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
969
+ ) -> ActionDraft:
970
+ """Classify a ``CREATE ...`` statement (dispatch on entity/modifiers)."""
971
+ i = 1
972
+ or_replace = False
973
+ if _kw(toks, i) == "OR" and _kw(toks, i + 1) == "REPLACE":
974
+ or_replace = True
975
+ i += 2
976
+ temp = _kw(toks, i) in ("TEMP", "TEMPORARY")
977
+ if temp:
978
+ i += 1
979
+ materialized = _kw(toks, i) == "MATERIALIZED"
980
+ if materialized:
981
+ i += 1
982
+ external = _kw(toks, i) == "EXTERNAL"
983
+ if external:
984
+ i += 1
985
+ snapshot = _kw(toks, i) == "SNAPSHOT"
986
+ if snapshot:
987
+ i += 1
988
+ entity = _kw(toks, i)
989
+
990
+ if entity == "TABLE" and _kw(toks, i + 1) == "FUNCTION":
991
+ return _finish(_ops_draft(stmt, "CREATE TABLE FUNCTION"), toks, "CREATE")
992
+ if entity == "PROCEDURE":
993
+ # The body is stored for later execution; DML within it is not an
994
+ # immediate read/write dependency of the CREATE action. Keeping the
995
+ # definition source-faithful also preserves the procedure owner's
996
+ # default-project resolution rules.
997
+ return _ops_draft(
998
+ stmt,
999
+ "CREATE PROCEDURE",
1000
+ warnings=[
1001
+ (
1002
+ "PROCEDURE_PRESERVED",
1003
+ "Stored procedure definition kept verbatim; its body is not "
1004
+ "linked as an immediately executed workflow dependency.",
1005
+ stmt.start,
1006
+ )
1007
+ ],
1008
+ )
1009
+ if entity not in ("TABLE", "VIEW"):
1010
+ kind = f"CREATE {entity}" if entity else "CREATE"
1011
+ return _finish(_ops_draft(stmt, kind), toks, "CREATE")
1012
+ i += 1
1013
+ ine = False
1014
+ if _kw(toks, i) == "IF" and _kw(toks, i + 1) == "NOT" and _kw(toks, i + 2) == "EXISTS":
1015
+ ine = True
1016
+ i += 3
1017
+ pm = parse_table_path(toks, i)
1018
+ if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
1019
+ draft = _ops_draft(
1020
+ stmt,
1021
+ f"CREATE {entity}",
1022
+ warnings=[
1023
+ (
1024
+ "FALLBACK_OPERATIONS",
1025
+ f"Could not parse the CREATE {entity} target path; " "statement kept verbatim.",
1026
+ stmt.start,
1027
+ )
1028
+ ],
1029
+ )
1030
+ return _finish(draft, toks, "CREATE")
1031
+ target = TableName.from_parts(list(pm.parts))
1032
+ tspan = (pm.start, pm.end)
1033
+ i = pm.next_index
1034
+
1035
+ if temp:
1036
+ draft = _ops_draft(
1037
+ stmt,
1038
+ f"CREATE TEMP {entity}",
1039
+ warnings=[
1040
+ (
1041
+ "TEMP_TABLE",
1042
+ "Temporary objects have no Dataform action equivalent; " "kept as operations.",
1043
+ stmt.start,
1044
+ )
1045
+ ],
1046
+ )
1047
+ return _finish(draft, toks, "CREATE")
1048
+ if external:
1049
+ draft = _ops_draft(
1050
+ stmt,
1051
+ "CREATE EXTERNAL TABLE",
1052
+ target=target,
1053
+ writes=True,
1054
+ tspan=tspan,
1055
+ warnings=[
1056
+ (
1057
+ "EXTERNAL_TABLE",
1058
+ f"External table DDL for {target.display()} kept as "
1059
+ "operations; consider replacing it with a Dataform "
1060
+ "declaration if the table is managed elsewhere.",
1061
+ stmt.start,
1062
+ )
1063
+ ],
1064
+ )
1065
+ return _finish(draft, toks, "CREATE")
1066
+ if snapshot:
1067
+ draft = _ops_draft(
1068
+ stmt,
1069
+ "CREATE SNAPSHOT TABLE",
1070
+ target=target,
1071
+ writes=True,
1072
+ tspan=tspan,
1073
+ warnings=[("SNAPSHOT_TABLE", "Snapshot DDL kept as operations.", stmt.start)],
1074
+ )
1075
+ _finish(draft, toks, "CREATE")
1076
+ if _kw(toks, i) == "CLONE":
1077
+ _append_table_ref(draft, toks, i + 1)
1078
+ return draft
1079
+
1080
+ if entity == "VIEW":
1081
+ return _classify_view_body(
1082
+ stmt, toks, text, opts, i, target, tspan, materialized, or_replace, ine
1083
+ )
1084
+ return _classify_table_body(stmt, toks, text, opts, i, target, tspan, or_replace, ine)
1085
+
1086
+
1087
+ def _scan_pre_as_clauses(
1088
+ toks: List[Token], i: int, text: str, allow_columns: bool
1089
+ ) -> Union[Tuple[str, str], Dict[str, Any]]:
1090
+ """Scan the clauses between a CREATE target and ``AS``.
1091
+
1092
+ Recognizes an optional column definition list (position-gated to
1093
+ directly after the target), ``PARTITION BY``, ``CLUSTER BY`` and
1094
+ ``OPTIONS(...)``. Anything else is a strict fallback: pretending to
1095
+ understand an unknown clause would risk silently changing semantics.
1096
+
1097
+ Args:
1098
+ toks: Token list (EOF-terminated).
1099
+ i: Index of the first token after the target path.
1100
+ text: Full source text.
1101
+ allow_columns: Whether a column list may appear (tables only).
1102
+
1103
+ Returns:
1104
+ On success, a dict with keys ``as_index`` (int or ``None``),
1105
+ ``has_columns`` (bool), ``partition`` (raw text or ``None``),
1106
+ ``cluster`` (list of raw texts) and ``options`` (entries or
1107
+ ``None``). On failure, ``("fallback", reason)``.
1108
+ """
1109
+ info: Dict[str, Any] = {
1110
+ "as_index": None,
1111
+ "has_columns": False,
1112
+ "partition": None,
1113
+ "cluster": [],
1114
+ "options": None,
1115
+ }
1116
+ first = True
1117
+ n = len(toks)
1118
+ while i < n:
1119
+ t = toks[i]
1120
+ if t.kind == EOF:
1121
+ break
1122
+ if t.kind == OP and t.text == "(":
1123
+ if allow_columns and first:
1124
+ info["has_columns"] = True
1125
+ i = _skip_balanced(toks, i)
1126
+ first = False
1127
+ continue
1128
+ return ("fallback", "unexpected '(' before AS")
1129
+ kw = _kw(toks, i)
1130
+ if kw == "AS":
1131
+ info["as_index"] = i
1132
+ break
1133
+ if kw == "PARTITION" and _kw(toks, i + 1) == "BY":
1134
+ j = i + 2
1135
+ if j >= n or toks[j].kind == EOF:
1136
+ return ("fallback", "empty PARTITION BY expression")
1137
+ start = toks[j].start
1138
+ endpos = start
1139
+ depth = 0
1140
+ while j < n and toks[j].kind != EOF:
1141
+ tj = toks[j]
1142
+ if tj.kind == OP:
1143
+ if tj.text in ("(", "["):
1144
+ depth += 1
1145
+ elif tj.text in (")", "]"):
1146
+ depth -= 1
1147
+ elif (
1148
+ tj.kind == IDENT
1149
+ and depth == 0
1150
+ and tj.upper in ("CLUSTER", "OPTIONS", "AS", "DEFAULT")
1151
+ ):
1152
+ break
1153
+ endpos = tj.end
1154
+ j += 1
1155
+ if endpos <= start:
1156
+ return ("fallback", "empty PARTITION BY expression")
1157
+ info["partition"] = text[start:endpos].strip()
1158
+ i = j
1159
+ first = False
1160
+ continue
1161
+ if kw == "CLUSTER" and _kw(toks, i + 1) == "BY":
1162
+ j = i + 2
1163
+ if j >= n or toks[j].kind == EOF:
1164
+ return ("fallback", "empty CLUSTER BY list")
1165
+ depth = 0
1166
+ item_start = toks[j].start
1167
+ last_end = item_start
1168
+ items: List[str] = []
1169
+ while j < n and toks[j].kind != EOF:
1170
+ tj = toks[j]
1171
+ if tj.kind == OP:
1172
+ if tj.text in ("(", "["):
1173
+ depth += 1
1174
+ elif tj.text in (")", "]"):
1175
+ depth -= 1
1176
+ elif tj.text == "," and depth == 0:
1177
+ items.append(text[item_start:last_end].strip())
1178
+ j += 1
1179
+ if j < n and toks[j].kind != EOF:
1180
+ item_start = toks[j].start
1181
+ last_end = item_start
1182
+ continue
1183
+ elif (
1184
+ tj.kind == IDENT
1185
+ and depth == 0
1186
+ and tj.upper in ("OPTIONS", "AS", "DEFAULT", "PARTITION")
1187
+ ):
1188
+ break
1189
+ last_end = tj.end
1190
+ j += 1
1191
+ if last_end > item_start:
1192
+ items.append(text[item_start:last_end].strip())
1193
+ if not items or not all(items):
1194
+ return ("fallback", "empty CLUSTER BY list")
1195
+ info["cluster"] = items
1196
+ i = j
1197
+ first = False
1198
+ continue
1199
+ if kw == "OPTIONS":
1200
+ parsed = _parse_options(toks, i, text)
1201
+ if parsed is None:
1202
+ return ("fallback", "unparseable OPTIONS(...) clause")
1203
+ info["options"], i = parsed
1204
+ first = False
1205
+ continue
1206
+ if kw == "DEFAULT":
1207
+ return ("fallback", "DEFAULT COLLATE has no Dataform config equivalent")
1208
+ if kw in ("LIKE", "CLONE", "COPY"):
1209
+ return ("fallback", f"CREATE TABLE ... {kw} has no typed Dataform equivalent")
1210
+ return ("fallback", f"unrecognized clause {toks[i].text!r} before AS")
1211
+ return info
1212
+
1213
+
1214
+ def _apply_table_metadata(config: Dict[str, Any], info: Dict[str, Any]) -> None:
1215
+ """Map scanned PARTITION/CLUSTER/OPTIONS metadata into a config dict.
1216
+
1217
+ ``description``, ``labels``, ``partition_expiration_days`` and
1218
+ ``require_partition_filter`` map onto first-class Dataform fields when
1219
+ their values are simple literals; every other option (or a complex
1220
+ value) is preserved verbatim in ``bigquery.additionalOptions`` so the
1221
+ compiled DDL keeps it.
1222
+
1223
+ Args:
1224
+ config: Draft config dict to mutate.
1225
+ info: Result of :func:`_scan_pre_as_clauses`.
1226
+ """
1227
+ bq: Dict[str, Any] = {}
1228
+ if info.get("partition"):
1229
+ bq["partitionBy"] = info["partition"]
1230
+ if info.get("cluster"):
1231
+ bq["clusterBy"] = list(info["cluster"])
1232
+ addl: Dict[str, str] = {}
1233
+ for key, raw, val in info.get("options") or []:
1234
+ lk = key.lower()
1235
+ if lk == "description" and isinstance(val, str):
1236
+ config["description"] = val
1237
+ elif lk == "labels" and isinstance(val, dict):
1238
+ bq["labels"] = val
1239
+ elif (
1240
+ lk == "partition_expiration_days"
1241
+ and isinstance(val, (int, float))
1242
+ and not isinstance(val, bool)
1243
+ ):
1244
+ bq["partitionExpirationDays"] = val
1245
+ elif lk == "require_partition_filter" and isinstance(val, bool):
1246
+ bq["requirePartitionFilter"] = val
1247
+ else:
1248
+ addl[key] = raw
1249
+ if addl:
1250
+ bq["additionalOptions"] = addl
1251
+ if bq:
1252
+ config["bigquery"] = bq
1253
+
1254
+
1255
+ def _classify_table_body(
1256
+ stmt: RawStatement,
1257
+ toks: List[Token],
1258
+ text: str,
1259
+ opts: ConversionOptions,
1260
+ i: int,
1261
+ target: TableName,
1262
+ tspan: Tuple[int, int],
1263
+ or_replace: bool,
1264
+ ine: bool,
1265
+ ) -> ActionDraft:
1266
+ """Classify ``CREATE [OR REPLACE] TABLE`` after its target path."""
1267
+ source_keyword = _kw(toks, i)
1268
+ if source_keyword in ("LIKE", "CLONE", "COPY"):
1269
+ draft = _ops_draft(
1270
+ stmt,
1271
+ "CREATE TABLE",
1272
+ target=target,
1273
+ writes=True,
1274
+ tspan=tspan,
1275
+ warnings=[
1276
+ (
1277
+ "FALLBACK_OPERATIONS",
1278
+ f"CREATE TABLE ... {source_keyword} has no typed Dataform "
1279
+ "equivalent; kept as operations.",
1280
+ stmt.start,
1281
+ )
1282
+ ],
1283
+ )
1284
+ _finish(draft, toks, "CREATE")
1285
+ _append_table_ref(draft, toks, i + 1)
1286
+ return draft
1287
+ info = _scan_pre_as_clauses(toks, i, text, allow_columns=True)
1288
+ if isinstance(info, tuple):
1289
+ draft = _ops_draft(
1290
+ stmt,
1291
+ "CREATE TABLE",
1292
+ target=target,
1293
+ writes=True,
1294
+ tspan=tspan,
1295
+ warnings=[
1296
+ ("FALLBACK_OPERATIONS", f"CREATE TABLE kept as operations: {info[1]}.", stmt.start)
1297
+ ],
1298
+ )
1299
+ return _finish(draft, toks, "CREATE")
1300
+ if info["as_index"] is None:
1301
+ return _plain_create(stmt, toks, opts, target, tspan, or_replace, ine, info)
1302
+ if info["has_columns"]:
1303
+ draft = _ops_draft(
1304
+ stmt,
1305
+ "CREATE TABLE",
1306
+ target=target,
1307
+ writes=True,
1308
+ tspan=tspan,
1309
+ warnings=[
1310
+ (
1311
+ "COLUMN_DDL",
1312
+ "CREATE TABLE ... AS with an explicit column "
1313
+ "list can carry types or constraints that "
1314
+ "Dataform's table type cannot express; kept "
1315
+ "as operations.",
1316
+ stmt.start,
1317
+ )
1318
+ ],
1319
+ )
1320
+ return _finish(draft, toks, "CREATE")
1321
+ if ine and opts.if_not_exists_strategy is IfNotExistsStrategy.OPERATIONS:
1322
+ draft = _ops_draft(
1323
+ stmt,
1324
+ "CREATE TABLE",
1325
+ target=target,
1326
+ writes=True,
1327
+ tspan=tspan,
1328
+ warnings=[
1329
+ (
1330
+ "IF_NOT_EXISTS",
1331
+ "IF NOT EXISTS preserved verbatim as " "operations per options.",
1332
+ stmt.start,
1333
+ )
1334
+ ],
1335
+ )
1336
+ return _finish(draft, toks, "CREATE")
1337
+ bi = info["as_index"] + 1
1338
+ if not _body_head_ok(toks, bi):
1339
+ draft = _ops_draft(
1340
+ stmt,
1341
+ "CREATE TABLE",
1342
+ target=target,
1343
+ writes=True,
1344
+ tspan=tspan,
1345
+ warnings=[
1346
+ (
1347
+ "FALLBACK_OPERATIONS",
1348
+ "CREATE TABLE AS is not followed by a query; " "kept as operations.",
1349
+ stmt.start,
1350
+ )
1351
+ ],
1352
+ )
1353
+ return _finish(draft, toks, "CREATE")
1354
+ warnings: List[Tuple[str, str, int]] = []
1355
+ if ine:
1356
+ warnings.append(
1357
+ (
1358
+ "IF_NOT_EXISTS",
1359
+ f"CREATE TABLE IF NOT EXISTS {target.display()} converted to "
1360
+ 'type "table": Dataform always creates or replaces, so the '
1361
+ "'only if absent' guard is lost. Use "
1362
+ "--if-not-exists operations to preserve it verbatim.",
1363
+ stmt.start,
1364
+ )
1365
+ )
1366
+ elif not or_replace:
1367
+ warnings.append(
1368
+ (
1369
+ "CREATE_REPLACE_SEMANTICS",
1370
+ f"CREATE TABLE {target.display()} converted to a Dataform table; "
1371
+ "subsequent Dataform runs rebuild the target instead of failing "
1372
+ "when it already exists.",
1373
+ stmt.start,
1374
+ )
1375
+ )
1376
+ config: Dict[str, Any] = {}
1377
+ _apply_table_metadata(config, info)
1378
+ draft = ActionDraft(
1379
+ action_type=ActionType.TABLE,
1380
+ target=target,
1381
+ writes_target=False,
1382
+ creates_target=True,
1383
+ body_start=toks[bi].start,
1384
+ body_end=stmt.end,
1385
+ stmt_start=stmt.start,
1386
+ stmt_end=stmt.terminator_end,
1387
+ config=config,
1388
+ original_kind="CREATE TABLE",
1389
+ warnings=warnings,
1390
+ )
1391
+ return _finish(draft, toks, "CREATE")
1392
+
1393
+
1394
+ def _plain_create(
1395
+ stmt: RawStatement,
1396
+ toks: List[Token],
1397
+ opts: ConversionOptions,
1398
+ target: TableName,
1399
+ tspan: Tuple[int, int],
1400
+ or_replace: bool,
1401
+ ine: bool,
1402
+ info: Dict[str, Any],
1403
+ ) -> ActionDraft:
1404
+ """Handle ``CREATE TABLE`` without ``AS`` (schema-only DDL)."""
1405
+ if opts.plain_create_strategy is PlainCreateStrategy.DECLARATION:
1406
+ config: Dict[str, Any] = {}
1407
+ _apply_table_metadata(config, info)
1408
+ config.pop("bigquery", None)
1409
+ draft = ActionDraft(
1410
+ action_type=ActionType.DECLARATION,
1411
+ target=target,
1412
+ writes_target=False,
1413
+ creates_target=True,
1414
+ body_start=stmt.end,
1415
+ body_end=stmt.end,
1416
+ stmt_start=stmt.start,
1417
+ stmt_end=stmt.terminator_end,
1418
+ config=config,
1419
+ original_kind="CREATE TABLE",
1420
+ warnings=[
1421
+ (
1422
+ "DECLARATION_DROPPED_DDL",
1423
+ f"Plain CREATE TABLE {target.display()} emitted as a "
1424
+ "declaration; the DDL itself was dropped (schema "
1425
+ "assumed to be managed outside Dataform).",
1426
+ stmt.start,
1427
+ )
1428
+ ],
1429
+ )
1430
+ return _finish(draft, toks, "CREATE")
1431
+ warns: List[Tuple[str, str, int]] = [
1432
+ (
1433
+ "CREATE_NO_AS",
1434
+ f"Plain CREATE TABLE {target.display()} (no AS query) kept as "
1435
+ "operations; use --plain-create declaration to emit a source "
1436
+ "declaration instead.",
1437
+ stmt.start,
1438
+ )
1439
+ ]
1440
+ if not ine and not or_replace:
1441
+ warns.append(
1442
+ (
1443
+ "RERUN_RISK",
1444
+ "This CREATE TABLE has neither OR REPLACE nor IF NOT EXISTS; "
1445
+ "as a repeated Dataform operation it will fail on the second "
1446
+ "run.",
1447
+ stmt.start,
1448
+ )
1449
+ )
1450
+ draft = _ops_draft(
1451
+ stmt, "CREATE TABLE", target=target, writes=True, tspan=tspan, warnings=warns
1452
+ )
1453
+ return _finish(draft, toks, "CREATE")
1454
+
1455
+
1456
+ def _parse_view_columns(
1457
+ toks: List[Token], i: int, text: str
1458
+ ) -> Optional[Tuple[List[Tuple[str, Optional[str]]], int]]:
1459
+ """Parse a view column list ``(name [OPTIONS(description='..')], ...)``.
1460
+
1461
+ Args:
1462
+ toks: Token list; ``toks[i]`` must be ``(``.
1463
+ i: Index of the opening parenthesis.
1464
+ text: Full source text.
1465
+
1466
+ Returns:
1467
+ ``([(name, description_or_None), ...], next_index)`` or ``None``
1468
+ for any other shape (caller falls back to operations).
1469
+ """
1470
+ j = i + 1
1471
+ out: List[Tuple[str, Optional[str]]] = []
1472
+ n = len(toks)
1473
+ while j < n:
1474
+ t = toks[j]
1475
+ if t.kind == BACKTICK:
1476
+ name = unquote_identifier(t.text)
1477
+ elif t.kind == IDENT and t.upper not in RESERVED:
1478
+ name = t.text
1479
+ else:
1480
+ return None
1481
+ j += 1
1482
+ desc: Optional[str] = None
1483
+ if _kw(toks, j) == "OPTIONS":
1484
+ parsed = _parse_options(toks, j, text)
1485
+ if parsed is None:
1486
+ return None
1487
+ entries, j = parsed
1488
+ if (
1489
+ len(entries) != 1
1490
+ or entries[0][0].lower() != "description"
1491
+ or not isinstance(entries[0][2], str)
1492
+ ):
1493
+ return None
1494
+ desc = entries[0][2]
1495
+ out.append((name, desc))
1496
+ if _is_op(toks, j, ","):
1497
+ j += 1
1498
+ continue
1499
+ if _is_op(toks, j, ")"):
1500
+ return out, j + 1
1501
+ return None
1502
+ return None
1503
+
1504
+
1505
+ def _classify_view_body(
1506
+ stmt: RawStatement,
1507
+ toks: List[Token],
1508
+ text: str,
1509
+ opts: ConversionOptions,
1510
+ i: int,
1511
+ target: TableName,
1512
+ tspan: Tuple[int, int],
1513
+ materialized: bool,
1514
+ or_replace: bool,
1515
+ ine: bool,
1516
+ ) -> ActionDraft:
1517
+ """Classify ``CREATE [MATERIALIZED] VIEW`` after its target path."""
1518
+ kind = "CREATE MATERIALIZED VIEW" if materialized else "CREATE VIEW"
1519
+
1520
+ def fallback(code: str, msg: str, source_index: Optional[int] = None) -> ActionDraft:
1521
+ draft = _ops_draft(
1522
+ stmt, kind, target=target, writes=True, tspan=tspan, warnings=[(code, msg, stmt.start)]
1523
+ )
1524
+ _finish(draft, toks, "CREATE")
1525
+ if source_index is not None:
1526
+ _append_table_ref(draft, toks, source_index)
1527
+ return draft
1528
+
1529
+ columns: Optional[List[Tuple[str, Optional[str]]]] = None
1530
+ if _is_op(toks, i, "("):
1531
+ parsed_cols = _parse_view_columns(toks, i, text)
1532
+ if parsed_cols is None:
1533
+ return fallback(
1534
+ "FALLBACK_OPERATIONS",
1535
+ "View column list uses a form that cannot be "
1536
+ "safely converted; kept as operations.",
1537
+ )
1538
+ columns, i = parsed_cols
1539
+ info = _scan_pre_as_clauses(toks, i, text, allow_columns=False)
1540
+ if isinstance(info, tuple):
1541
+ return fallback("FALLBACK_OPERATIONS", f"{kind} kept as operations: {info[1]}.")
1542
+ if info["as_index"] is None:
1543
+ return fallback("FALLBACK_OPERATIONS", f"{kind} has no AS query; kept as operations.")
1544
+ bi = info["as_index"] + 1
1545
+ if _kw(toks, bi) == "REPLICA":
1546
+ return fallback(
1547
+ "FALLBACK_OPERATIONS",
1548
+ "MATERIALIZED VIEW ... AS REPLICA OF has no typed " "equivalent; kept as operations.",
1549
+ bi + 2 if _kw(toks, bi + 1) == "OF" else None,
1550
+ )
1551
+ if not _body_head_ok(toks, bi):
1552
+ return fallback(
1553
+ "FALLBACK_OPERATIONS", f"{kind} AS is not followed by a query; kept as " "operations."
1554
+ )
1555
+ if ine and opts.if_not_exists_strategy is IfNotExistsStrategy.OPERATIONS:
1556
+ return fallback(
1557
+ "IF_NOT_EXISTS", "IF NOT EXISTS preserved verbatim as operations per " "options."
1558
+ )
1559
+ edits: List[Tuple[int, int, str]] = []
1560
+ if columns:
1561
+ maybe = _alias_select_list_edits(toks, bi, len(toks) - 1, [c[0] for c in columns])
1562
+ if maybe is None:
1563
+ return fallback(
1564
+ "FALLBACK_SELECT_ALIAS",
1565
+ "The view column list could not be safely pushed "
1566
+ "into the select list; kept as operations.",
1567
+ )
1568
+ edits = maybe
1569
+ warnings: List[Tuple[str, str, int]] = []
1570
+ if ine:
1571
+ warnings.append(
1572
+ (
1573
+ "IF_NOT_EXISTS",
1574
+ f"{kind} IF NOT EXISTS {target.display()} converted to type "
1575
+ '"view": Dataform always creates or replaces, so the guard is '
1576
+ "lost.",
1577
+ stmt.start,
1578
+ )
1579
+ )
1580
+ elif not or_replace:
1581
+ warnings.append(
1582
+ (
1583
+ "CREATE_REPLACE_SEMANTICS",
1584
+ f"{kind} {target.display()} converted to a Dataform view; "
1585
+ "subsequent Dataform runs rebuild the target instead of failing "
1586
+ "when it already exists.",
1587
+ stmt.start,
1588
+ )
1589
+ )
1590
+ config: Dict[str, Any] = {}
1591
+ if materialized:
1592
+ config["materialized"] = True
1593
+ _apply_table_metadata(config, info)
1594
+ if columns:
1595
+ described = {name: desc for name, desc in columns if desc}
1596
+ if described:
1597
+ config["columns"] = described
1598
+ draft = ActionDraft(
1599
+ action_type=ActionType.VIEW,
1600
+ target=target,
1601
+ writes_target=False,
1602
+ creates_target=True,
1603
+ body_start=toks[bi].start,
1604
+ body_end=stmt.end,
1605
+ stmt_start=stmt.start,
1606
+ stmt_end=stmt.terminator_end,
1607
+ edits=edits,
1608
+ config=config,
1609
+ original_kind=kind,
1610
+ warnings=warnings,
1611
+ )
1612
+ return _finish(draft, toks, "CREATE")
1613
+
1614
+
1615
+ # ---------------------------------------------------------------------------
1616
+ # INSERT handling
1617
+ # ---------------------------------------------------------------------------
1618
+
1619
+
1620
+ def _classify_insert(
1621
+ stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
1622
+ ) -> ActionDraft:
1623
+ """Classify ``INSERT [INTO] target [(cols)] <query|VALUES ...>``."""
1624
+ i = 1
1625
+ if _kw(toks, i) == "INTO":
1626
+ i += 1
1627
+ pm = parse_table_path(toks, i)
1628
+ if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
1629
+ draft = _ops_draft(
1630
+ stmt,
1631
+ "INSERT",
1632
+ warnings=[
1633
+ (
1634
+ "FALLBACK_OPERATIONS",
1635
+ "Could not parse the INSERT target path; kept verbatim.",
1636
+ stmt.start,
1637
+ )
1638
+ ],
1639
+ )
1640
+ return _finish(draft, toks, "INSERT")
1641
+ target = TableName.from_parts(list(pm.parts))
1642
+ tspan = (pm.start, pm.end)
1643
+ i = pm.next_index
1644
+ cols: Optional[List[str]] = None
1645
+ if _is_op(toks, i, "("):
1646
+ cols, ni = _parse_ident_list(toks, i)
1647
+ if cols is None:
1648
+ draft = _ops_draft(
1649
+ stmt,
1650
+ "INSERT",
1651
+ target=target,
1652
+ writes=True,
1653
+ tspan=tspan,
1654
+ warnings=[
1655
+ (
1656
+ "FALLBACK_OPERATIONS",
1657
+ "Unrecognized INSERT column list; kept " "verbatim.",
1658
+ stmt.start,
1659
+ )
1660
+ ],
1661
+ )
1662
+ return _finish(draft, toks, "INSERT")
1663
+ i = ni
1664
+ if _kw(toks, i) == "VALUES":
1665
+ draft = _ops_draft(
1666
+ stmt,
1667
+ "INSERT",
1668
+ target=target,
1669
+ writes=True,
1670
+ tspan=tspan,
1671
+ warnings=[
1672
+ (
1673
+ "INSERT_VALUES",
1674
+ "INSERT ... VALUES has no query body; kept as " "operations.",
1675
+ stmt.start,
1676
+ )
1677
+ ],
1678
+ )
1679
+ return _finish(draft, toks, "INSERT")
1680
+ if opts.insert_strategy is InsertStrategy.OPERATIONS:
1681
+ draft = _ops_draft(stmt, "INSERT", target=target, writes=True, tspan=tspan)
1682
+ return _finish(draft, toks, "INSERT")
1683
+ if not _body_head_ok(toks, i):
1684
+ draft = _ops_draft(
1685
+ stmt,
1686
+ "INSERT",
1687
+ target=target,
1688
+ writes=True,
1689
+ tspan=tspan,
1690
+ warnings=[
1691
+ (
1692
+ "FALLBACK_OPERATIONS",
1693
+ "INSERT body is not a plain query; kept " "verbatim.",
1694
+ stmt.start,
1695
+ )
1696
+ ],
1697
+ )
1698
+ return _finish(draft, toks, "INSERT")
1699
+ output_names = cols or _select_output_names(toks, i, len(toks) - 1)
1700
+ if (
1701
+ output_names is None
1702
+ or len({name.upper() for name in output_names}) != len(output_names)
1703
+ or any(not _dataform_insert_column(name) for name in output_names)
1704
+ ):
1705
+ draft = _ops_draft(
1706
+ stmt,
1707
+ "INSERT",
1708
+ target=target,
1709
+ writes=True,
1710
+ tspan=tspan,
1711
+ warnings=[
1712
+ (
1713
+ "FALLBACK_SELECT_ALIAS",
1714
+ "The incremental query's output column names "
1715
+ "are not all exactly derivable, unique, and "
1716
+ "safe for Dataform Core's runtime backtick "
1717
+ "quoting; the statement was kept verbatim.",
1718
+ stmt.start,
1719
+ )
1720
+ ],
1721
+ )
1722
+ return _finish(draft, toks, "INSERT")
1723
+ edits: List[Tuple[int, int, str]] = []
1724
+ if cols:
1725
+ maybe = _alias_select_list_edits(toks, i, len(toks) - 1, cols)
1726
+ if maybe is None:
1727
+ draft = _ops_draft(
1728
+ stmt,
1729
+ "INSERT",
1730
+ target=target,
1731
+ writes=True,
1732
+ tspan=tspan,
1733
+ warnings=[
1734
+ (
1735
+ "FALLBACK_SELECT_ALIAS",
1736
+ "The INSERT column list could not be "
1737
+ "safely pushed into the select list (star "
1738
+ "expansion, INTERVAL/typed constructors, "
1739
+ "or arity mismatch); kept verbatim.",
1740
+ stmt.start,
1741
+ )
1742
+ ],
1743
+ )
1744
+ return _finish(draft, toks, "INSERT")
1745
+ edits = maybe
1746
+ config: Dict[str, Any] = {}
1747
+ if opts.protect_incrementals:
1748
+ config["protected"] = True
1749
+ draft = ActionDraft(
1750
+ action_type=ActionType.INCREMENTAL,
1751
+ target=target,
1752
+ writes_target=False,
1753
+ creates_target=True,
1754
+ body_start=toks[i].start,
1755
+ body_end=stmt.end,
1756
+ stmt_start=stmt.start,
1757
+ stmt_end=stmt.terminator_end,
1758
+ edits=edits,
1759
+ config=config,
1760
+ original_kind="INSERT",
1761
+ warnings=[
1762
+ (
1763
+ "INSERT_INCREMENTAL",
1764
+ f"INSERT INTO {target.display()} converted to type "
1765
+ '"incremental": on first run or --full-refresh Dataform will '
1766
+ "(re)create the table from this query. Wrap date filters in "
1767
+ "${when(incremental(), ...)} if they should apply only to "
1768
+ "incremental runs. On later runs Dataform projects every "
1769
+ "existing target column by name, so the query output must match "
1770
+ "that target schema.",
1771
+ stmt.start,
1772
+ )
1773
+ ],
1774
+ )
1775
+ return _finish(draft, toks, "INSERT")
1776
+
1777
+
1778
+ # ---------------------------------------------------------------------------
1779
+ # MERGE handling
1780
+ # ---------------------------------------------------------------------------
1781
+
1782
+
1783
+ def _read_qualified_col(toks: Sequence[Token], j: int) -> Optional[Tuple[str, str, int]]:
1784
+ """Read a strictly qualified column ``qual.col`` at index ``j``.
1785
+
1786
+ Returns:
1787
+ ``(qualifier, column, next_index)`` or ``None``.
1788
+ """
1789
+ if j + 2 >= len(toks):
1790
+ return None
1791
+ a, dot, b = toks[j], toks[j + 1], toks[j + 2]
1792
+ if a.kind not in (IDENT, BACKTICK) or (a.kind == IDENT and a.upper in RESERVED):
1793
+ return None
1794
+ if not (dot.kind == OP and dot.text == "."):
1795
+ return None
1796
+ if b.kind not in (IDENT, BACKTICK) or (b.kind == IDENT and b.upper in RESERVED):
1797
+ return None
1798
+ return (unquote_identifier(a.text), unquote_identifier(b.text), j + 3)
1799
+
1800
+
1801
+ def _read_set_target_col(toks: Sequence[Token], j: int, tq: str) -> Optional[Tuple[str, int]]:
1802
+ """Read an UPDATE SET target column, optionally target-qualified.
1803
+
1804
+ Args:
1805
+ toks: Token list.
1806
+ j: Start index.
1807
+ tq: Uppercased target qualifier (alias or table name).
1808
+
1809
+ Returns:
1810
+ ``(column, next_index)`` or ``None``.
1811
+ """
1812
+ q = _read_qualified_col(toks, j)
1813
+ if q is not None:
1814
+ if q[0].upper() != tq:
1815
+ return None
1816
+ return q[1], q[2]
1817
+ if j < len(toks):
1818
+ t = toks[j]
1819
+ if t.kind == BACKTICK or (t.kind == IDENT and t.upper not in RESERVED):
1820
+ return unquote_identifier(t.text), j + 1
1821
+ return None
1822
+
1823
+
1824
+ def _merge_safe_extract(
1825
+ toks: List[Token], i: int, target: TableName, talias: Optional[str]
1826
+ ) -> Union[str, Tuple[int, int, List[str]]]:
1827
+ """Prove a MERGE equivalent to a Dataform ``uniqueKey`` incremental.
1828
+
1829
+ The proof requires *all* of: a subquery source with an alias; an ``ON``
1830
+ clause that is a pure conjunction of same-named target/source column
1831
+ equalities; exactly ``WHEN MATCHED THEN UPDATE SET`` with only direct
1832
+ same-named source assignments; exactly ``WHEN NOT MATCHED [BY TARGET]
1833
+ THEN INSERT`` with explicit matching source columns; and a
1834
+ source select list whose derivable output names are exactly the key
1835
+ columns plus the updated columns. Under those conditions the MERGE
1836
+ Dataform generates for ``type: "incremental"`` + ``uniqueKey`` has
1837
+ identical row-level effects.
1838
+
1839
+ Args:
1840
+ toks: Statement tokens (EOF-terminated).
1841
+ i: Index of the expected ``USING`` keyword.
1842
+ target: The MERGE target table.
1843
+ talias: The target alias, if any.
1844
+
1845
+ Returns:
1846
+ ``(body_start_offset, body_end_offset, unique_keys)`` on success,
1847
+ or a human-readable failure reason string.
1848
+ """
1849
+ if _kw(toks, i) != "USING":
1850
+ return "unrecognized MERGE shape"
1851
+ i += 1
1852
+ if not _is_op(toks, i, "("):
1853
+ return "USING source is a table, not a subquery"
1854
+ open_i = i
1855
+ after = _skip_balanced(toks, i)
1856
+ close_i = after - 1
1857
+ if not _is_op(toks, close_i, ")"):
1858
+ return "unterminated USING subquery"
1859
+ inner_ts, inner_te = open_i + 1, close_i
1860
+ if inner_ts >= inner_te:
1861
+ return "empty USING subquery"
1862
+ j = after
1863
+ if _kw(toks, j) == "AS":
1864
+ j += 1
1865
+ salias: Optional[str] = None
1866
+ if j < len(toks):
1867
+ t = toks[j]
1868
+ if t.kind == BACKTICK or (t.kind == IDENT and t.upper not in RESERVED):
1869
+ salias = unquote_identifier(t.text)
1870
+ j += 1
1871
+ if salias is None:
1872
+ return "the USING subquery has no alias"
1873
+ if _kw(toks, j) != "ON":
1874
+ return "unrecognized MERGE shape after USING"
1875
+ j += 1
1876
+ tq = (talias or target.table).upper()
1877
+ sq = salias.upper()
1878
+ if tq == sq:
1879
+ return "target and source aliases are not distinct"
1880
+ keys: List[str] = []
1881
+ while True:
1882
+ left = _read_qualified_col(toks, j)
1883
+ if left is None:
1884
+ return "ON is not a conjunction of simple qualified equalities"
1885
+ lq, lc, j = left
1886
+ if not _is_op(toks, j, "="):
1887
+ return "ON is not a conjunction of simple qualified equalities"
1888
+ j += 1
1889
+ right = _read_qualified_col(toks, j)
1890
+ if right is None:
1891
+ return "ON is not a conjunction of simple qualified equalities"
1892
+ rq, rc, j = right
1893
+ if lc.upper() != rc.upper():
1894
+ return "ON joins differently named columns"
1895
+ if {lq.upper(), rq.upper()} != {tq, sq}:
1896
+ return "ON does not compare target columns to source columns"
1897
+ keys.append(lc if lq.upper() == tq else rc)
1898
+ nxt = _kw(toks, j)
1899
+ if nxt == "AND":
1900
+ j += 1
1901
+ continue
1902
+ if nxt == "WHEN":
1903
+ break
1904
+ return "unsupported ON clause"
1905
+ if not (_kw(toks, j) == "WHEN" and _kw(toks, j + 1) == "MATCHED"):
1906
+ return "first WHEN clause is not WHEN MATCHED"
1907
+ j += 2
1908
+ if _kw(toks, j) == "AND":
1909
+ return "conditional WHEN MATCHED clause"
1910
+ if not (_kw(toks, j) == "THEN" and _kw(toks, j + 1) == "UPDATE" and _kw(toks, j + 2) == "SET"):
1911
+ return "WHEN MATCHED does not perform a plain UPDATE SET"
1912
+ j += 3
1913
+ set_cols: List[str] = []
1914
+ while True:
1915
+ left_set = _read_set_target_col(toks, j, tq)
1916
+ if left_set is None:
1917
+ return "UPDATE SET has a non-trivial assignment target"
1918
+ lc, j = left_set
1919
+ if not _is_op(toks, j, "="):
1920
+ return "UPDATE SET has a non-trivial assignment"
1921
+ j += 1
1922
+ right = _read_qualified_col(toks, j)
1923
+ if right is None or right[0].upper() != sq:
1924
+ return "UPDATE SET assigns something other than a source column"
1925
+ if right[1].upper() != lc.upper():
1926
+ return "UPDATE SET renames a column"
1927
+ j = right[2]
1928
+ set_cols.append(lc)
1929
+ if _is_op(toks, j, ","):
1930
+ j += 1
1931
+ continue
1932
+ if _kw(toks, j) == "WHEN":
1933
+ break
1934
+ return "unsupported UPDATE SET clause"
1935
+ if not (_kw(toks, j) == "WHEN" and _kw(toks, j + 1) == "NOT" and _kw(toks, j + 2) == "MATCHED"):
1936
+ return "second WHEN clause is not WHEN NOT MATCHED"
1937
+ j += 3
1938
+ if _kw(toks, j) == "BY":
1939
+ if _kw(toks, j + 1) != "TARGET":
1940
+ return "WHEN NOT MATCHED BY SOURCE is not expressible"
1941
+ j += 2
1942
+ if _kw(toks, j) == "AND":
1943
+ return "conditional WHEN NOT MATCHED clause"
1944
+ if not (_kw(toks, j) == "THEN" and _kw(toks, j + 1) == "INSERT"):
1945
+ return "WHEN NOT MATCHED does not perform a plain INSERT"
1946
+ j += 2
1947
+ outputs = _select_output_names(toks, inner_ts, inner_te)
1948
+ if outputs is None:
1949
+ return "cannot derive the source subquery's output column names"
1950
+ if len({output.upper() for output in outputs}) != len(outputs):
1951
+ return "the source subquery has duplicate output column names"
1952
+ if len({key.upper() for key in keys}) != len(keys):
1953
+ return "the MERGE ON clause repeats a unique-key column"
1954
+ if len({column.upper() for column in set_cols}) != len(set_cols):
1955
+ return "UPDATE SET assigns the same target column more than once"
1956
+ out_set = {o.upper() for o in outputs}
1957
+ key_set = {k.upper() for k in keys}
1958
+ set_col_set = {c.upper() for c in set_cols}
1959
+ if not key_set.issubset(set_col_set):
1960
+ return (
1961
+ "Dataform updates unique-key columns during its generated "
1962
+ "MERGE, but the original UPDATE SET does not"
1963
+ )
1964
+ if any(not _dataform_merge_column(output) for output in outputs):
1965
+ return (
1966
+ "Dataform cannot safely emit a quoted or reserved source "
1967
+ "column name in its generated MERGE"
1968
+ )
1969
+ need = key_set | set_col_set
1970
+ if out_set != need:
1971
+ return "source columns do not exactly cover the key + updated columns"
1972
+ if _kw(toks, j) == "ROW":
1973
+ return (
1974
+ "INSERT ROW depends on the target table's physical column "
1975
+ "order, which cannot be proven from SQL text alone"
1976
+ )
1977
+ else:
1978
+ if not _is_op(toks, j, "("):
1979
+ return "WHEN NOT MATCHED INSERT is not (columns) VALUES or ROW"
1980
+ cols, j = _parse_ident_list(toks, j)
1981
+ if cols is None:
1982
+ return "WHEN NOT MATCHED INSERT is not (columns) VALUES or ROW"
1983
+ if len({column.upper() for column in cols}) != len(cols):
1984
+ return "INSERT column list contains duplicates"
1985
+ if _kw(toks, j) != "VALUES" or not _is_op(toks, j + 1, "("):
1986
+ return "WHEN NOT MATCHED INSERT is not (columns) VALUES (...)"
1987
+ j += 2
1988
+ for idx, col in enumerate(cols):
1989
+ right = _read_qualified_col(toks, j)
1990
+ if right is None or right[0].upper() != sq or right[1].upper() != col.upper():
1991
+ return "INSERT VALUES are not the matching source columns"
1992
+ j = right[2]
1993
+ if idx < len(cols) - 1:
1994
+ if not _is_op(toks, j, ","):
1995
+ return "INSERT VALUES are not the matching source columns"
1996
+ j += 1
1997
+ if not _is_op(toks, j, ")"):
1998
+ return "INSERT VALUES are not the matching source columns"
1999
+ j += 1
2000
+ if {c.upper() for c in cols} != out_set:
2001
+ return "INSERT column list does not match the source columns"
2002
+ if [c.upper() for c in cols] != [o.upper() for o in outputs]:
2003
+ return "source output order does not match the explicit INSERT " "column order"
2004
+ if toks[j].kind != EOF:
2005
+ return "unsupported trailing MERGE clauses"
2006
+ return (toks[inner_ts].start, toks[inner_te - 1].end, keys)
2007
+
2008
+
2009
+ def _classify_merge(
2010
+ stmt: RawStatement, toks: List[Token], text: str, opts: ConversionOptions
2011
+ ) -> ActionDraft:
2012
+ """Classify a ``MERGE`` statement per the configured strategy."""
2013
+ i = 1
2014
+ if _kw(toks, i) == "INTO":
2015
+ i += 1
2016
+ pm = parse_table_path(toks, i)
2017
+ if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
2018
+ draft = _ops_draft(
2019
+ stmt,
2020
+ "MERGE",
2021
+ warnings=[
2022
+ (
2023
+ "FALLBACK_OPERATIONS",
2024
+ "Could not parse the MERGE target path; kept verbatim.",
2025
+ stmt.start,
2026
+ )
2027
+ ],
2028
+ )
2029
+ return _finish(draft, toks, "MERGE")
2030
+ target = TableName.from_parts(list(pm.parts))
2031
+ tspan = (pm.start, pm.end)
2032
+ i = pm.next_index
2033
+ talias: Optional[str] = None
2034
+ if _kw(toks, i) == "AS":
2035
+ if toks[i + 1].kind in (IDENT, BACKTICK):
2036
+ talias = unquote_identifier(toks[i + 1].text)
2037
+ i += 2
2038
+ elif toks[i].kind == BACKTICK or (toks[i].kind == IDENT and toks[i].upper not in RESERVED):
2039
+ talias = unquote_identifier(toks[i].text)
2040
+ i += 1
2041
+ if opts.merge_strategy is MergeStrategy.INCREMENTAL_WHEN_SAFE:
2042
+ result = _merge_safe_extract(toks, i, target, talias)
2043
+ if not isinstance(result, str):
2044
+ body_start, body_end, keys = result
2045
+ config: Dict[str, Any] = {"uniqueKey": keys}
2046
+ if opts.protect_incrementals:
2047
+ config["protected"] = True
2048
+ draft = ActionDraft(
2049
+ action_type=ActionType.INCREMENTAL,
2050
+ target=target,
2051
+ writes_target=False,
2052
+ creates_target=True,
2053
+ body_start=body_start,
2054
+ body_end=body_end,
2055
+ stmt_start=stmt.start,
2056
+ stmt_end=stmt.terminator_end,
2057
+ config=config,
2058
+ original_kind="MERGE",
2059
+ warnings=[
2060
+ (
2061
+ "MERGE_INCREMENTAL",
2062
+ f"MERGE into {target.display()} proven equivalent to an "
2063
+ f"incremental table with uniqueKey {keys}, provided the "
2064
+ "existing target schema exactly matches the source "
2065
+ "output; converted.",
2066
+ stmt.start,
2067
+ ),
2068
+ (
2069
+ "TARGET_SCHEMA_REQUIRED",
2070
+ "Dataform builds incremental MERGE assignments from "
2071
+ "existing target metadata. Verify that its columns "
2072
+ "exactly match this query before the first incremental "
2073
+ "run.",
2074
+ stmt.start,
2075
+ ),
2076
+ ],
2077
+ )
2078
+ return _finish(draft, toks, "MERGE")
2079
+ draft = _ops_draft(
2080
+ stmt,
2081
+ "MERGE",
2082
+ target=target,
2083
+ writes=True,
2084
+ tspan=tspan,
2085
+ warnings=[("MERGE_FALLBACK", f"MERGE kept as operations ({result}).", stmt.start)],
2086
+ )
2087
+ return _finish(draft, toks, "MERGE")
2088
+ draft = _ops_draft(stmt, "MERGE", target=target, writes=True, tspan=tspan)
2089
+ return _finish(draft, toks, "MERGE")
2090
+
2091
+
2092
+ # ---------------------------------------------------------------------------
2093
+ # Other DML / DDL with identifiable write targets
2094
+ # ---------------------------------------------------------------------------
2095
+
2096
+
2097
+ def _dml_common(
2098
+ stmt: RawStatement, toks: List[Token], pm: Optional[Any], kind: str, skip_target: bool
2099
+ ) -> ActionDraft:
2100
+ """Finalize a DML/DDL operations draft with an optional write target.
2101
+
2102
+ Args:
2103
+ stmt: The raw statement.
2104
+ toks: Statement tokens.
2105
+ pm: Parsed target path (or ``None``).
2106
+ kind: Uppercase statement kind.
2107
+ skip_target: Exclude the target span from reference scanning
2108
+ (needed for ``DELETE FROM target``, where the scanner would
2109
+ otherwise see the target as a FROM-clause table read).
2110
+
2111
+ Returns:
2112
+ The draft.
2113
+ """
2114
+ if pm is None or not (1 <= len(pm.parts) <= 3) or not all(pm.parts):
2115
+ return _finish(_ops_draft(stmt, kind), toks, kind)
2116
+ target = TableName.from_parts(list(pm.parts))
2117
+ tspan = (pm.start, pm.end)
2118
+ draft = _ops_draft(stmt, kind, target=target, writes=True, tspan=tspan)
2119
+ return _finish(draft, toks, kind, skip_spans=[tspan] if skip_target else ())
2120
+
2121
+
2122
+ def _classify_drop_alter(stmt: RawStatement, toks: List[Token], head: str) -> ActionDraft:
2123
+ """Classify DROP/ALTER, extracting the table/view target when present."""
2124
+ i = 1
2125
+ ent = _kw(toks, i)
2126
+ if ent in ("MATERIALIZED", "EXTERNAL", "SNAPSHOT"):
2127
+ nxt = _kw(toks, i + 1)
2128
+ if (ent == "MATERIALIZED" and nxt == "VIEW") or (
2129
+ ent in ("EXTERNAL", "SNAPSHOT") and nxt == "TABLE"
2130
+ ):
2131
+ i += 2
2132
+ else:
2133
+ return _finish(_ops_draft(stmt, head), toks, head)
2134
+ elif ent in ("TABLE", "VIEW"):
2135
+ i += 1
2136
+ else:
2137
+ return _finish(_ops_draft(stmt, head), toks, head)
2138
+ if _kw(toks, i) == "IF" and _kw(toks, i + 1) == "EXISTS":
2139
+ i += 2
2140
+ pm = parse_table_path(toks, i)
2141
+ draft = _dml_common(stmt, toks, pm, head, skip_target=False)
2142
+ if (
2143
+ head == "ALTER"
2144
+ and pm is not None
2145
+ and 1 <= len(pm.parts) <= 3
2146
+ and all(pm.parts)
2147
+ and _kw(toks, pm.next_index) == "RENAME"
2148
+ and _kw(toks, pm.next_index + 1) == "TO"
2149
+ ):
2150
+ renamed_match = parse_table_path(toks, pm.next_index + 2)
2151
+ if (
2152
+ renamed_match is not None
2153
+ and 1 <= len(renamed_match.parts) <= 3
2154
+ and all(renamed_match.parts)
2155
+ ):
2156
+ original = TableName.from_parts(list(pm.parts))
2157
+ renamed = TableName.from_parts(list(renamed_match.parts))
2158
+ if renamed.project is None and renamed.dataset is None:
2159
+ renamed = TableName(
2160
+ original.project,
2161
+ original.dataset,
2162
+ renamed.table,
2163
+ )
2164
+ elif renamed.project is None and original.project is not None:
2165
+ renamed = TableName(
2166
+ original.project,
2167
+ renamed.dataset,
2168
+ renamed.table,
2169
+ )
2170
+ draft.extra_write_targets.append(renamed)
2171
+ return draft
2172
+
2173
+
2174
+ def _classify_dml(stmt: RawStatement, toks: List[Token], head: str) -> ActionDraft:
2175
+ """Classify UPDATE/DELETE/TRUNCATE/DROP/ALTER/LOAD statements."""
2176
+ if head == "UPDATE":
2177
+ return _dml_common(stmt, toks, parse_table_path(toks, 1), "UPDATE", skip_target=False)
2178
+ if head == "DELETE":
2179
+ i = 2 if _kw(toks, 1) == "FROM" else 1
2180
+ return _dml_common(stmt, toks, parse_table_path(toks, i), "DELETE", skip_target=True)
2181
+ if head == "TRUNCATE":
2182
+ i = 2 if _kw(toks, 1) == "TABLE" else 1
2183
+ return _dml_common(stmt, toks, parse_table_path(toks, i), "TRUNCATE", skip_target=False)
2184
+ if head in ("DROP", "ALTER"):
2185
+ return _classify_drop_alter(stmt, toks, head)
2186
+ if head == "LOAD":
2187
+ i = 1
2188
+ if _kw(toks, i) == "DATA":
2189
+ i += 1
2190
+ if _kw(toks, i) in ("INTO", "OVERWRITE"):
2191
+ i += 1
2192
+ return _dml_common(stmt, toks, parse_table_path(toks, i), "LOAD", skip_target=False)
2193
+ return _finish(_ops_draft(stmt, head), toks, head) # pragma: no cover