sql2sqlx 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

sql2sqlx/converter.py

@@ -46,7 +46,13 @@ from concurrent.futures import ProcessPoolExecutor
 from pathlib import Path, PurePosixPath
 from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
 
-from sql2sqlx.emitter import apply_edits_escaped, build_sqlx, ref_expr, sqlx_escape_edits
+from sql2sqlx.emitter import (
+    apply_edits_escaped,
+    build_sqlx,
+    normalize_sqlx_comment,
+    ref_expr,
+    sqlx_escape_edits,
+)
 from sql2sqlx.errors import ConversionError, LexError
 from sql2sqlx.lexer import IDENT, OP, LineIndex, Token, tokenize
 from sql2sqlx.model import (
@@ -1058,12 +1064,12 @@ class _Linker:
         leading = None
         spans = [(a, b) for a, b in self._comments(f) if prev_end <= a and b <= d.stmt_start]
         if spans:
-            leading = "\n".join(f.text[a:b] for a, b in spans)
+            leading = "\n".join(normalize_sqlx_comment(f.text[a:b]) for a, b in spans)
         trailing = None
         if is_last_in_file:
             tail_spans = [(a, b) for a, b in self._comments(f) if a >= d.stmt_end]
             if tail_spans:
-                trailing = "\n".join(f.text[a:b] for a, b in tail_spans)
+                trailing = "\n".join(normalize_sqlx_comment(f.text[a:b]) for a, b in tail_spans)
         body = None
         if d.action_type is not ActionType.DECLARATION:
             # Semantic rewrites own their complete source spans. In
@@ -1077,11 +1083,17 @@ class _Linker:
                     semantic[0] < escape[1] and escape[0] < semantic[1] for semantic in d.edits
                 )
             ]
+            comment_edits = []
+            for a, b in self._comments(f):
+                if d.body_start <= a and b <= d.body_end:
+                    normalized = normalize_sqlx_comment(f.text[a:b])
+                    if normalized != f.text[a:b]:
+                        comment_edits.append((a, b, normalized))
             body = apply_edits_escaped(
                 f.text,
                 d.body_start,
                 d.body_end,
-                d.edits + escape_edits,
+                d.edits + escape_edits + comment_edits,
             )
         content = build_sqlx(config, body, annotation, leading, trailing)
 

sql2sqlx/emitter.py

@@ -12,12 +12,15 @@ select-list aliases, ``${self()}`` substitutions) to slices of the
 **original** source text - tokens are never re-serialized, so the user's
 formatting, casing and inline comments survive after input decoding.
 
-Two SQLX-specific safety rules live here:
+SQLX-specific safety rules live here:
 
 * Any literal ``${`` in a SQL string or quoted identifier is emitted through
   a constant JavaScript placeholder. Dataform does not provide a backslash
   escape for SQL placeholders, so ``\\${`` is insufficient. Replacement text
   inserted *by* sql2sqlx (``${ref(...)}``, ``${self()}``) stays active.
+* GoogleSQL comments with SQLX-specific lexical hazards (``#`` comments and
+  exact ``---`` separator-looking comments) are normalized without changing
+  BigQuery semantics.
 * Operations bodies have exactly one trailing semicolon stripped:
   Dataform executes the body as a BigQuery script, and a trailing empty
   statement is at best noise.
@@ -29,7 +32,7 @@ import json
 import re
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
 
-from sql2sqlx.lexer import BACKTICK, PARAM, STRING, Token
+from sql2sqlx.lexer import BACKTICK, COMMENT, PARAM, STRING, Token
 from sql2sqlx.model import TableName
 
 #: Canonical top-level config key order (unknown keys follow, insertion-ordered).
@@ -62,6 +65,12 @@ _BQ_ORDER = (
 )
 
 _JS_IDENT_RE = re.compile(r"^[A-Za-z_$][A-Za-z0-9_$]*$")
+_SEPARATOR_COMMENT_RE = re.compile(r"^---[^\S\r\n]*$")
+#: Control characters that JSON encoding turns into ``\n``/``\t``/``\uXXXX``
+#: escapes. Dataform's placeholder-string lexer accepts only ``\"`` and ``\\``
+#: escapes (``/"(?:\\["\\]|[^\n"\\])*"/``), so a whole-string placeholder that
+#: embeds one of these characters cannot be lexed by Dataform.
+_JS_STRING_UNSAFE_RE = re.compile(r"[\x00-\x1f]")
 
 
 def _json(value: str) -> str:
@@ -74,32 +83,61 @@ def sqlx_constant(value: str) -> str:
     return "${" + _json(value) + "}"
 
 
-def sqlx_escape_edits(tokens: Sequence[Token]) -> List[Tuple[int, int, str]]:
-    """Build edits that preserve literal ``${`` through Dataform compile.
+def normalize_sqlx_comment(comment: str) -> str:
+    """Return a SQLX-safe spelling of a single GoogleSQL comment token.
+
+    Dataform SQLX treats ``---`` at the start of a line as a statement
+    separator and does not recognize GoogleSQL ``#`` line comments.  The
+    rewrites here preserve BigQuery semantics while preventing comment text
+    from being parsed as SQLX syntax.
+    """
+    if comment.startswith("#"):
+        # Rewriting ``#`` to ``--`` can itself produce a ``---`` separator
+        # line (e.g. the comment ``#-``), so fall through to the separator
+        # guard below instead of returning early.
+        comment = "--" + comment[1:]
+    if _SEPARATOR_COMMENT_RE.match(comment):
+        return "-- " + comment
+    return comment
 
-    GoogleSQL only permits ``$`` in the relevant valid inputs inside string
-    literals or quoted identifiers. String occurrences can be replaced by a
-    constant placeholder for the two characters. A quoted identifier must be
-    replaced as a whole because a SQLX placeholder nested between BigQuery
-    backticks produces invalid generated JavaScript.
+
+def sqlx_escape_edits(tokens: Sequence[Token]) -> List[Tuple[int, int, str]]:
+    """Build edits that keep literal SQL safe when parsed as SQLX.
+
+    Literal ``${`` in SQL strings, quoted identifiers, and quoted parameters
+    is emitted through a constant JavaScript placeholder because Dataform has
+    no SQL-level escape for placeholders.  String literals are replaced as
+    whole tokens so safety does not depend on Dataform's per-quote lexer
+    states.  A string that carries a control character (for example a
+    multi-line triple-quoted literal) cannot be embedded whole - JSON encoding
+    would introduce a ``\\n``/``\\t`` escape that Dataform's placeholder-string
+    lexer rejects - so each ``${`` in such a string is escaped in place, where
+    the surrounding SQL string state carries the raw characters unchanged.
+    GoogleSQL comments that are unsafe in SQLX are normalized as token edits.
     """
     edits: List[Tuple[int, int, str]] = []
     for token in tokens:
-        if "${" not in token.text:
+        if token.kind == COMMENT:
+            normalized = normalize_sqlx_comment(token.text)
+            if normalized != token.text:
+                edits.append((token.start, token.end, normalized))
             continue
-        if token.kind == BACKTICK or (token.kind == PARAM and token.text.startswith("@`")):
-            edits.append((token.start, token.end, sqlx_constant(token.text)))
+        if "${" not in token.text:
             continue
-        if token.kind != STRING:
+        if token.kind == STRING and _JS_STRING_UNSAFE_RE.search(token.text):
+            offset = 0
+            while True:
+                offset = token.text.find("${", offset)
+                if offset < 0:
+                    break
+                start = token.start + offset
+                edits.append((start, start + 2, sqlx_constant("${")))
+                offset += 2
             continue
-        offset = 0
-        while True:
-            offset = token.text.find("${", offset)
-            if offset < 0:
-                break
-            start = token.start + offset
-            edits.append((start, start + 2, sqlx_constant("${")))
-            offset += 2
+        if token.kind in {BACKTICK, STRING} or (
+            token.kind == PARAM and token.text.startswith("@`")
+        ):
+            edits.append((token.start, token.end, sqlx_constant(token.text)))
     return edits
 
 

sql2sqlx/version.py

@@ -12,4 +12,4 @@ without circular-import risk, and so packaging tools can read it
 statically.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"

{sql2sqlx-0.1.0.dist-info → sql2sqlx-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql2sqlx
-Version: 0.1.0
+Version: 0.1.1
 Summary: Convert BigQuery SQL into Dataform SQLX with conservative, dependency-aware migration tooling.
 Author-email: Soumyadip Sarkar <soumyadip@soumyadipsarkar.com>
 License-Expression: Apache-2.0
@@ -8,7 +8,6 @@ Project-URL: Homepage, https://github.com/neuralsorcerer/sql2sqlx
 Project-URL: Documentation, https://neuralsorcerer.github.io/sql2sqlx/
 Project-URL: Source, https://github.com/neuralsorcerer/sql2sqlx
 Project-URL: Issues, https://github.com/neuralsorcerer/sql2sqlx/issues
-Project-URL: Changelog, https://github.com/neuralsorcerer/sql2sqlx/blob/main/CHANGELOG.md
 Keywords: analytics-engineering,bigquery,bigquery-sql,code-generation,data-engineering,dataform,dataform-sqlx,ddl,dependency-graph,dml,elt,etl,gcp,google-cloud,googlesql,migration,sql,sql-converter,sql-migration,sql-parser,sqlx,warehouse-migration
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -56,13 +55,13 @@ BigQuery SQL -> Dataform SQLX migration tooling.
 
 <div align="center">
 
-[![Current Release](https://img.shields.io/github/release/neuralsorcerer/sql2sqlx.svg)](https://github.com/neuralsorcerer/sql2sqlx/releases)
 [![PyPI](https://img.shields.io/pypi/v/sql2sqlx.svg?logo=pypi&logoColor=white)](https://pypi.org/project/sql2sqlx/)
 [![Python 3.10+](https://img.shields.io/badge/Python-3.10+-fcbc2c.svg?logo=python&logoColor=white)](https://www.python.org/downloads/)
 [![Test Ubuntu](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_ubuntu.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_ubuntu.yml?query=branch%3Amain)
 [![Test Windows](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_windows.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_windows.yml?query=branch%3Amain)
 [![Test MacOS](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_macos.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/test_macos.yml?query=branch%3Amain)
 [![Lints](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/lints.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/lints.yml?query=branch%3Amain)
+[![CodeQL](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/codeql.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/codeql.yml?query=branch%3Amain)
 [![Dataform Compile](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/dataform_compile.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/dataform_compile.yml?query=branch%3Amain)
 [![Build](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/build.yml/badge.svg)](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/build.yml?query=branch%3Amain)
 [![Documentation](https://github.com/neuralsorcerer/sql2sqlx/actions/workflows/docs.yml/badge.svg)](https://neuralsorcerer.github.io/sql2sqlx/)

{sql2sqlx-0.1.0.dist-info → sql2sqlx-0.1.1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sql2sqlx/__init__.py,sha256=kT1TsG_BQaD9I0nMhQJOIIHbmQ_AHeyLFENt_g5wAFY,2101
 sql2sqlx/__main__.py,sha256=_w-oHujFK7NsgnIQmZ9RW22bvH9YZrbajk6YSS3csYE,363
 sql2sqlx/cli.py,sha256=jBZ6QeLs-to8vjhY_vbiTreE-svBrIcYsGGSXZjhuPQ,11850
-sql2sqlx/converter.py,sha256=M4m5phsGjkwS2rzrss8d2fYjogpLIPCrB4la0KVrIi0,50474
-sql2sqlx/emitter.py,sha256=vMJiIKyz4EJrkgVmylgO-I67BfHxAa2TLSroxZOvjnU,9517
+sql2sqlx/converter.py,sha256=7QLLnnCD7hOQKtmx3a4iKxdagDepPcCuucrpZENpA1I,50903
+sql2sqlx/emitter.py,sha256=wjykaralrmwSvuc1c-9jg0MrqFJXNUY7mfT3c6nFKQg,11563
 sql2sqlx/errors.py,sha256=XpNecYBPylxO_PevnGeJLjrO6n4zYnf9U3QeK9yTvXs,3231
 sql2sqlx/keywords.py,sha256=4ofwRHXp2gDn_z1dNoti2_kaOiHEcdNOgdzadW7imSQ,3559
 sql2sqlx/lexer.py,sha256=bVMDwcSgGN_k9eCQa-rziyQPqdx17OMkiPeEx9Ub26E,18837
@@ -11,10 +11,10 @@ sql2sqlx/parser.py,sha256=yKs5FujN_9OgbSoKqU3ioN0PKJXgzAT2TDS9BrQZdYs,77049
 sql2sqlx/py.typed,sha256=daEdpEyAJIa8b2VkCqSKcw8PaExcB6Qro80XNes_sHA,2
 sql2sqlx/refs.py,sha256=UZtom1SYw_QWYVwigniy882eBaL6M9orz_y9lPBRYVI,33916
 sql2sqlx/splitter.py,sha256=Ppo_gxaWmzZO6iYO_0Tc28W1bI4pbP1I3EF1juAw4Yg,14735
-sql2sqlx/version.py,sha256=1AtcjToFoNlIFnG4TyKDefXN69BkLcebIAzKKIS8leY,476
-sql2sqlx-0.1.0.dist-info/licenses/LICENSE,sha256=eK7fnHYjw-1st_pakYslBS41i1dEaHkvmtJzEVoubcE,11347
-sql2sqlx-0.1.0.dist-info/METADATA,sha256=YdE0HgPLhJylIAznnmfil5clKicuam_v9Pxw8-O4_20,28835
-sql2sqlx-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-sql2sqlx-0.1.0.dist-info/entry_points.txt,sha256=n30wxr_Q4nH5h7tTEZO5ohXdJWqePYesmwpA32D4ygk,47
-sql2sqlx-0.1.0.dist-info/top_level.txt,sha256=nbDaiS9bRjW8S9naDOgtVvFBiFhiXgX9k8PcKCF_goI,9
-sql2sqlx-0.1.0.dist-info/RECORD,,
+sql2sqlx/version.py,sha256=p3LTeKIkF_Myl1AFF5RPWlTN28MJvmQHvBnzwyT6YOo,476
+sql2sqlx-0.1.1.dist-info/licenses/LICENSE,sha256=eK7fnHYjw-1st_pakYslBS41i1dEaHkvmtJzEVoubcE,11347
+sql2sqlx-0.1.1.dist-info/METADATA,sha256=B86YhahfoD9Z3WgDejmjTUwytgc7GDGZzY1gdWWhM9M,28792
+sql2sqlx-0.1.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+sql2sqlx-0.1.1.dist-info/entry_points.txt,sha256=n30wxr_Q4nH5h7tTEZO5ohXdJWqePYesmwpA32D4ygk,47
+sql2sqlx-0.1.1.dist-info/top_level.txt,sha256=nbDaiS9bRjW8S9naDOgtVvFBiFhiXgX9k8PcKCF_goI,9
+sql2sqlx-0.1.1.dist-info/RECORD,,