sql2sqlx 0.1.0__py3-none-any.whl

sql2sqlx/emitter.py

@@ -0,0 +1,275 @@
+# Copyright (c) Soumyadip Sarkar.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Render Dataform ``.sqlx`` files.
+
+The emitter's contract is *character fidelity for untouched SQL*:
+generated bodies are produced by applying span edits (reference rewrites,
+select-list aliases, ``${self()}`` substitutions) to slices of the
+**original** source text - tokens are never re-serialized, so the user's
+formatting, casing and inline comments survive after input decoding.
+
+Two SQLX-specific safety rules live here:
+
+* Any literal ``${`` in a SQL string or quoted identifier is emitted through
+  a constant JavaScript placeholder. Dataform does not provide a backslash
+  escape for SQL placeholders, so ``\\${`` is insufficient. Replacement text
+  inserted *by* sql2sqlx (``${ref(...)}``, ``${self()}``) stays active.
+* Operations bodies have exactly one trailing semicolon stripped:
+  Dataform executes the body as a BigQuery script, and a trailing empty
+  statement is at best noise.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
+from sql2sqlx.lexer import BACKTICK, PARAM, STRING, Token
+from sql2sqlx.model import TableName
+
+#: Canonical top-level config key order (unknown keys follow, insertion-ordered).
+_KEY_ORDER = (
+    "type",
+    "database",
+    "schema",
+    "name",
+    "materialized",
+    "protected",
+    "hasOutput",
+    "uniqueKey",
+    "description",
+    "columns",
+    "dependencies",
+    "tags",
+    "bigquery",
+    "disabled",
+)
+
+#: Canonical key order inside the ``bigquery`` block.
+_BQ_ORDER = (
+    "partitionBy",
+    "clusterBy",
+    "updatePartitionFilter",
+    "labels",
+    "partitionExpirationDays",
+    "requirePartitionFilter",
+    "additionalOptions",
+)
+
+_JS_IDENT_RE = re.compile(r"^[A-Za-z_$][A-Za-z0-9_$]*$")
+
+
+def _json(value: str) -> str:
+    """JSON-encode a string for embedding in JS (non-ASCII preserved)."""
+    return json.dumps(value, ensure_ascii=False)
+
+
+def sqlx_constant(value: str) -> str:
+    """Return a SQLX placeholder that evaluates to the constant ``value``."""
+    return "${" + _json(value) + "}"
+
+
+def sqlx_escape_edits(tokens: Sequence[Token]) -> List[Tuple[int, int, str]]:
+    """Build edits that preserve literal ``${`` through Dataform compile.
+
+    GoogleSQL only permits ``$`` in the relevant valid inputs inside string
+    literals or quoted identifiers. String occurrences can be replaced by a
+    constant placeholder for the two characters. A quoted identifier must be
+    replaced as a whole because a SQLX placeholder nested between BigQuery
+    backticks produces invalid generated JavaScript.
+    """
+    edits: List[Tuple[int, int, str]] = []
+    for token in tokens:
+        if "${" not in token.text:
+            continue
+        if token.kind == BACKTICK or (token.kind == PARAM and token.text.startswith("@`")):
+            edits.append((token.start, token.end, sqlx_constant(token.text)))
+            continue
+        if token.kind != STRING:
+            continue
+        offset = 0
+        while True:
+            offset = token.text.find("${", offset)
+            if offset < 0:
+                break
+            start = token.start + offset
+            edits.append((start, start + 2, sqlx_constant("${")))
+            offset += 2
+    return edits
+
+
+def _js_key(key: str) -> str:
+    """Render an object key: bare when a valid JS identifier, else quoted."""
+    return key if _JS_IDENT_RE.match(key) else _json(key)
+
+
+def _js_value(value: Any, indent: int) -> str:
+    """Render a Python value as JavaScript object-literal source.
+
+    Args:
+        value: ``str``/``bool``/``int``/``float``/``list``/``dict`` (nested).
+        indent: Current indentation level (2 spaces per level).
+
+    Returns:
+        JS source text for the value.
+    """
+    pad = "  " * indent
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        return str(int(value)) if value.is_integer() else repr(value)
+    if isinstance(value, str):
+        return _json(value)
+    if isinstance(value, (list, tuple)):
+        inner = [_js_value(v, indent + 1) for v in value]
+        one_line = "[" + ", ".join(inner) + "]"
+        if len(one_line) <= 72 and "\n" not in one_line:
+            return one_line
+        ip = "  " * (indent + 1)
+        return "[\n" + ",\n".join(ip + v for v in inner) + "\n" + pad + "]"
+    if isinstance(value, dict):
+        if not value:
+            return "{}"
+        ip = "  " * (indent + 1)
+        lines = [f"{ip}{_js_key(k)}: {_js_value(v, indent + 1)}" for k, v in value.items()]
+        return "{\n" + ",\n".join(lines) + "\n" + pad + "}"
+    return _json(str(value))  # pragma: no cover - defensive
+
+
+def render_config(config: Dict[str, Any]) -> str:
+    """Render a Dataform ``config { ... }`` block.
+
+    Keys are emitted in canonical order (:data:`_KEY_ORDER`, then any
+    remaining keys in insertion order); the nested ``bigquery`` object is
+    ordered by :data:`_BQ_ORDER`. Ordering is purely cosmetic but makes
+    output deterministic and diff-friendly.
+
+    Args:
+        config: The config mapping.
+
+    Returns:
+        The complete ``config { ... }`` block, without a trailing newline.
+    """
+    ordered: Dict[str, Any] = {}
+    for key in _KEY_ORDER:
+        if key in config:
+            ordered[key] = config[key]
+    for key, val in config.items():
+        if key not in ordered:
+            ordered[key] = val
+    bq = ordered.get("bigquery")
+    if isinstance(bq, dict):
+        obq: Dict[str, Any] = {k: bq[k] for k in _BQ_ORDER if k in bq}
+        for k, v in bq.items():
+            if k not in obq:
+                obq[k] = v
+        ordered["bigquery"] = obq
+    lines = [f"  {_js_key(k)}: {_js_value(v, 1)}" for k, v in ordered.items()]
+    return "config {\n" + ",\n".join(lines) + "\n}"
+
+
+def apply_edits_escaped(
+    text: str, start: int, end: int, edits: Iterable[Tuple[int, int, str]]
+) -> str:
+    """Slice ``text[start:end]`` and apply non-overlapping span edits.
+
+    Edits use *absolute* offsets into ``text``. Edits outside the slice or
+    overlapping an already-applied edit are skipped defensively. Literal SQLX
+    interpolation is handled by :func:`sqlx_escape_edits`; replacements such
+    as ``ref()`` and ``self()`` are inserted verbatim and remain active.
+
+    Args:
+        text: Full original source text.
+        start: Slice start offset.
+        end: Slice end offset.
+        edits: ``(start, end, replacement)`` triples, any order.
+
+    Returns:
+        The edited, escaped slice.
+    """
+    segments: List[str] = []
+    pos = start
+    for a, b, replacement in sorted(edits, key=lambda e: (e[0], e[1])):
+        if a < pos or a < start or b > end or b < a:
+            continue
+        segments.append(text[pos:a])
+        segments.append(replacement)
+        pos = b
+    segments.append(text[pos:end])
+    return "".join(segments)
+
+
+def ref_expr(name: TableName) -> str:
+    """Build the ``${ref(...)}`` expression for a produced table.
+
+    The reference carries exactly the qualification the *producer* was
+    declared with: ``${ref("name")}``, ``${ref("schema", "name")}``, or
+    the object form ``${ref({database: ..., schema: ..., name: ...})}``
+    when a project was explicit.
+
+    Args:
+        name: The producer's original (pre-resolution) table name.
+
+    Returns:
+        The interpolation expression text.
+    """
+    if name.project:
+        return "${ref({database: %s, schema: %s, name: %s})}" % (
+            _json(name.project),
+            _json(name.dataset or ""),
+            _json(name.table),
+        )
+    if name.dataset:
+        return "${ref(%s, %s)}" % (_json(name.dataset), _json(name.table))
+    return "${ref(%s)}" % _json(name.table)
+
+
+def build_sqlx(
+    config: Dict[str, Any],
+    body: Optional[str] = None,
+    annotation: Optional[str] = None,
+    leading_comments: Optional[str] = None,
+    trailing_comments: Optional[str] = None,
+) -> str:
+    """Assemble a complete ``.sqlx`` file.
+
+    Layout: ``config`` block, blank line, then (in order) the provenance
+    annotation comment, the statement's original leading comments, and the
+    SQL body. One trailing semicolon is stripped from the body. When the
+    body is ``None`` (declarations) only the config block is emitted.
+
+    Args:
+        config: The config mapping (see :func:`render_config`).
+        body: Edited SQL body, or ``None``.
+        annotation: Provenance comment line, or ``None``.
+        leading_comments: Comments that preceded the statement, or ``None``.
+        trailing_comments: Final file comments after the statement's
+            terminating semicolon, or ``None``.
+
+    Returns:
+        The full file contents, newline-terminated.
+    """
+    head = render_config(config)
+    bits: List[str] = []
+    if annotation:
+        bits.append(annotation)
+    if leading_comments and leading_comments.strip():
+        bits.append(leading_comments.strip())
+    if body is not None:
+        cleaned = body.strip()
+        if cleaned.endswith(";"):
+            cleaned = cleaned[:-1].rstrip()
+        if cleaned:
+            bits.append(cleaned)
+    if trailing_comments and trailing_comments.strip():
+        bits.append(trailing_comments.strip())
+    if bits:
+        return head + "\n\n" + "\n".join(bits) + "\n"
+    return head + "\n"

sql2sqlx/errors.py

@@ -0,0 +1,93 @@
+# Copyright (c) Soumyadip Sarkar.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Exception hierarchy for :mod:`sql2sqlx`.
+
+All exceptions raised deliberately by this library derive from
+:class:`Sql2SqlxError`, so callers can catch a single base class.
+Location-aware errors (lexing/splitting problems) carry a 1-based
+``line`` and ``column`` pointing at the offending character in the
+original SQL text.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+
+class Sql2SqlxError(Exception):
+    """Base class for all errors raised by sql2sqlx."""
+
+
+class LexError(Sql2SqlxError):
+    """Raised when the SQL text cannot be tokenized.
+
+    Typical causes are unterminated string literals, unterminated
+    backtick-quoted identifiers, or unterminated ``/* ... */`` block
+    comments.
+
+    Attributes:
+        message: Human-readable description of the problem.
+        line: 1-based line number of the offending character.
+        column: 1-based column number of the offending character.
+    """
+
+    def __init__(self, message: str, line: int, column: int) -> None:
+        """Initialize the error.
+
+        Args:
+            message: Human-readable description of the problem.
+            line: 1-based line number in the source text.
+            column: 1-based column number in the source text.
+        """
+        super().__init__(f"{message} (line {line}, column {column})")
+        self.message = message
+        self.line = line
+        self.column = column
+
+
+class SplitError(Sql2SqlxError):
+    """Raised when statement splitting fails irrecoverably.
+
+    In practice the splitter is lenient (mismatched ``END`` markers are
+    tolerated), so this error is reserved for structural impossibilities
+    such as unbalanced parentheses at end of input.
+    """
+
+    def __init__(self, message: str, line: int = 0, column: int = 0) -> None:
+        """Initialize the error.
+
+        Args:
+            message: Human-readable description of the problem.
+            line: 1-based line number in the source text (0 if unknown).
+            column: 1-based column number in the source text (0 if unknown).
+        """
+        loc = f" (line {line}, column {column})" if line else ""
+        super().__init__(f"{message}{loc}")
+        self.message = message
+        self.line = line
+        self.column = column
+
+
+class ConversionError(Sql2SqlxError):
+    """Raised when a statement or file cannot be converted at all.
+
+    Note that *unsupported-but-valid* SQL never raises: it falls back to
+    a Dataform ``operations`` action and a warning is recorded in the
+    :class:`~sql2sqlx.model.ConversionReport`. This exception is for
+    genuinely broken input (e.g. unreadable files, lexer failures).
+    """
+
+    def __init__(self, message: str, path: Optional[str] = None) -> None:
+        """Initialize the error.
+
+        Args:
+            message: Human-readable description of the problem.
+            path: Path of the file being converted, if applicable.
+        """
+        super().__init__(f"{path}: {message}" if path else message)
+        self.message = message
+        self.path = path

sql2sqlx/keywords.py

@@ -0,0 +1,169 @@
+# Copyright (c) Soumyadip Sarkar.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""GoogleSQL reserved keywords and related token classification sets.
+
+:data:`RESERVED` is the official GoogleSQL reserved-keyword list. It is
+used to decide whether a bare identifier can be a table alias (reserved
+words cannot be used as unquoted aliases), which keeps the reference
+scanner from mistaking clause keywords for aliases.
+
+:data:`FROM_CLAUSE_ENDERS` are keywords that terminate the table list of
+a ``FROM`` clause at the same nesting depth; encountering one switches
+the reference scanner out of "expecting a table path" mode.
+"""
+
+from __future__ import annotations
+
+from typing import FrozenSet
+
+#: Official GoogleSQL reserved keywords (case-insensitive).
+RESERVED: FrozenSet[str] = frozenset(
+    {
+        "ALL",
+        "AND",
+        "ANY",
+        "ARRAY",
+        "AS",
+        "ASC",
+        "ASSERT_ROWS_MODIFIED",
+        "AT",
+        "BETWEEN",
+        "BY",
+        "CASE",
+        "CAST",
+        "COLLATE",
+        "CONTAINS",
+        "CREATE",
+        "CROSS",
+        "CUBE",
+        "CURRENT",
+        "DEFAULT",
+        "DEFINE",
+        "DESC",
+        "DISTINCT",
+        "ELSE",
+        "END",
+        "ENUM",
+        "ESCAPE",
+        "EXCEPT",
+        "EXCLUDE",
+        "EXISTS",
+        "EXTRACT",
+        "FALSE",
+        "FETCH",
+        "FOLLOWING",
+        "FOR",
+        "FROM",
+        "FULL",
+        "GRAPH_TABLE",
+        "GROUP",
+        "GROUPING",
+        "GROUPS",
+        "HASH",
+        "HAVING",
+        "IF",
+        "IGNORE",
+        "IN",
+        "INNER",
+        "INTERSECT",
+        "INTERVAL",
+        "INTO",
+        "IS",
+        "JOIN",
+        "LATERAL",
+        "LEFT",
+        "LIKE",
+        "LIMIT",
+        "LOOKUP",
+        "MERGE",
+        "NATURAL",
+        "NEW",
+        "NO",
+        "NOT",
+        "NULL",
+        "NULLS",
+        "OF",
+        "ON",
+        "OR",
+        "ORDER",
+        "OUTER",
+        "OVER",
+        "PARTITION",
+        "PRECEDING",
+        "PROTO",
+        "QUALIFY",
+        "RANGE",
+        "RECURSIVE",
+        "RESPECT",
+        "RIGHT",
+        "ROLLUP",
+        "ROWS",
+        "SELECT",
+        "SET",
+        "SOME",
+        "STRUCT",
+        "TABLESAMPLE",
+        "THEN",
+        "TO",
+        "TREAT",
+        "TRUE",
+        "UNBOUNDED",
+        "UNION",
+        "UNNEST",
+        "USING",
+        "WHEN",
+        "WHERE",
+        "WINDOW",
+        "WITH",
+        "WITHIN",
+    }
+)
+
+#: Keywords that end the table list of a FROM clause at the same depth.
+FROM_CLAUSE_ENDERS: FrozenSet[str] = frozenset(
+    {
+        "WHERE",
+        "GROUP",
+        "HAVING",
+        "QUALIFY",
+        "WINDOW",
+        "ORDER",
+        "LIMIT",
+        "UNION",
+        "INTERSECT",
+        "EXCEPT",
+        "SET",
+        "WHEN",
+        "THEN",
+        "RETURNING",
+    }
+)
+
+#: Date/time part identifiers; presence at top level of a select-list item
+#: makes implicit-alias detection ambiguous (``INTERVAL 1 DAY``), so items
+#: containing INTERVAL trigger a safe fallback instead of a guess.
+DATE_PARTS: FrozenSet[str] = frozenset(
+    {
+        "YEAR",
+        "QUARTER",
+        "MONTH",
+        "WEEK",
+        "ISOWEEK",
+        "DAY",
+        "DAYOFWEEK",
+        "DAYOFYEAR",
+        "HOUR",
+        "MINUTE",
+        "SECOND",
+        "MILLISECOND",
+        "MICROSECOND",
+        "ISOYEAR",
+        "DATE",
+        "TIME",
+        "DATETIME",
+    }
+)