sqlglot 27.29.0__py3-none-any.whl → 28.4.0__py3-none-any.whl

sqlglot/planner.py

@@ -94,7 +94,7 @@ class Step:
         """
         ctes = ctes or {}
         expression = expression.unnest()
-        with_ = expression.args.get("with")
+        with_ = expression.args.get("with_")
 
         # CTEs break the mold of scope and introduce themselves to all in the context.
         if with_:
@@ -104,7 +104,7 @@ class Step:
                 step.name = cte.alias
                 ctes[step.name] = step  # type: ignore
 
-        from_ = expression.args.get("from")
+        from_ = expression.args.get("from_")
 
         if isinstance(expression, exp.Select) and from_:
             step = Scan.from_expression(from_.this, ctes)

sqlglot/schema.py

@@ -18,7 +18,13 @@ if t.TYPE_CHECKING:
 class Schema(abc.ABC):
     """Abstract base class for database schemas"""
 
-    dialect: DialectType
+    @property
+    def dialect(self) -> t.Optional[Dialect]:
+        """
+        Returns None by default. Subclasses that require dialect-specific
+        behavior should override this property.
+        """
+        return None
 
     @abc.abstractmethod
     def add_table(
@@ -222,15 +228,20 @@ class MappingSchema(AbstractMappingSchema, Schema):
         dialect: DialectType = None,
         normalize: bool = True,
     ) -> None:
-        self.dialect = dialect
         self.visible = {} if visible is None else visible
         self.normalize = normalize
+        self._dialect = Dialect.get_or_raise(dialect)
         self._type_mapping_cache: t.Dict[str, exp.DataType] = {}
         self._depth = 0
         schema = {} if schema is None else schema
 
         super().__init__(self._normalize(schema) if self.normalize else schema)
 
+    @property
+    def dialect(self) -> Dialect:
+        """Returns the dialect for this mapping schema."""
+        return self._dialect
+
     @classmethod
     def from_mapping_schema(cls, mapping_schema: MappingSchema) -> MappingSchema:
         return MappingSchema(
@@ -455,8 +466,8 @@ class MappingSchema(AbstractMappingSchema, Schema):
             The resulting expression type.
         """
         if schema_type not in self._type_mapping_cache:
-            dialect = dialect or self.dialect
-            udt = Dialect.get_or_raise(dialect).SUPPORTS_USER_DEFINED_TYPES
+            dialect = Dialect.get_or_raise(dialect) if dialect else self.dialect
+            udt = dialect.SUPPORTS_USER_DEFINED_TYPES
 
             try:
                 expression = exp.DataType.build(schema_type, dialect=dialect, udt=udt)

sqlglot/tokens.py

@@ -41,6 +41,7 @@ class TokenType(AutoName):
     DCOLON = auto()
     DCOLONDOLLAR = auto()
     DCOLONPERCENT = auto()
+    DCOLONQMARK = auto()
     DQMARK = auto()
     SEMICOLON = auto()
     STAR = auto()
@@ -82,7 +83,10 @@ class TokenType(AutoName):
     PARAMETER = auto()
     SESSION = auto()
     SESSION_PARAMETER = auto()
+    SESSION_USER = auto()
     DAMP = auto()
+    AMP_LT = auto()
+    AMP_GT = auto()
     XOR = auto()
     DSTAR = auto()
     QMARK_AMP = auto()
@@ -131,6 +135,7 @@ class TokenType(AutoName):
     UINT = auto()
     BIGINT = auto()
     UBIGINT = auto()
+    BIGNUM = auto()  # unlimited precision int
     INT128 = auto()
     UINT128 = auto()
     INT256 = auto()
@@ -143,6 +148,7 @@ class TokenType(AutoName):
     DECIMAL64 = auto()
     DECIMAL128 = auto()
     DECIMAL256 = auto()
+    DECFLOAT = auto()
     UDECIMAL = auto()
     BIGDECIMAL = auto()
     CHAR = auto()
@@ -165,6 +171,7 @@ class TokenType(AutoName):
     JSONB = auto()
     TIME = auto()
     TIMETZ = auto()
+    TIME_NS = auto()
     TIMESTAMP = auto()
     TIMESTAMPTZ = auto()
     TIMESTAMPLTZ = auto()
@@ -198,6 +205,8 @@ class TokenType(AutoName):
     POINT = auto()
     RING = auto()
     LINESTRING = auto()
+    LOCALTIME = auto()
+    LOCALTIMESTAMP = auto()
     MULTILINESTRING = auto()
     POLYGON = auto()
     MULTIPOLYGON = auto()
@@ -270,6 +279,8 @@ class TokenType(AutoName):
     CURRENT_TIME = auto()
     CURRENT_TIMESTAMP = auto()
     CURRENT_USER = auto()
+    CURRENT_ROLE = auto()
+    CURRENT_CATALOG = auto()
     DECLARE = auto()
     DEFAULT = auto()
     DELETE = auto()
@@ -289,6 +300,7 @@ class TokenType(AutoName):
     EXISTS = auto()
     FALSE = auto()
     FETCH = auto()
+    FILE = auto()
     FILE_FORMAT = auto()
     FILTER = auto()
     FINAL = auto()
@@ -312,6 +324,7 @@ class TokenType(AutoName):
     ILIKE = auto()
     IN = auto()
     INDEX = auto()
+    INDEXED_BY = auto()
     INNER = auto()
     INSERT = auto()
     INSTALL = auto()
@@ -336,6 +349,7 @@ class TokenType(AutoName):
     LOAD = auto()
     LOCK = auto()
     MAP = auto()
+    MATCH = auto()
     MATCH_CONDITION = auto()
     MATCH_RECOGNIZE = auto()
     MEMBER_OF = auto()
@@ -375,6 +389,7 @@ class TokenType(AutoName):
     PUT = auto()
     QUALIFY = auto()
     QUOTE = auto()
+    QDCOLON = auto()
     RANGE = auto()
     RECURSIVE = auto()
     REFRESH = auto()
@@ -538,6 +553,7 @@ class _Tokenizer(type):
         }
 
         klass._STRING_ESCAPES = set(klass.STRING_ESCAPES)
+        klass._ESCAPE_FOLLOW_CHARS = set(klass.ESCAPE_FOLLOW_CHARS)
         klass._IDENTIFIER_ESCAPES = set(klass.IDENTIFIER_ESCAPES)
         klass._COMMENTS = {
             **dict(
@@ -589,6 +605,7 @@ class _Tokenizer(type):
                 tokens_preceding_hint={
                     _TOKEN_TYPE_TO_INDEX[v] for v in klass.TOKENS_PRECEDING_HINT
                 },
+                escape_follow_chars=klass._ESCAPE_FOLLOW_CHARS,
             )
             token_types = RsTokenTypeSettings(
                 bit_string=_TOKEN_TYPE_TO_INDEX[TokenType.BIT_STRING],
@@ -658,6 +675,7 @@ class Tokenizer(metaclass=_Tokenizer):
     QUOTES: t.List[t.Tuple[str, str] | str] = ["'"]
     STRING_ESCAPES = ["'"]
     VAR_SINGLE_TOKENS: t.Set[str] = set()
+    ESCAPE_FOLLOW_CHARS: t.List[str] = []
 
     # The strings in this list can always be used as escapes, regardless of the surrounding
     # identifier delimiters. By default, the closing delimiter is assumed to also act as an
@@ -688,6 +706,7 @@ class Tokenizer(metaclass=_Tokenizer):
     _STRING_ESCAPES: t.Set[str] = set()
     _KEYWORD_TRIE: t.Dict = {}
     _RS_TOKENIZER: t.Optional[t.Any] = None
+    _ESCAPE_FOLLOW_CHARS: t.Set[str] = set()
 
     KEYWORDS: t.Dict[str, TokenType] = {
         **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")},
@@ -697,6 +716,7 @@ class Tokenizer(metaclass=_Tokenizer):
         HINT_START: TokenType.HINT,
         "==": TokenType.EQ,
         "::": TokenType.DCOLON,
+        "?::": TokenType.QDCOLON,
         "||": TokenType.DPIPE,
         "|>": TokenType.PIPE_GT,
         ">=": TokenType.GTE,
@@ -747,6 +767,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "CURRENT_TIME": TokenType.CURRENT_TIME,
         "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP,
         "CURRENT_USER": TokenType.CURRENT_USER,
+        "CURRENT_CATALOG": TokenType.CURRENT_CATALOG,
         "DATABASE": TokenType.DATABASE,
         "DEFAULT": TokenType.DEFAULT,
         "DELETE": TokenType.DELETE,
@@ -766,6 +787,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "FALSE": TokenType.FALSE,
         "FETCH": TokenType.FETCH,
         "FILTER": TokenType.FILTER,
+        "FILE": TokenType.FILE,
         "FIRST": TokenType.FIRST,
         "FULL": TokenType.FULL,
         "FUNCTION": TokenType.FUNCTION,
@@ -798,6 +820,8 @@ class Tokenizer(metaclass=_Tokenizer):
         "LIKE": TokenType.LIKE,
         "LIMIT": TokenType.LIMIT,
         "LOAD": TokenType.LOAD,
+        "LOCALTIME": TokenType.LOCALTIME,
+        "LOCALTIMESTAMP": TokenType.LOCALTIMESTAMP,
         "LOCK": TokenType.LOCK,
         "MERGE": TokenType.MERGE,
         "NAMESPACE": TokenType.NAMESPACE,
@@ -844,6 +868,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "SELECT": TokenType.SELECT,
         "SEMI": TokenType.SEMI,
         "SESSION": TokenType.SESSION,
+        "SESSION_USER": TokenType.SESSION_USER,
         "SET": TokenType.SET,
         "SETTINGS": TokenType.SETTINGS,
         "SHOW": TokenType.SHOW,
@@ -908,8 +933,10 @@ class Tokenizer(metaclass=_Tokenizer):
         "DECIMAL64": TokenType.DECIMAL64,
         "DECIMAL128": TokenType.DECIMAL128,
         "DECIMAL256": TokenType.DECIMAL256,
+        "DECFLOAT": TokenType.DECFLOAT,
         "BIGDECIMAL": TokenType.BIGDECIMAL,
         "BIGNUMERIC": TokenType.BIGDECIMAL,
+        "BIGNUM": TokenType.BIGNUM,
         "LIST": TokenType.LIST,
         "MAP": TokenType.MAP,
         "NULLABLE": TokenType.NULLABLE,
@@ -951,6 +978,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "VARBINARY": TokenType.VARBINARY,
         "TIME": TokenType.TIME,
         "TIMETZ": TokenType.TIMETZ,
+        "TIME_NS": TokenType.TIME_NS,
         "TIMESTAMP": TokenType.TIMESTAMP,
         "TIMESTAMPTZ": TokenType.TIMESTAMPTZ,
         "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ,
@@ -1340,6 +1368,8 @@ class Tokenizer(metaclass=_Tokenizer):
             elif self._peek.upper() == "E" and not scientific:
                 scientific += 1
                 self._advance()
+            elif self._peek == "_" and self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED:
+                self._advance()
             elif self._peek.isidentifier():
                 number_text = self._text
                 literal = ""
@@ -1354,12 +1384,8 @@ class Tokenizer(metaclass=_Tokenizer):
                     self._add(TokenType.NUMBER, number_text)
                     self._add(TokenType.DCOLON, "::")
                     return self._add(token_type, literal)
-                else:
-                    replaced = literal.replace("_", "")
-                    if self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED and replaced.isdigit():
-                        return self._add(TokenType.NUMBER, number_text + replaced)
-                    if self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT:
-                        return self._add(TokenType.VAR)
+                elif self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT:
+                    return self._add(TokenType.VAR)
 
                 self._advance(-len(literal))
                 return self._add(TokenType.NUMBER, number_text)
@@ -1495,14 +1521,23 @@ class Tokenizer(metaclass=_Tokenizer):
                     self._advance(2)
                     text += unescaped_sequence
                     continue
+
+            is_valid_custom_escape = (
+                self.ESCAPE_FOLLOW_CHARS
+                and self._char == "\\"
+                and self._peek not in self.ESCAPE_FOLLOW_CHARS
+            )
+
             if (
                 (self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS or not raw_string)
                 and self._char in escapes
-                and (self._peek == delimiter or self._peek in escapes)
+                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
                 and (self._char not in self._QUOTES or self._char == self._peek)
             ):
                 if self._peek == delimiter:
                     text += self._peek
+                elif is_valid_custom_escape and self._char != self._peek:
+                    text += self._peek
                 else:
                     text += self._char + self._peek
 

sqlglot/transforms.py

@@ -4,7 +4,7 @@ import typing as t
 
 from sqlglot import expressions as exp
 from sqlglot.errors import UnsupportedError
-from sqlglot.helper import find_new_name, name_sequence
+from sqlglot.helper import find_new_name, name_sequence, seq_get
 
 
 if t.TYPE_CHECKING:
@@ -14,6 +14,7 @@ if t.TYPE_CHECKING:
 
 def preprocess(
     transforms: t.List[t.Callable[[exp.Expression], exp.Expression]],
+    generator: t.Optional[t.Callable[[Generator, exp.Expression], str]] = None,
 ) -> t.Callable[[Generator, exp.Expression], str]:
     """
     Creates a new transform by chaining a sequence of transformations and converts the resulting
@@ -37,6 +38,9 @@ def preprocess(
         except UnsupportedError as unsupported_error:
             self.unsupported(str(unsupported_error))
 
+        if generator:
+            return generator(self, expression)
+
         _sql_handler = getattr(self, expression.key + "_sql", None)
         if _sql_handler:
             return _sql_handler(expression)
@@ -110,10 +114,10 @@ def unnest_generate_date_array_using_recursive_cte(expression: exp.Expression) -
             count += 1
 
         if recursive_ctes:
-            with_expression = expression.args.get("with") or exp.With()
+            with_expression = expression.args.get("with_") or exp.With()
             with_expression.set("recursive", True)
             with_expression.set("expressions", [*recursive_ctes, *with_expression.expressions])
-            expression.set("with", with_expression)
+            expression.set("with_", with_expression)
 
     return expression
 
@@ -310,14 +314,14 @@ def unnest_to_explode(
         return exp.Inline if has_multi_expr else exp.Explode
 
     if isinstance(expression, exp.Select):
-        from_ = expression.args.get("from")
+        from_ = expression.args.get("from_")
 
         if from_ and isinstance(from_.this, exp.Unnest):
             unnest = from_.this
             alias = unnest.args.get("alias")
             exprs = unnest.expressions
             has_multi_expr = len(exprs) > 1
-            this, *expressions = _unnest_zip_exprs(unnest, exprs, has_multi_expr)
+            this, *_ = _unnest_zip_exprs(unnest, exprs, has_multi_expr)
 
             columns = alias.columns if alias else []
             offset = unnest.args.get("offset")
@@ -328,10 +332,7 @@ def unnest_to_explode(
 
             unnest.replace(
                 exp.Table(
-                    this=_udtf_type(unnest, has_multi_expr)(
-                        this=this,
-                        expressions=expressions,
-                    ),
+                    this=_udtf_type(unnest, has_multi_expr)(this=this),
                     alias=exp.TableAlias(this=alias.this, columns=columns) if alias else None,
                 )
             )
@@ -494,7 +495,7 @@ def explode_projection_to_unnest(
                         expression.set("expressions", expressions)
 
                     if not arrays:
-                        if expression.args.get("from"):
+                        if expression.args.get("from_"):
                             expression.join(series, copy=False, join_type="CROSS")
                         else:
                             expression.from_(series, copy=False)
@@ -638,7 +639,7 @@ def eliminate_full_outer_join(expression: exp.Expression) -> exp.Expression:
             expression.set("limit", None)
             index, full_outer_join = full_outer_joins[0]
 
-            tables = (expression.args["from"].alias_or_name, full_outer_join.alias_or_name)
+            tables = (expression.args["from_"].alias_or_name, full_outer_join.alias_or_name)
             join_conditions = full_outer_join.args.get("on") or exp.and_(
                 *[
                     exp.column(col, tables[0]).eq(exp.column(col, tables[1]))
@@ -647,10 +648,12 @@ def eliminate_full_outer_join(expression: exp.Expression) -> exp.Expression:
             )
 
             full_outer_join.set("side", "left")
-            anti_join_clause = exp.select("1").from_(expression.args["from"]).where(join_conditions)
+            anti_join_clause = (
+                exp.select("1").from_(expression.args["from_"]).where(join_conditions)
+            )
             expression_copy.args["joins"][index].set("side", "right")
             expression_copy = expression_copy.where(exp.Exists(this=anti_join_clause).not_())
-            expression_copy.set("with", None)  # remove CTEs from RIGHT side
+            expression_copy.set("with_", None)  # remove CTEs from RIGHT side
             expression.set("order", None)  # remove order by from LEFT side
 
             return exp.union(expression, expression_copy, copy=False, distinct=False)
@@ -670,14 +673,14 @@ def move_ctes_to_top_level(expression: E) -> E:
 
     TODO: handle name clashes whilst moving CTEs (it can get quite tricky & costly).
     """
-    top_level_with = expression.args.get("with")
+    top_level_with = expression.args.get("with_")
     for inner_with in expression.find_all(exp.With):
         if inner_with.parent is expression:
             continue
 
         if not top_level_with:
             top_level_with = inner_with.pop()
-            expression.set("with", top_level_with)
+            expression.set("with_", top_level_with)
         else:
             if inner_with.recursive:
                 top_level_with.set("recursive", True)
@@ -874,13 +877,12 @@ def eliminate_join_marks(expression: exp.Expression) -> exp.Expression:
         where = query.args.get("where")
         joins = query.args.get("joins", [])
 
-        # knockout: we do not support left correlation (see point 2)
-        assert not scope.is_correlated_subquery, "Correlated queries are not supported"
-
-        # nothing to do - we check it here after knockout above
         if not where or not any(c.args.get("join_mark") for c in where.find_all(exp.Column)):
             continue
 
+        # knockout: we do not support left correlation (see point 2)
+        assert not scope.is_correlated_subquery, "Correlated queries are not supported"
+
         # make sure we have AND of ORs to have clear join terms
         where = normalize(where.this)
         assert normalized(where), "Cannot normalize JOIN predicates"
@@ -904,7 +906,7 @@ def eliminate_join_marks(expression: exp.Expression) -> exp.Expression:
 
         old_joins = {join.alias_or_name: join for join in joins}
         new_joins = {}
-        query_from = query.args["from"]
+        query_from = query.args["from_"]
 
         for table, predicates in joins_ons.items():
             join_what = old_joins.get(table, query_from).this.copy()
@@ -930,11 +932,11 @@ def eliminate_join_marks(expression: exp.Expression) -> exp.Expression:
             ), "Cannot determine which table to use in the new FROM clause"
 
             new_from_name = list(only_old_joins)[0]
-            query.set("from", exp.From(this=old_joins[new_from_name].this))
+            query.set("from_", exp.From(this=old_joins[new_from_name].this))
 
         if new_joins:
             for n, j in old_joins.items():  # preserve any other joins
-                if n not in new_joins and n != query.args["from"].name:
+                if n not in new_joins and n != query.args["from_"].name:
                     if not j.kind:
                         j.set("kind", "CROSS")
                     new_joins[n] = j
@@ -999,3 +1001,56 @@ def eliminate_window_clause(expression: exp.Expression) -> exp.Expression:
             _inline_inherited_window(window)
 
     return expression
+
+
+def inherit_struct_field_names(expression: exp.Expression) -> exp.Expression:
+    """
+    Inherit field names from the first struct in an array.
+
+    BigQuery supports implicitly inheriting names from the first STRUCT in an array:
+
+    Example:
+        ARRAY[
+          STRUCT('Alice' AS name, 85 AS score),  -- defines names
+          STRUCT('Bob', 92),                     -- inherits names
+          STRUCT('Diana', 95)                    -- inherits names
+        ]
+
+    This transformation makes the field names explicit on all structs by adding
+    PropertyEQ nodes, in order to facilitate transpilation to other dialects.
+
+    Args:
+        expression: The expression tree to transform
+
+    Returns:
+        The modified expression with field names inherited in all structs
+    """
+    if (
+        isinstance(expression, exp.Array)
+        and expression.args.get("struct_name_inheritance")
+        and isinstance(first_item := seq_get(expression.expressions, 0), exp.Struct)
+        and all(isinstance(fld, exp.PropertyEQ) for fld in first_item.expressions)
+    ):
+        field_names = [fld.this for fld in first_item.expressions]
+
+        # Apply field names to subsequent structs that don't have them
+        for struct in expression.expressions[1:]:
+            if not isinstance(struct, exp.Struct) or len(struct.expressions) != len(field_names):
+                continue
+
+            # Convert unnamed expressions to PropertyEQ with inherited names
+            new_expressions = []
+            for i, expr in enumerate(struct.expressions):
+                if not isinstance(expr, exp.PropertyEQ):
+                    # Create PropertyEQ: field_name := value
+                    new_expressions.append(
+                        exp.PropertyEQ(
+                            this=exp.Identifier(this=field_names[i].copy()), expression=expr
+                        )
+                    )
+                else:
+                    new_expressions.append(expr)
+
+            struct.set("expressions", new_expressions)
+
+    return expression