sqlglot 26.26.0__py3-none-any.whl → 26.28.1__py3-none-any.whl

sqlglot/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '26.26.0'
-__version_tuple__ = version_tuple = (26, 26, 0)
+__version__ = version = '26.28.1'
+__version_tuple__ = version_tuple = (26, 28, 1)

sqlglot/dialects/hive.py

@@ -557,6 +557,7 @@ class Hive(Dialect):
             exp.GenerateDateArray: sequence_sql,
             exp.If: if_sql(),
             exp.ILike: no_ilike_sql,
+            exp.IntDiv: lambda self, e: self.binary(e, "DIV"),
             exp.IsNan: rename_func("ISNAN"),
             exp.JSONExtract: lambda self, e: self.func("GET_JSON_OBJECT", e.this, e.expression),
             exp.JSONExtractScalar: lambda self, e: self.func(

sqlglot/dialects/mysql.py

@@ -489,6 +489,27 @@ class MySQL(Dialect):
         VALUES_FOLLOWED_BY_PAREN = False
         SUPPORTS_PARTITION_SELECTION = True
 
+        def _parse_generated_as_identity(
+            self,
+        ) -> (
+            exp.GeneratedAsIdentityColumnConstraint
+            | exp.ComputedColumnConstraint
+            | exp.GeneratedAsRowColumnConstraint
+        ):
+            this = super()._parse_generated_as_identity()
+
+            if self._match_texts(("STORED", "VIRTUAL")):
+                persisted = self._prev.text.upper() == "STORED"
+
+                if isinstance(this, exp.ComputedColumnConstraint):
+                    this.set("persisted", persisted)
+                elif isinstance(this, exp.GeneratedAsIdentityColumnConstraint):
+                    this = self.expression(
+                        exp.ComputedColumnConstraint, this=this.expression, persisted=persisted
+                    )
+
+            return this
+
         def _parse_primary_key_part(self) -> t.Optional[exp.Expression]:
             this = self._parse_id_var()
             if not self._match(TokenType.L_PAREN):
@@ -1154,6 +1175,10 @@ class MySQL(Dialect):
             "zerofill",
         }
 
+        def computedcolumnconstraint_sql(self, expression: exp.ComputedColumnConstraint) -> str:
+            persisted = "STORED" if expression.args.get("persisted") else "VIRTUAL"
+            return f"GENERATED ALWAYS AS ({self.sql(expression.this.unnest())}) {persisted}"
+
         def array_sql(self, expression: exp.Array) -> str:
             self.unsupported("Arrays are not supported by MySQL")
             return self.function_fallback_sql(expression)

sqlglot/dialects/postgres.py

@@ -512,6 +512,18 @@ class Postgres(Dialect):
 
             return this
 
+        def _parse_user_defined_type(
+            self, identifier: exp.Identifier
+        ) -> t.Optional[exp.Expression]:
+            udt_type: exp.Identifier | exp.Dot = identifier
+
+            while self._match(TokenType.DOT):
+                part = self._parse_id_var()
+                if part:
+                    udt_type = exp.Dot(this=udt_type, expression=part)
+
+            return exp.DataType.build(udt_type, udt=True)
+
     class Generator(generator.Generator):
         SINGLE_STRING_INTERVAL = True
         RENAME_TABLE_WITH_DB = False

sqlglot/dialects/snowflake.py

@@ -863,8 +863,14 @@ class Snowflake(Dialect):
                 properties=self._parse_properties(),
             )
 
-        def _parse_get(self) -> exp.Get | exp.Command:
+        def _parse_get(self) -> t.Optional[exp.Expression]:
             start = self._prev
+
+            # If we detect GET( then we need to parse a function, not a statement
+            if self._match(TokenType.L_PAREN):
+                self._retreat(self._index - 2)
+                return self._parse_expression()
+
             target = self._parse_location_path()
 
             # Parse as command if unquoted file path
@@ -1100,6 +1106,7 @@ class Snowflake(Dialect):
                 self, e, func_name="CHARINDEX", supports_position=True
             ),
             exp.StrToDate: lambda self, e: self.func("DATE", e.this, self.format_time(e)),
+            exp.StringToArray: rename_func("STRTOK_TO_ARRAY"),
             exp.Stuff: rename_func("INSERT"),
             exp.StPoint: rename_func("ST_MAKEPOINT"),
             exp.TimeAdd: date_delta_sql("TIMEADD"),

sqlglot/dialects/sqlite.py

@@ -109,7 +109,9 @@ class SQLite(Dialect):
             "DATETIME": lambda args: exp.Anonymous(this="DATETIME", expressions=args),
             "TIME": lambda args: exp.Anonymous(this="TIME", expressions=args),
         }
+
         STRING_ALIASES = True
+        ALTER_RENAME_REQUIRES_COLUMN = False
 
         def _parse_unique(self) -> exp.UniqueColumnConstraint:
             # Do not consume more tokens if UNIQUE is used as a standalone constraint, e.g:

sqlglot/expressions.py

@@ -51,8 +51,8 @@ class _Expression(type):
     def __new__(cls, clsname, bases, attrs):
         klass = super().__new__(cls, clsname, bases, attrs)
 
-        # When an Expression class is created, its key is automatically set to be
-        # the lowercase version of the class' name.
+        # When an Expression class is created, its key is automatically set
+        # to be the lowercase version of the class' name.
         klass.key = clsname.lower()
 
         # This is so that docstrings are not inherited in pdoc
@@ -1724,15 +1724,15 @@ class Column(Condition):
             if self.args.get(part)
         ]
 
-    def to_dot(self) -> Dot | Identifier:
+    def to_dot(self, include_dots: bool = True) -> Dot | Identifier:
         """Converts the column into a dot expression."""
         parts = self.parts
         parent = self.parent
 
-        while parent:
-            if isinstance(parent, Dot):
+        if include_dots:
+            while isinstance(parent, Dot):
                 parts.append(parent.expression)
-            parent = parent.parent
+                parent = parent.parent
 
         return Dot.build(deepcopy(parts)) if len(parts) > 1 else parts[0]
 
@@ -4753,6 +4753,8 @@ class DataType(Expression):
                 if udt:
                     return DataType(this=DataType.Type.USERDEFINED, kind=dtype, **kwargs)
                 raise
+        elif isinstance(dtype, (Identifier, Dot)) and udt:
+            return DataType(this=DataType.Type.USERDEFINED, kind=dtype, **kwargs)
         elif isinstance(dtype, DataType.Type):
             data_type_exp = DataType(this=dtype)
         elif isinstance(dtype, DataType):
@@ -4794,9 +4796,6 @@ class DataType(Expression):
         return False
 
 
-DATA_TYPE = t.Union[str, DataType, DataType.Type]
-
-
 # https://www.postgresql.org/docs/15/datatype-pseudo.html
 class PseudoType(DataType):
     arg_types = {"this": True}
@@ -4926,6 +4925,10 @@ class AddConstraint(Expression):
     arg_types = {"expressions": True}
 
 
+class AddPartition(Expression):
+    arg_types = {"this": True, "exists": False}
+
+
 class AttachOption(Expression):
     arg_types = {"this": True, "expression": False}
 
@@ -5026,6 +5029,9 @@ class Dot(Binary):
         return parts
 
 
+DATA_TYPE = t.Union[str, Identifier, Dot, DataType, DataType.Type]
+
+
 class DPipe(Binary):
     arg_types = {"this": True, "expression": True, "safe": False}
 
@@ -5587,7 +5593,7 @@ class String(Func):
 
 
 class StringToArray(Func):
-    arg_types = {"this": True, "expression": True, "null": False}
+    arg_types = {"this": True, "expression": False, "null": False}
     _sql_names = ["STRING_TO_ARRAY", "SPLIT_BY_STRING", "STRTOK_TO_ARRAY"]
 
 

sqlglot/generator.py

@@ -3498,6 +3498,10 @@ class Generator(metaclass=_Generator):
     def addconstraint_sql(self, expression: exp.AddConstraint) -> str:
         return f"ADD {self.expressions(expression)}"
 
+    def addpartition_sql(self, expression: exp.AddPartition) -> str:
+        exists = "IF NOT EXISTS " if expression.args.get("exists") else ""
+        return f"ADD {exists}{self.sql(expression.this)}"
+
     def distinct_sql(self, expression: exp.Distinct) -> str:
         this = self.expressions(expression, flat=True)
 

sqlglot/optimizer/scope.py

@@ -758,6 +758,8 @@ def _traverse_tables(scope):
             expressions.extend(join.this for join in expression.args.get("joins") or [])
             continue
 
+        child_scope = None
+
         for child_scope in _traverse_scope(
             scope.branch(
                 expression,
@@ -775,8 +777,9 @@ def _traverse_tables(scope):
             sources[expression.alias] = child_scope
 
         # append the final child_scope yielded
-        scopes.append(child_scope)
-        scope.table_scopes.append(child_scope)
+        if child_scope:
+            scopes.append(child_scope)
+            scope.table_scopes.append(child_scope)
 
     scope.sources.update(sources)
 

sqlglot/parser.py

@@ -932,11 +932,14 @@ class Parser(metaclass=_Parser):
 
     PIPE_SYNTAX_TRANSFORM_PARSERS = {
         "SELECT": lambda self, query: self._parse_pipe_syntax_select(query),
-        "WHERE": lambda self, query: self._parse_pipe_syntax_where(query),
-        "ORDER BY": lambda self, query: query.order_by(self._parse_order(), copy=False),
+        "WHERE": lambda self, query: query.where(self._parse_where(), copy=False),
+        "ORDER BY": lambda self, query: query.order_by(
+            self._parse_order(), append=False, copy=False
+        ),
         "LIMIT": lambda self, query: self._parse_pipe_syntax_limit(query),
-        "OFFSET": lambda self, query: query.offset(self._parse_offset(), copy=False),
         "AGGREGATE": lambda self, query: self._parse_pipe_syntax_aggregate(query),
+        "PIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query),
+        "UNPIVOT": lambda self, query: self._parse_pipe_syntax_pivot(query),
     }
 
     PROPERTY_PARSERS: t.Dict[str, t.Callable] = {
@@ -1125,97 +1128,6 @@ class Parser(metaclass=_Parser):
         "TRUNCATE": lambda self: self._parse_partitioned_by_bucket_or_truncate(),
     }
 
-    def _parse_pipe_syntax_select(self, query: exp.Query) -> exp.Query:
-        select = self._parse_select()
-        if isinstance(select, exp.Select):
-            return select.from_(query.subquery(copy=False), copy=False)
-        return query
-
-    def _parse_pipe_syntax_where(self, query: exp.Query) -> exp.Query:
-        where = self._parse_where()
-        return query.where(where, copy=False)
-
-    def _parse_pipe_syntax_limit(self, query: exp.Query) -> exp.Query:
-        limit = self._parse_limit()
-        offset = self._parse_offset()
-        if limit:
-            query.limit(limit, copy=False)
-        if offset:
-            query.offset(offset, copy=False)
-        return query
-
-    def _parse_pipe_syntax_aggregate_fields(self) -> t.Optional[exp.Expression]:
-        this = self._parse_assignment()
-        if self._match_text_seq("GROUP", "AND", advance=False):
-            return this
-
-        this = self._parse_alias(this)
-
-        if self._match_set((TokenType.ASC, TokenType.DESC), advance=False):
-            return self._parse_ordered(lambda: this)
-
-        return this
-
-    def _parse_pipe_syntax_aggregate_group_order_by(
-        self, query: exp.Query, group_by_exists: bool = True
-    ) -> exp.Query:
-        expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields)
-        aggregates_or_groups, orders = [], []
-        for element in expr:
-            if isinstance(element, exp.Ordered):
-                this = element.this
-                if isinstance(this, exp.Alias):
-                    element.set("this", this.args["alias"])
-                orders.append(element)
-            else:
-                this = element
-            aggregates_or_groups.append(this)
-
-        if group_by_exists and isinstance(query, exp.Select):
-            query = query.select(*aggregates_or_groups, copy=False).group_by(
-                *[element.args.get("alias", element) for element in aggregates_or_groups],
-                copy=False,
-            )
-        else:
-            query = exp.select(*aggregates_or_groups, copy=False).from_(
-                query.subquery(copy=False), copy=False
-            )
-
-        if orders:
-            return query.order_by(*orders, copy=False)
-
-        return query
-
-    def _parse_pipe_syntax_aggregate(self, query: exp.Query) -> exp.Query:
-        self._match_text_seq("AGGREGATE")
-        query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False)
-
-        if self._match(TokenType.GROUP_BY) or (
-            self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY)
-        ):
-            return self._parse_pipe_syntax_aggregate_group_order_by(query)
-
-        return query
-
-    def _parse_pipe_syntax_set_operator(
-        self, query: t.Optional[exp.Query]
-    ) -> t.Optional[exp.Query]:
-        first_setop = self.parse_set_operation(this=query)
-
-        if not first_setop or not query:
-            return None
-
-        first_setop.this.pop()
-        distinct = first_setop.args.pop("distinct")
-
-        setops = [first_setop.expression.pop(), *self._parse_expressions()]
-
-        if isinstance(first_setop, exp.Union):
-            return query.union(*setops, distinct=distinct, **first_setop.args)
-        if isinstance(first_setop, exp.Except):
-            return query.except_(*setops, distinct=distinct, **first_setop.args)
-        return query.intersect(*setops, distinct=distinct, **first_setop.args)
-
     def _parse_partitioned_by_bucket_or_truncate(self) -> exp.Expression:
         klass = (
             exp.PartitionedByBucket
@@ -1596,6 +1508,9 @@ class Parser(metaclass=_Parser):
     # Whether the 'AS' keyword is optional in the CTE definition syntax
     OPTIONAL_ALIAS_TOKEN_CTE = True
 
+    # Whether renaming a column with an ALTER statement requires the presence of the COLUMN keyword
+    ALTER_RENAME_REQUIRES_COLUMN = True
+
     __slots__ = (
         "error_level",
         "error_message_context",
@@ -1609,6 +1524,7 @@ class Parser(metaclass=_Parser):
         "_next",
         "_prev",
         "_prev_comments",
+        "_pipe_cte_counter",
     )
 
     # Autofilled
@@ -1639,6 +1555,7 @@ class Parser(metaclass=_Parser):
         self._next = None
         self._prev = None
         self._prev_comments = None
+        self._pipe_cte_counter = 0
 
     def parse(
         self, raw_tokens: t.List[Token], sql: t.Optional[str] = None
@@ -3335,9 +3252,11 @@ class Parser(metaclass=_Parser):
         elif self._match(TokenType.VALUES, advance=False):
             this = self._parse_derived_table_values()
         elif from_:
-            this = exp.select("*").from_(from_.this, copy=False)
             if self._match(TokenType.PIPE_GT, advance=False):
-                return self._parse_pipe_syntax_query(this)
+                return self._parse_pipe_syntax_query(
+                    exp.Select().from_(from_.this, append=False, copy=False)
+                )
+            this = exp.select("*").from_(from_.this, copy=False)
         elif self._match(TokenType.SUMMARIZE):
             table = self._match(TokenType.TABLE)
             this = self._parse_select() or self._parse_string() or self._parse_table()
@@ -5203,6 +5122,14 @@ class Parser(metaclass=_Parser):
             exp.DataTypeParam, this=this, expression=self._parse_var(any_token=True)
         )
 
+    def _parse_user_defined_type(self, identifier: exp.Identifier) -> t.Optional[exp.Expression]:
+        type_name = identifier.name
+
+        while self._match(TokenType.DOT):
+            type_name = f"{type_name}.{self._advance_any() and self._prev.text}"
+
+        return exp.DataType.build(type_name, udt=True)
+
     def _parse_types(
         self, check_func: bool = False, schema: bool = False, allow_identifiers: bool = True
     ) -> t.Optional[exp.Expression]:
@@ -5224,12 +5151,7 @@ class Parser(metaclass=_Parser):
                 if tokens[0].token_type in self.TYPE_TOKENS:
                     self._prev = tokens[0]
                 elif self.dialect.SUPPORTS_USER_DEFINED_TYPES:
-                    type_name = identifier.name
-
-                    while self._match(TokenType.DOT):
-                        type_name = f"{type_name}.{self._advance_any() and self._prev.text}"
-
-                    this = exp.DataType.build(type_name, udt=True)
+                    this = self._parse_user_defined_type(identifier)
                 else:
                     self._retreat(self._index - 1)
                     return None
@@ -5587,18 +5509,12 @@ class Parser(metaclass=_Parser):
             else:
                 field = self._parse_field(any_token=True, anonymous_func=True)
 
+            # Function calls can be qualified, e.g., x.y.FOO()
+            # This converts the final AST to a series of Dots leading to the function call
+            # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules
             if isinstance(field, (exp.Func, exp.Window)) and this:
-                # BQ & snowflake allow function calls like x.y.count(...), SAFE.SUBSTR(...) etc
-                # https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference#function_call_rules
-                this = exp.replace_tree(
-                    this,
-                    lambda n: (
-                        self.expression(exp.Dot, this=n.args.get("table"), expression=n.this)
-                        if n.table
-                        else n.this
-                    )
-                    if isinstance(n, exp.Column)
-                    else n,
+                this = this.transform(
+                    lambda n: n.to_dot(include_dots=False) if isinstance(n, exp.Column) else n
                 )
 
             if op:
@@ -5973,7 +5889,11 @@ class Parser(metaclass=_Parser):
             constraints.append(
                 self.expression(
                     exp.ColumnConstraint,
-                    kind=exp.ComputedColumnConstraint(this=self._parse_disjunction()),
+                    kind=exp.ComputedColumnConstraint(
+                        this=self._parse_disjunction(),
+                        persisted=self._match_texts(("STORED", "VIRTUAL"))
+                        and self._prev.text.upper() == "STORED",
+                    ),
                 )
             )
 
@@ -7236,23 +7156,6 @@ class Parser(metaclass=_Parser):
 
         return this
 
-    def _parse_pipe_syntax_query(self, query: exp.Query) -> t.Optional[exp.Query]:
-        while self._match(TokenType.PIPE_GT):
-            start = self._curr
-            parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(self._curr.text.upper())
-            if not parser:
-                set_op_query = self._parse_pipe_syntax_set_operator(query)
-                if not set_op_query:
-                    self._retreat(start)
-                    self.raise_error(f"Unsupported pipe syntax operator: '{start.text.upper()}'.")
-                    break
-
-                query = set_op_query
-            else:
-                query = parser(self, query)
-
-        return query
-
     def _parse_wrapped_id_vars(self, optional: bool = False) -> t.List[exp.Expression]:
         return self._parse_wrapped_csv(self._parse_id_var, optional=optional)
 
@@ -7331,24 +7234,29 @@ class Parser(metaclass=_Parser):
         self._match(TokenType.TABLE)
         return self.expression(exp.Refresh, this=self._parse_string() or self._parse_table())
 
-    def _parse_add_column(self) -> t.Optional[exp.Expression]:
+    def _parse_add_column(self) -> t.Optional[exp.ColumnDef]:
         if not self._prev.text.upper() == "ADD":
             return None
 
+        start = self._index
         self._match(TokenType.COLUMN)
+
         exists_column = self._parse_exists(not_=True)
         expression = self._parse_field_def()
 
-        if expression:
-            expression.set("exists", exists_column)
+        if not isinstance(expression, exp.ColumnDef):
+            self._retreat(start)
+            return None
 
-            # https://docs.databricks.com/delta/update-schema.html#explicitly-update-schema-to-add-columns
-            if self._match_texts(("FIRST", "AFTER")):
-                position = self._prev.text
-                column_position = self.expression(
-                    exp.ColumnPosition, this=self._parse_column(), position=position
-                )
-                expression.set("position", column_position)
+        expression.set("exists", exists_column)
+
+        # https://docs.databricks.com/delta/update-schema.html#explicitly-update-schema-to-add-columns
+        if self._match_texts(("FIRST", "AFTER")):
+            position = self._prev.text
+            column_position = self.expression(
+                exp.ColumnPosition, this=self._parse_column(), position=position
+            )
+            expression.set("position", column_position)
 
         return expression
 
@@ -7365,13 +7273,24 @@ class Parser(metaclass=_Parser):
         )
 
     def _parse_alter_table_add(self) -> t.List[exp.Expression]:
-        def _parse_add_column_or_constraint():
+        def _parse_add_alteration() -> t.Optional[exp.Expression]:
             self._match_text_seq("ADD")
             if self._match_set(self.ADD_CONSTRAINT_TOKENS, advance=False):
                 return self.expression(
                     exp.AddConstraint, expressions=self._parse_csv(self._parse_constraint)
                 )
-            return self._parse_add_column()
+
+            column_def = self._parse_add_column()
+            if isinstance(column_def, exp.ColumnDef):
+                return column_def
+
+            exists = self._parse_exists(not_=True)
+            if self._match_pair(TokenType.PARTITION, TokenType.L_PAREN, advance=False):
+                return self.expression(
+                    exp.AddPartition, exists=exists, this=self._parse_field(any_token=True)
+                )
+
+            return None
 
         if not self.dialect.ALTER_TABLE_ADD_REQUIRED_FOR_EACH_COLUMN or self._match_text_seq(
             "COLUMNS"
@@ -7380,7 +7299,7 @@ class Parser(metaclass=_Parser):
 
             return ensure_list(schema) if schema else self._parse_csv(self._parse_field_def)
 
-        return self._parse_csv(_parse_add_column_or_constraint)
+        return self._parse_csv(_parse_add_alteration)
 
     def _parse_alter_table_alter(self) -> t.Optional[exp.Expression]:
         if self._match_texts(self.ALTER_ALTER_PARSERS):
@@ -7458,7 +7377,7 @@ class Parser(metaclass=_Parser):
         return self._parse_csv(self._parse_drop_column)
 
     def _parse_alter_table_rename(self) -> t.Optional[exp.AlterRename | exp.RenameColumn]:
-        if self._match(TokenType.COLUMN):
+        if self._match(TokenType.COLUMN) or not self.ALTER_RENAME_REQUIRES_COLUMN:
             exists = self._parse_exists()
             old_column = self._parse_column()
             to = self._match_text_seq("TO")
@@ -8408,3 +8327,160 @@ class Parser(metaclass=_Parser):
         expression = self.expression(exp.Identifier, this=token.text, **kwargs)
         expression.update_positions(token)
         return expression
+
+    def _build_pipe_cte(self, query: exp.Query, expressions: t.List[exp.Expression]) -> exp.Select:
+        if not query.selects:
+            query = query.select("*", copy=False)
+
+        self._pipe_cte_counter += 1
+        new_cte = f"__tmp{self._pipe_cte_counter}"
+
+        with_ = query.args.get("with")
+        ctes = with_.pop() if with_ else None
+
+        new_select = exp.select(*expressions, copy=False).from_(new_cte, copy=False)
+        if ctes:
+            new_select.set("with", ctes)
+
+        return new_select.with_(new_cte, as_=query, copy=False)
+
+    def _parse_pipe_syntax_select(self, query: exp.Select) -> exp.Select:
+        select = self._parse_select()
+        if not select:
+            return query
+
+        if not query.selects:
+            return self._build_pipe_cte(query.select(*select.expressions), [exp.Star()])
+
+        return self._build_pipe_cte(query, select.expressions)
+
+    def _parse_pipe_syntax_limit(self, query: exp.Select) -> exp.Select:
+        limit = self._parse_limit()
+        offset = self._parse_offset()
+        if limit:
+            curr_limit = query.args.get("limit", limit)
+            if curr_limit.expression.to_py() >= limit.expression.to_py():
+                query.limit(limit, copy=False)
+        if offset:
+            curr_offset = query.args.get("offset")
+            curr_offset = curr_offset.expression.to_py() if curr_offset else 0
+            query.offset(exp.Literal.number(curr_offset + offset.expression.to_py()), copy=False)
+
+        return query
+
+    def _parse_pipe_syntax_aggregate_fields(self) -> t.Optional[exp.Expression]:
+        this = self._parse_assignment()
+        if self._match_text_seq("GROUP", "AND", advance=False):
+            return this
+
+        this = self._parse_alias(this)
+
+        if self._match_set((TokenType.ASC, TokenType.DESC), advance=False):
+            return self._parse_ordered(lambda: this)
+
+        return this
+
+    def _parse_pipe_syntax_aggregate_group_order_by(
+        self, query: exp.Select, group_by_exists: bool = True
+    ) -> exp.Select:
+        expr = self._parse_csv(self._parse_pipe_syntax_aggregate_fields)
+        aggregates_or_groups, orders = [], []
+        for element in expr:
+            if isinstance(element, exp.Ordered):
+                this = element.this
+                if isinstance(this, exp.Alias):
+                    element.set("this", this.args["alias"])
+                orders.append(element)
+            else:
+                this = element
+            aggregates_or_groups.append(this)
+
+        if group_by_exists:
+            query = query.select(*aggregates_or_groups, copy=False).group_by(
+                *[projection.args.get("alias", projection) for projection in aggregates_or_groups],
+                copy=False,
+            )
+        else:
+            query = query.select(*aggregates_or_groups, copy=False)
+
+        if orders:
+            return query.order_by(*orders, append=False, copy=False)
+
+        return query
+
+    def _parse_pipe_syntax_aggregate(self, query: exp.Select) -> exp.Select:
+        self._match_text_seq("AGGREGATE")
+        query = self._parse_pipe_syntax_aggregate_group_order_by(query, group_by_exists=False)
+
+        if self._match(TokenType.GROUP_BY) or (
+            self._match_text_seq("GROUP", "AND") and self._match(TokenType.ORDER_BY)
+        ):
+            query = self._parse_pipe_syntax_aggregate_group_order_by(query)
+
+        return self._build_pipe_cte(query, [exp.Star()])
+
+    def _parse_pipe_syntax_set_operator(
+        self, query: t.Optional[exp.Query]
+    ) -> t.Optional[exp.Select]:
+        first_setop = self.parse_set_operation(this=query)
+
+        if not first_setop or not query:
+            return None
+
+        first_setop.this.pop()
+        distinct = first_setop.args.pop("distinct")
+        setops = [first_setop.expression.pop(), *self._parse_expressions()]
+
+        query = self._build_pipe_cte(query, [exp.Star()])
+        with_ = query.args.get("with")
+        ctes = with_.pop() if with_ else None
+
+        if isinstance(first_setop, exp.Union):
+            query = query.union(*setops, distinct=distinct, copy=False, **first_setop.args)
+        elif isinstance(first_setop, exp.Except):
+            query = query.except_(*setops, distinct=distinct, copy=False, **first_setop.args)
+        else:
+            query = query.intersect(*setops, distinct=distinct, copy=False, **first_setop.args)
+
+        query.set("with", ctes)
+
+        return self._build_pipe_cte(query, [exp.Star()])
+
+    def _parse_pipe_syntax_join(self, query: exp.Select) -> t.Optional[exp.Select]:
+        join = self._parse_join()
+        if not join:
+            return None
+
+        return query.join(join, copy=False)
+
+    def _parse_pipe_syntax_pivot(self, query: exp.Select) -> exp.Select:
+        pivots = self._parse_pivots()
+        if not pivots:
+            return query
+
+        from_ = query.args.get("from")
+        if from_:
+            from_.this.set("pivots", pivots)
+
+        return self._build_pipe_cte(query, [exp.Star()])
+
+    def _parse_pipe_syntax_query(self, query: exp.Select) -> t.Optional[exp.Select]:
+        while self._match(TokenType.PIPE_GT):
+            start = self._curr
+            parser = self.PIPE_SYNTAX_TRANSFORM_PARSERS.get(self._curr.text.upper())
+            if not parser:
+                parsed_query = self._parse_pipe_syntax_set_operator(
+                    query
+                ) or self._parse_pipe_syntax_join(query)
+                if not parsed_query:
+                    self._retreat(start)
+                    self.raise_error(f"Unsupported pipe syntax operator: '{start.text.upper()}'.")
+                    break
+                query = parsed_query
+            else:
+                query = parser(self, query)
+
+        if query and not query.selects:
+            return query.select("*", copy=False)
+
+        return query

sqlglot/transforms.py

@@ -842,113 +842,122 @@ def struct_kv_to_alias(expression: exp.Expression) -> exp.Expression:
 
 
 def eliminate_join_marks(expression: exp.Expression) -> exp.Expression:
-    """
-    Remove join marks from an AST. This rule assumes that all marked columns are qualified.
-    If this does not hold for a query, consider running `sqlglot.optimizer.qualify` first.
+    """https://docs.oracle.com/cd/B19306_01/server.102/b14200/queries006.htm#sthref3178
 
-    For example,
-        SELECT * FROM a, b WHERE a.id = b.id(+)    -- ... is converted to
-        SELECT * FROM a LEFT JOIN b ON a.id = b.id -- this
+    1. You cannot specify the (+) operator in a query block that also contains FROM clause join syntax.
 
-    Args:
-        expression: The AST to remove join marks from.
+    2. The (+) operator can appear only in the WHERE clause or, in the context of left-correlation (that is, when specifying the TABLE clause) in the FROM clause, and can be applied only to a column of a table or view.
 
-    Returns:
-       The AST with join marks removed.
+    The (+) operator does not produce an outer join if you specify one table in the outer query and the other table in an inner query.
+
+    You cannot use the (+) operator to outer-join a table to itself, although self joins are valid.
+
+    The (+) operator can be applied only to a column, not to an arbitrary expression. However, an arbitrary expression can contain one or more columns marked with the (+) operator.
+
+    A WHERE condition containing the (+) operator cannot be combined with another condition using the OR logical operator.
+
+    A WHERE condition cannot use the IN comparison condition to compare a column marked with the (+) operator with an expression.
+
+    A WHERE condition cannot compare any column marked with the (+) operator with a subquery.
+
+    -- example with WHERE
+    SELECT d.department_name, sum(e.salary) as total_salary
+    FROM departments d, employees e
+    WHERE e.department_id(+) = d.department_id
+    group by department_name
+
+    -- example of left correlation in select
+    SELECT d.department_name, (
+        SELECT SUM(e.salary)
+            FROM employees e
+            WHERE e.department_id(+) = d.department_id) AS total_salary
+    FROM departments d;
+
+    -- example of left correlation in from
+    SELECT d.department_name, t.total_salary
+    FROM departments d, (
+            SELECT SUM(e.salary) AS total_salary
+            FROM employees e
+            WHERE e.department_id(+) = d.department_id
+        ) t
     """
+
     from sqlglot.optimizer.scope import traverse_scope
+    from sqlglot.optimizer.normalize import normalize, normalized
+    from collections import defaultdict
 
-    for scope in traverse_scope(expression):
+    # we go in reverse to check the main query for left correlation
+    for scope in reversed(traverse_scope(expression)):
         query = scope.expression
 
         where = query.args.get("where")
-        joins = query.args.get("joins")
+        joins = query.args.get("joins", [])
 
-        if not where or not joins:
+        # knockout: we do not support left correlation (see point 2)
+        assert not scope.is_correlated_subquery, "Correlated queries are not supported"
+
+        # nothing to do - we check it here after knockout above
+        if not where or not any(c.args.get("join_mark") for c in where.find_all(exp.Column)):
             continue
 
-        query_from = query.args["from"]
+        # make sure we have AND of ORs to have clear join terms
+        where = normalize(where.this)
+        assert normalized(where), "Cannot normalize JOIN predicates"
 
-        # These keep track of the joins to be replaced
-        new_joins: t.Dict[str, exp.Join] = {}
-        old_joins = {join.alias_or_name: join for join in joins}
+        joins_ons = defaultdict(list)  # dict of {name: list of join AND conditions}
+        for cond in [where] if not isinstance(where, exp.And) else where.flatten():
+            join_cols = [col for col in cond.find_all(exp.Column) if col.args.get("join_mark")]
 
-        for column in scope.columns:
-            if not column.args.get("join_mark"):
+            left_join_table = set(col.table for col in join_cols)
+            if not left_join_table:
                 continue
 
-            predicate = column.find_ancestor(exp.Predicate, exp.Select)
-            assert isinstance(
-                predicate, exp.Binary
-            ), "Columns can only be marked with (+) when involved in a binary operation"
-
-            predicate_parent = predicate.parent
-            join_predicate = predicate.pop()
-
-            left_columns = [
-                c for c in join_predicate.left.find_all(exp.Column) if c.args.get("join_mark")
-            ]
-            right_columns = [
-                c for c in join_predicate.right.find_all(exp.Column) if c.args.get("join_mark")
-            ]
-
             assert not (
-                left_columns and right_columns
-            ), "The (+) marker cannot appear in both sides of a binary predicate"
-
-            marked_column_tables = set()
-            for col in left_columns or right_columns:
-                table = col.table
-                assert table, f"Column {col} needs to be qualified with a table"
+                len(left_join_table) > 1
+            ), "Cannot combine JOIN predicates from different tables"
 
+            for col in join_cols:
                 col.set("join_mark", False)
-                marked_column_tables.add(table)
 
-            assert (
-                len(marked_column_tables) == 1
-            ), "Columns of only a single table can be marked with (+) in a given binary predicate"
-
-            # Add predicate if join already copied, or add join if it is new
-            join_this = old_joins.get(col.table, query_from).this
-            existing_join = new_joins.get(join_this.alias_or_name)
-            if existing_join:
-                existing_join.set("on", exp.and_(existing_join.args["on"], join_predicate))
-            else:
-                new_joins[join_this.alias_or_name] = exp.Join(
-                    this=join_this.copy(), on=join_predicate.copy(), kind="LEFT"
-                )
+            joins_ons[left_join_table.pop()].append(cond)
 
-            # If the parent of the target predicate is a binary node, then it now has only one child
-            if isinstance(predicate_parent, exp.Binary):
-                if predicate_parent.left is None:
-                    predicate_parent.replace(predicate_parent.right)
-                else:
-                    predicate_parent.replace(predicate_parent.left)
+        old_joins = {join.alias_or_name: join for join in joins}
+        new_joins = {}
+        query_from = query.args["from"]
+
+        for table, predicates in joins_ons.items():
+            join_what = old_joins.get(table, query_from).this.copy()
+            new_joins[join_what.alias_or_name] = exp.Join(
+                this=join_what, on=exp.and_(*predicates), kind="LEFT"
+            )
 
-        only_old_join_sources = old_joins.keys() - new_joins.keys()
+            for p in predicates:
+                while isinstance(p.parent, exp.Paren):
+                    p.parent.replace(p)
+
+                parent = p.parent
+                p.pop()
+                if isinstance(parent, exp.Binary):
+                    parent.replace(parent.right if parent.left is None else parent.left)
+                elif isinstance(parent, exp.Where):
+                    parent.pop()
 
         if query_from.alias_or_name in new_joins:
+            only_old_joins = old_joins.keys() - new_joins.keys()
             assert (
-                len(only_old_join_sources) >= 1
+                len(only_old_joins) >= 1
             ), "Cannot determine which table to use in the new FROM clause"
 
-            new_from_name = list(only_old_join_sources)[0]
-            query.set("from", exp.From(this=old_joins.pop(new_from_name).this))
-            only_old_join_sources.remove(new_from_name)
+            new_from_name = list(only_old_joins)[0]
+            query.set("from", exp.From(this=old_joins[new_from_name].this))
 
         if new_joins:
-            only_old_join_expressions = []
-            for old_join_source in only_old_join_sources:
-                old_join_expression = old_joins[old_join_source]
-                if not old_join_expression.kind:
-                    old_join_expression.set("kind", "CROSS")
-
-                only_old_join_expressions.append(old_join_expression)
-
-            query.set("joins", list(new_joins.values()) + only_old_join_expressions)
-
-        if not where.this:
-            where.pop()
+            for n, j in old_joins.items():  # preserve any other joins
+                if n not in new_joins and n != query.args["from"].name:
+                    if not j.kind:
+                        j.set("kind", "CROSS")
+                    new_joins[n] = j
+            query.set("joins", list(new_joins.values()))
 
     return expression
 

{sqlglot-26.26.0.dist-info → sqlglot-26.28.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlglot
-Version: 26.26.0
+Version: 26.28.1
 Summary: An easily customizable SQL parser and transpiler
 Author-email: Toby Mao <toby.mao@gmail.com>
 License: MIT License

{sqlglot-26.26.0.dist-info → sqlglot-26.28.1.dist-info}/RECORD

@@ -1,22 +1,22 @@
 sqlglot/__init__.py,sha256=za08rtdPh2v7dOpGdNomttlIVGgTrKja7rPd6sQwaTg,5391
 sqlglot/__main__.py,sha256=022c173KqxsiABWTEpUIq_tJUxuNiW7a7ABsxBXqvu8,2069
 sqlglot/_typing.py,sha256=-1HPyr3w5COlSJWqlgt8jhFk2dyMvBuvVBqIX1wyVCM,642
-sqlglot/_version.py,sha256=YIRxjNMWWorDaNkCL1dWusQPFAthcmiMLy53HgsVsoQ,515
+sqlglot/_version.py,sha256=lTxpjPlB8VNbv3452Opk2GFByRI5SLtHItxl9sne84Q,515
 sqlglot/diff.py,sha256=PtOllQMQa1Sw1-V2Y8eypmDqGujXYPaTOp_WLsWkAWk,17314
 sqlglot/errors.py,sha256=QNKMr-pzLUDR-tuMmn_GK6iMHUIVdb_YSJ_BhGEvuso,2126
-sqlglot/expressions.py,sha256=vVF2fwwgwo2TWmgpBU4_kYz28xVZ7hhSoaIfpbeyMck,242714
-sqlglot/generator.py,sha256=4JFrruvOtWB1vM470PzTNQC2yGG_7XOiogAjEukuhek,212229
+sqlglot/expressions.py,sha256=oE7OmkFEstTWoPqM7yCls2I2JNyia8Spr-jVi3n77-A,242992
+sqlglot/generator.py,sha256=4iJ0BxkzinmosIhfhb34xjxaFpzw3Zo7fvmknaf5uRs,212432
 sqlglot/helper.py,sha256=9nZjFVRBtMKFC3EdzpDQ6jkazFO19po6BF8xHiNGZIo,15111
 sqlglot/jsonpath.py,sha256=dKdI3PNINNGimmSse2IIv-GbPN_3lXncXh_70QH7Lss,7664
 sqlglot/lineage.py,sha256=kXBDSErmZZluZx_kkrMj4MPEOAbkvcbX1tbOW7Bpl-U,15303
-sqlglot/parser.py,sha256=Qz6C1DARQxi60XGcAicOtUdb9K6RNEH2Fe689s0jrzA,317671
+sqlglot/parser.py,sha256=TksM9cVq6bbbyM0sgglcOb-p6_1_Xk6EPIS2Buj-048,320530
 sqlglot/planner.py,sha256=ql7Li-bWJRcyXzNaZy_n6bQ6B2ZfunEIB8Ztv2xaxq4,14634
 sqlglot/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlglot/schema.py,sha256=13H2qKQs27EKdTpDLOvcNnSTDAUbYNKjWtJs4aQCSOA,20509
 sqlglot/serde.py,sha256=DQVJ95WrIvhYfe02Ytb4NQug2aMwDCEwpMBW1LKDqzE,2031
 sqlglot/time.py,sha256=Q62gv6kL40OiRBF6BMESxKJcMVn7ZLNw7sv8H34z5FI,18400
 sqlglot/tokens.py,sha256=R0B8GQSbQ9GoDc0NlaT5Tc8RjgEOx2IYIkYU5rY8Rg8,48742
-sqlglot/transforms.py,sha256=iTwRPMHTyRx_RG25ItSOnigw_v2tnG9cgwMq0Nwcy2U,39778
+sqlglot/transforms.py,sha256=3jpbHeVTLK9hmQi5f3_vmK-5jZB32_ittCkO7poxCs4,40631
 sqlglot/trie.py,sha256=v27uXMrHfqrXlJ6GmeTSMovsB_3o0ctnlKhdNt7W6fI,2245
 sqlglot/dialects/__init__.py,sha256=aZTLpe2SwgWqiVrRabmfV8TVLPVHFydGwb_zhcVhRss,3499
 sqlglot/dialects/athena.py,sha256=xjy75ej0T3douCUfFKhE1I3kqvPEuQY29x24WG1--Vw,6307
@@ -29,19 +29,19 @@ sqlglot/dialects/drill.py,sha256=FOh7_KjPx_77pv0DiHKZog0CcmzqeF9_PEmGnJ1ESSM,582
 sqlglot/dialects/druid.py,sha256=kh3snZtneehNOWqs3XcPjsrhNaRbkCQ8E4hHbWJ1fHM,690
 sqlglot/dialects/duckdb.py,sha256=alEYXBW5uUApRC8IRYnsapeiJq7JJwUmrK18C56RYsg,47780
 sqlglot/dialects/dune.py,sha256=gALut-fFfN2qMsr8LvZ1NQK3F3W9z2f4PwMvTMXVVVg,375
-sqlglot/dialects/hive.py,sha256=IKAM2elf_n3LgRcPK_4-JuE1j6shd6FhE1QJvBaP55U,31665
+sqlglot/dialects/hive.py,sha256=PO6DLT1kHL-U2kFfV1CsNgQFT7A32LuGN71gnTXEOfY,31728
 sqlglot/dialects/materialize.py,sha256=_DPLPt8YrdQIIXNrGJw1IMcGOoAEJ9NO9X9pDfy4hxs,3494
-sqlglot/dialects/mysql.py,sha256=PnhqX2B15J71WUROefPTc7ZOP0vybbkZGWIDrxYN5Dc,48159
+sqlglot/dialects/mysql.py,sha256=prZecn3zeoifZX7l54UuLG64ar7I-or_z9lF-rT8bds,49233
 sqlglot/dialects/oracle.py,sha256=llxu2LzndrsGyceTod-Leh03vuPWEUKzVHB5gQY-tY8,15313
-sqlglot/dialects/postgres.py,sha256=p3sKOp1ImqsQkSz_BWP8pTTMasPFXDJscnMXccXN_5U,30594
+sqlglot/dialects/postgres.py,sha256=KUyMoLkm1_sZKUbdjn6bjXx9xz7sbEMKa-fl5Mzfrsk,31025
 sqlglot/dialects/presto.py,sha256=ltKbQ44efeq1HM0T8Qq0rsBSx6B6bF9RoKtUBVeoz70,33155
 sqlglot/dialects/prql.py,sha256=OF2LfDb4uzKIF7kpCfpL5G7VP1pnzLbjfW5QFUnuPvo,7803
 sqlglot/dialects/redshift.py,sha256=H8H8lGizHIAd4qLoPeFchyiGZKO1I8U_B058woukuGw,15366
 sqlglot/dialects/risingwave.py,sha256=hwEOPjMw0ZM_3fjQcBUE00oy6I8V6mzYOOYmcwwS8mw,2898
-sqlglot/dialects/snowflake.py,sha256=qbrtxaBrpJNIaBth1kkVe6ep5TZktMN3lcWEp1-m0hs,61467
+sqlglot/dialects/snowflake.py,sha256=m4Gekw4NhoD3q4WF1TJhetRmmwkh8XG9Rqq8mL3P31E,61761
 sqlglot/dialects/spark.py,sha256=fbmiTKAQiKqG9yE_HAxYGgQiOjdxB9tJyjOtgdqF100,7645
 sqlglot/dialects/spark2.py,sha256=8er7nHDm5Wc57m9AOxKN0sd_DVzbhAL44H_udlFh9O8,14258
-sqlglot/dialects/sqlite.py,sha256=ZhVkBzse5yh5pxyYN_daU6CMr4Yint4pIoH3n4gbnns,12394
+sqlglot/dialects/sqlite.py,sha256=UzJwIdY1PsLArMxNt5lKvk8COHvXeo4FoqW41LqVmM8,12440
 sqlglot/dialects/starrocks.py,sha256=fHNgvq5Nz7dI4QUWCTOO5VDOYjasBxRRlcg9TbY0UZE,11235
 sqlglot/dialects/tableau.py,sha256=oIawDzUITxGCWaEMB8OaNMPWhbC3U-2y09pYPm4eazc,2190
 sqlglot/dialects/teradata.py,sha256=xWa-9kSTsT-eM1NePi_oIM1dPHmXW89GLU5Uda3_6Ao,14036
@@ -69,11 +69,11 @@ sqlglot/optimizer/pushdown_projections.py,sha256=7NoK5NAUVYVhs0YnYyo6WuXfaO-BShS
 sqlglot/optimizer/qualify.py,sha256=oAPfwub7dEkrlCrsptcJWpLya4BgKhN6M5SwIs_86LY,4002
 sqlglot/optimizer/qualify_columns.py,sha256=X2Iydssan_Fw84cd-mrzqxG3eRfRdpP6HVRofSbfHlg,40515
 sqlglot/optimizer/qualify_tables.py,sha256=5f5enBAh-bpNB9ewF97W9fx9h1TGXj1Ih5fncvH42sY,6486
-sqlglot/optimizer/scope.py,sha256=Fqz9GpBqO1GWzRAnqdflXXNz44ot_1JqVBC-DnYAU_E,30063
+sqlglot/optimizer/scope.py,sha256=lZWJsR1k-vx1VdxOn0yvbF_LcviXbK357WlrgOLXGEs,30123
 sqlglot/optimizer/simplify.py,sha256=S0Blqg5Mq2KRRWhWz-Eivch9sBjBhg9fRJA6EdBzj2g,50704
 sqlglot/optimizer/unnest_subqueries.py,sha256=kzWUVDlxs8z9nmRx-8U-pHXPtVZhEIwkKqmKhr2QLvc,10908
-sqlglot-26.26.0.dist-info/licenses/LICENSE,sha256=AI3__mHZfOtzY3EluR_pIYBm3_pE7TbVx7qaHxoZ114,1065
-sqlglot-26.26.0.dist-info/METADATA,sha256=7uj_XRkGPWpSUuokcBiin3WhwIBUs8rRcwSyx1zszbY,20732
-sqlglot-26.26.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlglot-26.26.0.dist-info/top_level.txt,sha256=5kRskCGA_gVADF9rSfSzPdLHXqvfMusDYeHePfNY2nQ,8
-sqlglot-26.26.0.dist-info/RECORD,,
+sqlglot-26.28.1.dist-info/licenses/LICENSE,sha256=AI3__mHZfOtzY3EluR_pIYBm3_pE7TbVx7qaHxoZ114,1065
+sqlglot-26.28.1.dist-info/METADATA,sha256=ElrNZkPPdEmAmU1gVJgndWkFCWlhnYqLLkGB4562Bd4,20732
+sqlglot-26.28.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlglot-26.28.1.dist-info/top_level.txt,sha256=5kRskCGA_gVADF9rSfSzPdLHXqvfMusDYeHePfNY2nQ,8
+sqlglot-26.28.1.dist-info/RECORD,,