sqlleaf 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlleaf/__init__.py +4 -0
- sqlleaf/dialects/__init__.py +0 -0
- sqlleaf/dialects/plpgsql/__init__.py +0 -0
- sqlleaf/dialects/plpgsql/p_classes.py +275 -0
- sqlleaf/dialects/plpgsql/p_dialect.py +148 -0
- sqlleaf/dialects/plpgsql/p_generator.py +538 -0
- sqlleaf/dialects/plpgsql/p_parser.py +1244 -0
- sqlleaf/dialects/plpgsql/p_tokenizer.py +41 -0
- sqlleaf/exception.py +67 -0
- sqlleaf/lineage.py +326 -0
- sqlleaf/mappings.py +376 -0
- sqlleaf/models/__init__.py +0 -0
- sqlleaf/models/context.py +153 -0
- sqlleaf/models/hook.py +9 -0
- sqlleaf/models/node/__init__.py +27 -0
- sqlleaf/models/node/base.py +148 -0
- sqlleaf/models/node/column.py +345 -0
- sqlleaf/models/node/dynamodb.py +39 -0
- sqlleaf/models/node/function.py +33 -0
- sqlleaf/models/node/interval.py +21 -0
- sqlleaf/models/node/json.py +57 -0
- sqlleaf/models/node/literal.py +17 -0
- sqlleaf/models/node/null.py +14 -0
- sqlleaf/models/node/pivot.py +34 -0
- sqlleaf/models/node/program.py +27 -0
- sqlleaf/models/node/sequence.py +18 -0
- sqlleaf/models/node/star.py +11 -0
- sqlleaf/models/node/stream.py +14 -0
- sqlleaf/models/node/user_defined_function.py +33 -0
- sqlleaf/models/node/var.py +7 -0
- sqlleaf/models/node/variable.py +8 -0
- sqlleaf/models/node/window.py +22 -0
- sqlleaf/models/query/__init__.py +33 -0
- sqlleaf/models/query/base.py +248 -0
- sqlleaf/models/query/call.py +111 -0
- sqlleaf/models/query/copy.py +144 -0
- sqlleaf/models/query/ctas.py +61 -0
- sqlleaf/models/query/database.py +48 -0
- sqlleaf/models/query/delete.py +26 -0
- sqlleaf/models/query/execute.py +66 -0
- sqlleaf/models/query/holder.py +49 -0
- sqlleaf/models/query/insert.py +69 -0
- sqlleaf/models/query/merge.py +29 -0
- sqlleaf/models/query/multitable_insert.py +42 -0
- sqlleaf/models/query/prepare.py +61 -0
- sqlleaf/models/query/procedure.py +72 -0
- sqlleaf/models/query/put.py +38 -0
- sqlleaf/models/query/replace.py +32 -0
- sqlleaf/models/query/schema.py +47 -0
- sqlleaf/models/query/select.py +27 -0
- sqlleaf/models/query/sequence.py +33 -0
- sqlleaf/models/query/set.py +74 -0
- sqlleaf/models/query/stage.py +78 -0
- sqlleaf/models/query/table.py +220 -0
- sqlleaf/models/query/trigger.py +42 -0
- sqlleaf/models/query/type.py +42 -0
- sqlleaf/models/query/unload.py +64 -0
- sqlleaf/models/query/update.py +45 -0
- sqlleaf/models/query/user_defined_function.py +297 -0
- sqlleaf/models/query/values.py +20 -0
- sqlleaf/models/query/view.py +41 -0
- sqlleaf/path.py +194 -0
- sqlleaf/processors/__init__.py +0 -0
- sqlleaf/processors/collector/__init__.py +0 -0
- sqlleaf/processors/collector/collector.py +827 -0
- sqlleaf/processors/collector/substitute.py +396 -0
- sqlleaf/processors/generator/__init__.py +0 -0
- sqlleaf/processors/generator/dialects/__init__.py +5 -0
- sqlleaf/processors/generator/dialects/athena.py +45 -0
- sqlleaf/processors/generator/dialects/base.py +572 -0
- sqlleaf/processors/generator/dialects/mysql.py +19 -0
- sqlleaf/processors/generator/dialects/postgres.py +182 -0
- sqlleaf/processors/generator/dialects/redshift.py +192 -0
- sqlleaf/processors/generator/dialects/snowflake.py +90 -0
- sqlleaf/processors/generator/generator.py +495 -0
- sqlleaf/processors/transformer/__init__.py +31 -0
- sqlleaf/processors/transformer/base.py +899 -0
- sqlleaf/processors/transformer/call.py +9 -0
- sqlleaf/processors/transformer/copy.py +63 -0
- sqlleaf/processors/transformer/ctas.py +39 -0
- sqlleaf/processors/transformer/delete.py +14 -0
- sqlleaf/processors/transformer/execute.py +9 -0
- sqlleaf/processors/transformer/expressions/__init__.py +11 -0
- sqlleaf/processors/transformer/expressions/functional_notation.py +22 -0
- sqlleaf/processors/transformer/expressions/row.py +451 -0
- sqlleaf/processors/transformer/expressions/values.py +318 -0
- sqlleaf/processors/transformer/insert.py +125 -0
- sqlleaf/processors/transformer/merge.py +14 -0
- sqlleaf/processors/transformer/multitable_insert.py +14 -0
- sqlleaf/processors/transformer/replace.py +23 -0
- sqlleaf/processors/transformer/transformer.py +141 -0
- sqlleaf/processors/transformer/udf.py +235 -0
- sqlleaf/processors/transformer/unload.py +30 -0
- sqlleaf/processors/transformer/update.py +174 -0
- sqlleaf/processors/transformer/values.py +10 -0
- sqlleaf/settings/__init__.py +19 -0
- sqlleaf/settings/athena.py +23 -0
- sqlleaf/settings/base.py +37 -0
- sqlleaf/settings/postgres.py +52 -0
- sqlleaf/system_tables/load.py +0 -0
- sqlleaf/typing.py +93 -0
- sqlleaf/util/__init__.py +34 -0
- sqlleaf/util/expression.py +282 -0
- sqlleaf/util/graph.py +116 -0
- sqlleaf/util/helpers.py +42 -0
- sqlleaf/util/iterators.py +37 -0
- sqlleaf-0.0.1.dist-info/METADATA +46 -0
- sqlleaf-0.0.1.dist-info/RECORD +111 -0
- sqlleaf-0.0.1.dist-info/WHEEL +5 -0
- sqlleaf-0.0.1.dist-info/licenses/LICENSE +21 -0
- sqlleaf-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import typing as t
|
|
3
|
+
|
|
4
|
+
from sqlglot import exp
|
|
5
|
+
|
|
6
|
+
from sqlleaf import mappings, util
|
|
7
|
+
from sqlleaf.models.query import UserDefinedFunctionQuery
|
|
8
|
+
from sqlleaf.processors.collector import substitute
|
|
9
|
+
from sqlleaf.processors.transformer.expressions import normalize_values
|
|
10
|
+
from sqlleaf.processors.transformer.expressions.row import transform_row_function_to_subquery
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger("sqlleaf")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def find_next_udf_call(
|
|
19
|
+
expression: exp.Expr, object_mapping: mappings.ObjectMapping
|
|
20
|
+
) -> t.Tuple[t.Optional[exp.Anonymous], t.Optional[UserDefinedFunctionQuery]]:
|
|
21
|
+
"""
|
|
22
|
+
Searches the AST for the next UDF call that matches any of the provided UDF definitions.
|
|
23
|
+
Returns the call node and the matched UDF definition.
|
|
24
|
+
"""
|
|
25
|
+
for node in expression.find_all(exp.Anonymous):
|
|
26
|
+
best_match = object_mapping.lookup_udf_call(node)
|
|
27
|
+
if best_match:
|
|
28
|
+
return node, best_match
|
|
29
|
+
|
|
30
|
+
return None, None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _get_alias(node: exp.Expr) -> t.Optional[str]:
|
|
34
|
+
"""Helper to extract an alias from a node or its parent."""
|
|
35
|
+
alias = node.alias
|
|
36
|
+
if not alias:
|
|
37
|
+
if isinstance(node.parent, exp.Table):
|
|
38
|
+
alias = node.parent.alias
|
|
39
|
+
|
|
40
|
+
if not alias and hasattr(node, "this"):
|
|
41
|
+
alias = node.this
|
|
42
|
+
|
|
43
|
+
return alias
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def create_subquery_with_alias(
|
|
47
|
+
replacement_expr: exp.Expr, query: UserDefinedFunctionQuery, alias: str = "t"
|
|
48
|
+
) -> exp.Subquery:
|
|
49
|
+
"""Creates a Subquery with a table alias and the UDF's return columns."""
|
|
50
|
+
return exp.Subquery(
|
|
51
|
+
this=replacement_expr,
|
|
52
|
+
alias=exp.TableAlias(
|
|
53
|
+
this=exp.Identifier(this=alias, quoted=False),
|
|
54
|
+
columns=[c.this if isinstance(c, exp.ColumnDef) else exp.to_identifier(c) for c in query.return_columns],
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def transform_udf_to_subquery_if_table_reference(
|
|
60
|
+
node: exp.Anonymous, replacement_expr: exp.Expr, query: UserDefinedFunctionQuery
|
|
61
|
+
) -> None:
|
|
62
|
+
"""
|
|
63
|
+
Transforms a UDF call into a subquery when used in a FROM clause as a table.
|
|
64
|
+
|
|
65
|
+
Example:
|
|
66
|
+
SELECT FROM hello() -> SELECT FROM (SELECT ...) AS t(name, age)
|
|
67
|
+
where hello() returns (name, age)
|
|
68
|
+
"""
|
|
69
|
+
if query.return_columns:
|
|
70
|
+
alias = _get_alias(node) or "t"
|
|
71
|
+
replacement_expr = create_subquery_with_alias(replacement_expr, query, alias=alias)
|
|
72
|
+
|
|
73
|
+
node.parent.replace(replacement_expr)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def replace_scalar_call(target_node: exp.Expr, replacement_expr: exp.Expr) -> None:
|
|
77
|
+
"""
|
|
78
|
+
Replaces a scalar UDF call, wrapping query-like expressions in a Subquery.
|
|
79
|
+
|
|
80
|
+
Example:
|
|
81
|
+
Given hello() as `SELECT 1`,
|
|
82
|
+
Before: `SELECT hello()`
|
|
83
|
+
After: `SELECT (SELECT 1)`
|
|
84
|
+
"""
|
|
85
|
+
if isinstance(replacement_expr, (exp.Select, exp.Values)):
|
|
86
|
+
replacement_expr = exp.Subquery(this=replacement_expr)
|
|
87
|
+
copied = replacement_expr.copy()
|
|
88
|
+
target_node.replace(copied)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def get_target_node(node: exp.Anonymous) -> exp.Expr:
|
|
92
|
+
"""
|
|
93
|
+
Returns the node to be replaced.
|
|
94
|
+
"""
|
|
95
|
+
parent = node.parent
|
|
96
|
+
# Qualified with a schema (e.g., myschema.myfunc())
|
|
97
|
+
if isinstance(parent, exp.Dot) and isinstance(parent.left, exp.Identifier):
|
|
98
|
+
return parent
|
|
99
|
+
return node
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def resolve_returning_to_select(
|
|
103
|
+
stmt: exp.Expr,
|
|
104
|
+
param_map: t.Dict[str, exp.Expr],
|
|
105
|
+
query: UserDefinedFunctionQuery,
|
|
106
|
+
positional_map: t.Dict[str, exp.Expr],
|
|
107
|
+
) -> exp.Select:
|
|
108
|
+
"""
|
|
109
|
+
Given a statement with a RETURNING clause (INSERT, UPDATE, DELETE, MERGE),
|
|
110
|
+
resolve each RETURNING expression and return a SELECT expression.
|
|
111
|
+
|
|
112
|
+
Notes on INSERT ... VALUES ... RETURNING:
|
|
113
|
+
- We intentionally resolve INSERT ... VALUES ... RETURNING by selecting the
|
|
114
|
+
requested columns from the target table rather than from the VALUES-derived
|
|
115
|
+
UNION. This preserves the semantics that RETURNING reflects the post-insert
|
|
116
|
+
target-row value.
|
|
117
|
+
|
|
118
|
+
Example:
|
|
119
|
+
INSERT INTO people (age) VALUES (5), (2) RETURNING age
|
|
120
|
+
→ SELECT people.age FROM people
|
|
121
|
+
"""
|
|
122
|
+
returning_node = stmt.args.get("returning")
|
|
123
|
+
if isinstance(stmt, exp.Insert) and isinstance(stmt.expression, exp.Values):
|
|
124
|
+
# TODO: this shouldn't be the select expr, but just direct lineage from the target column to the returning column
|
|
125
|
+
# For INSERT ... VALUES ... RETURNING, select the RETURNING columns from the target table.
|
|
126
|
+
table = stmt.find(exp.Table)
|
|
127
|
+
returning_select = exp.select(*[r.copy() for r in returning_node.expressions]).from_(table.copy())
|
|
128
|
+
return substitute.substitute_parameters(returning_select, query, param_map, positional_map)
|
|
129
|
+
|
|
130
|
+
col_to_val = {}
|
|
131
|
+
resolved = []
|
|
132
|
+
for ret_col in returning_node.expressions:
|
|
133
|
+
col_name = ret_col.name if isinstance(ret_col, exp.Column) else ret_col.alias_or_name
|
|
134
|
+
resolved_val = col_to_val.get(col_name, ret_col).copy()
|
|
135
|
+
# Apply parameter substitution on the resolved value
|
|
136
|
+
resolved_val = substitute.substitute_parameters(resolved_val, query, param_map, positional_map)
|
|
137
|
+
resolved.append(resolved_val)
|
|
138
|
+
|
|
139
|
+
return exp.select(*resolved)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def transform_inner_query(
|
|
143
|
+
stmt: exp.Expr,
|
|
144
|
+
param_map: t.Dict[str, exp.Expr],
|
|
145
|
+
query: UserDefinedFunctionQuery,
|
|
146
|
+
positional_map: t.Dict[str, exp.Expr],
|
|
147
|
+
) -> exp.Expr:
|
|
148
|
+
"""
|
|
149
|
+
Performs replacement and transformations over a UDF's inner query.
|
|
150
|
+
|
|
151
|
+
Example:
|
|
152
|
+
Given a UDF body statement `SELECT $1 * 2 AS val` with param_map `{"$1": 5}`:
|
|
153
|
+
→ SELECT 5 * 2 AS val
|
|
154
|
+
|
|
155
|
+
Given a UDF body statement `INSERT INTO people (age) VALUES (5) RETURNING age`:
|
|
156
|
+
→ SELECT people.age FROM people
|
|
157
|
+
"""
|
|
158
|
+
# handle INSERT/UPDATE/DELETE/MERGE ... RETURNING inside a UDF body
|
|
159
|
+
if isinstance(stmt, (exp.Insert, exp.Update, exp.Delete, exp.Merge)) and stmt.args.get("returning"):
|
|
160
|
+
return resolve_returning_to_select(stmt, param_map, query, positional_map)
|
|
161
|
+
|
|
162
|
+
if isinstance(stmt, exp.Values):
|
|
163
|
+
stmt = normalize_values(query, stmt)
|
|
164
|
+
|
|
165
|
+
logger.debug(f"Transforming inner query: {stmt.sql()}")
|
|
166
|
+
replacement_expr = substitute.substitute_parameters(stmt, query, param_map, positional_map)
|
|
167
|
+
logger.debug(f"Query after parameter substitution: {replacement_expr.sql()}")
|
|
168
|
+
|
|
169
|
+
# Handle ROW(...)::table_name replacement (composite/table type casting)
|
|
170
|
+
replacement_expr = transform_row_function_to_subquery(replacement_expr, query)
|
|
171
|
+
|
|
172
|
+
new_expr = replacement_expr
|
|
173
|
+
return new_expr
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def build_replacement_exprs(node: exp.Anonymous, query: UserDefinedFunctionQuery) -> t.List[exp.Expr]:
|
|
177
|
+
"""
|
|
178
|
+
Builds the expressions that will replace a UDF call.
|
|
179
|
+
|
|
180
|
+
Example:
|
|
181
|
+
Given a UDF `hello()` defined as `SELECT 'Hello'` and a call site `hello()`:
|
|
182
|
+
→ [SELECT 'Greetings!' AS greeting]
|
|
183
|
+
|
|
184
|
+
Given a UDF `hello()` with a RETURNING body `INSERT INTO people (age) VALUES (5) RETURNING age`:
|
|
185
|
+
→ [SELECT people.age FROM people]
|
|
186
|
+
"""
|
|
187
|
+
if query.return_type and query.return_type.this == exp.DataType.Type.NULL:
|
|
188
|
+
return [exp.select(exp.Null())]
|
|
189
|
+
|
|
190
|
+
param_map, positional_map = substitute.transform_arguments(node, query)
|
|
191
|
+
|
|
192
|
+
replacement_exprs = []
|
|
193
|
+
# Collect all child queries from the holder
|
|
194
|
+
for child in query.holder.downstream_holders:
|
|
195
|
+
stmt = child.original.statement
|
|
196
|
+
stmt = util.copy_expression(stmt)
|
|
197
|
+
replacement_exprs.append(transform_inner_query(stmt, param_map, query, positional_map))
|
|
198
|
+
|
|
199
|
+
return replacement_exprs
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def substitute_udf(
|
|
203
|
+
node: exp.Anonymous,
|
|
204
|
+
query: UserDefinedFunctionQuery,
|
|
205
|
+
) -> t.List[exp.Expr]:
|
|
206
|
+
"""
|
|
207
|
+
Returns the UDF's inner body with arguments substituted.
|
|
208
|
+
Does NOT modify the caller query.
|
|
209
|
+
Mirrors the contract of substitute_call / substitute_execute_with_plan.
|
|
210
|
+
"""
|
|
211
|
+
return build_replacement_exprs(node, query)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def apply_replacement(target_node: exp.Expr, replacement_expr: exp.Expr, query: UserDefinedFunctionQuery) -> None:
|
|
215
|
+
"""
|
|
216
|
+
Applies the replacement of a UDF call node with its body logic.
|
|
217
|
+
|
|
218
|
+
Dispatches to the appropriate replacement strategy based on the call context:
|
|
219
|
+
- LATERAL context: wraps the replacement in a LATERAL subquery.
|
|
220
|
+
- Table reference context: wraps the replacement as a derived table.
|
|
221
|
+
- Field access (dot notation): replaces with a field-access subquery.
|
|
222
|
+
- Scalar call: replaces the call node inline.
|
|
223
|
+
|
|
224
|
+
Example:
|
|
225
|
+
Given `SELECT hello()` where `hello()` resolves to `SELECT 'Greetings!' AS greeting`:
|
|
226
|
+
→ SELECT (SELECT 'Greetings!' AS greeting) AS hello
|
|
227
|
+
"""
|
|
228
|
+
logger.debug(f"Applying replacement for UDF: {query.name}")
|
|
229
|
+
|
|
230
|
+
if isinstance(target_node.parent, exp.Table):
|
|
231
|
+
logger.debug("Applying Table reference replacement")
|
|
232
|
+
transform_udf_to_subquery_if_table_reference(target_node, replacement_expr, query)
|
|
233
|
+
else:
|
|
234
|
+
logger.debug("Applying Scalar call replacement")
|
|
235
|
+
replace_scalar_call(target_node, replacement_expr)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
UnloadTransformer — handles UNLOAD statement transformations.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from sqlglot import exp
|
|
6
|
+
|
|
7
|
+
from sqlleaf.processors.transformer.base import BaseQueryTransformer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class UnloadTransformer(BaseQueryTransformer):
|
|
11
|
+
"""Transformer for UNLOAD statements."""
|
|
12
|
+
|
|
13
|
+
def transform(self, statement: exp.Insert) -> exp.Insert:
|
|
14
|
+
return self._convert_unload_to_insert(statement)
|
|
15
|
+
|
|
16
|
+
def _convert_unload_to_insert(self, statement: exp.Select) -> exp.Insert:
|
|
17
|
+
"""
|
|
18
|
+
Convert the UNLOAD statement into an INSERT statement.
|
|
19
|
+
|
|
20
|
+
UNLOAD ('SELECT * FROM fruit.raw') TO 's3://object-path/name-prefix'
|
|
21
|
+
-> INSERT INTO 's3://object-path/name-prefix' SELECT * FROM fruit.raw
|
|
22
|
+
"""
|
|
23
|
+
query = self.query
|
|
24
|
+
table = exp.table_(query.target_info.expression.name)
|
|
25
|
+
insert_expr = exp.insert(
|
|
26
|
+
expression=statement,
|
|
27
|
+
into=table,
|
|
28
|
+
dialect=query.dialect,
|
|
29
|
+
)
|
|
30
|
+
return insert_expr
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""
|
|
2
|
+
UpdateTransformer — handles UPDATE (and MERGE → UPDATE, ON CONFLICT) statement transformations.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from sqlglot import exp
|
|
6
|
+
|
|
7
|
+
from sqlleaf import exception
|
|
8
|
+
from sqlleaf.processors.transformer.base import BaseQueryTransformer
|
|
9
|
+
from sqlleaf.processors.transformer.expressions import normalize_values
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UpdateTransformer(BaseQueryTransformer):
|
|
13
|
+
"""Transformer for UPDATE statements."""
|
|
14
|
+
|
|
15
|
+
def transform(self, statement: exp.Insert) -> exp.Insert:
|
|
16
|
+
statement = self._convert_on_conflict_to_update(statement)
|
|
17
|
+
if isinstance(statement, (exp.Insert, exp.Update)):
|
|
18
|
+
statement = self._add_information_from_merge(statement)
|
|
19
|
+
if isinstance(statement, exp.Update):
|
|
20
|
+
statement = self._convert_update_defaults_to_values(statement)
|
|
21
|
+
statement = self._convert_update_to_insert(statement)
|
|
22
|
+
if isinstance(statement, (exp.Insert, exp.Merge, exp.Update, exp.Delete)):
|
|
23
|
+
statement = self._process_inner_ctes(statement)
|
|
24
|
+
return statement
|
|
25
|
+
|
|
26
|
+
def _convert_on_conflict_to_update(self, statement: exp.OnConflict | exp.Update) -> exp.Update:
|
|
27
|
+
"""
|
|
28
|
+
Convert the 'DO UPDATE' in:
|
|
29
|
+
INSERT INTO <table> as t
|
|
30
|
+
ON CONFLICT
|
|
31
|
+
DO UPDATE SET name = EXCLUDED.name
|
|
32
|
+
to:
|
|
33
|
+
UPDATE <table> AS t
|
|
34
|
+
SET name = name
|
|
35
|
+
so that the expression has the correct columns/values.
|
|
36
|
+
"""
|
|
37
|
+
if not isinstance(statement, exp.OnConflict):
|
|
38
|
+
return statement
|
|
39
|
+
|
|
40
|
+
# Note: this holder's `preprocess()`/`normalize_all_values()` only walks
|
|
41
|
+
# descendants of `statement` (the OnConflict node itself), so it never sees the
|
|
42
|
+
# sibling VALUES living under the ancestor Insert/Create's `.expression` in this
|
|
43
|
+
# holder's own copy of the tree. That conversion must still happen here.
|
|
44
|
+
parent_insert_expr = None
|
|
45
|
+
values_alias_name: str | None = None
|
|
46
|
+
parent = statement.parent
|
|
47
|
+
parent_expr = statement.parent.expression
|
|
48
|
+
|
|
49
|
+
update_expr = exp.update(table=self.query.get_target_as_table())
|
|
50
|
+
parent_table = parent_expr.args.get("from_", None)
|
|
51
|
+
if parent_table:
|
|
52
|
+
update_expr = update_expr.from_(parent_table.this)
|
|
53
|
+
|
|
54
|
+
if isinstance(parent_expr, exp.Values):
|
|
55
|
+
# Capture the VALUES alias (used in MySQL: VALUES (...) AS <alias>) before we rewrite it
|
|
56
|
+
alias_expr = parent_expr.args.get("alias")
|
|
57
|
+
|
|
58
|
+
if alias_expr is not None:
|
|
59
|
+
# exp.TableAlias typically exposes the alias via `.alias`; fallback to `.name` if needed
|
|
60
|
+
values_alias_name = alias_expr.alias_or_name
|
|
61
|
+
|
|
62
|
+
parent_insert_expr = normalize_values(self.query, parent)
|
|
63
|
+
statement = parent_insert_expr.args["conflict"]
|
|
64
|
+
|
|
65
|
+
elif isinstance(parent_expr, exp.Select):
|
|
66
|
+
parent_insert_expr = parent
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# Rewrite the expressions in the UPDATE
|
|
70
|
+
for eq_expr in list(statement.expressions):
|
|
71
|
+
right_expr = eq_expr.right
|
|
72
|
+
|
|
73
|
+
if self.query.dialect == "mysql" and isinstance(right_expr, exp.Anonymous) and right_expr.name.upper() == "VALUES":
|
|
74
|
+
# Replace the VALUES() expression with its associated expression in the SELECT.
|
|
75
|
+
# The inner column is a reference to the SELECT expression in the INSERT list, not the column itself.
|
|
76
|
+
# e.g. INSERT INTO users (id, name) SELECT 'a', 'b' ... UPDATE SET name = VALUES(name)
|
|
77
|
+
# would replace VALUES(name) with 'b'
|
|
78
|
+
values_col = right_expr.expressions[0].name
|
|
79
|
+
column_names = self._extract_insert_columns(parent, self.query.target_info.expression, include_system=False)
|
|
80
|
+
if values_col not in column_names:
|
|
81
|
+
raise exception.MappingError(f"Column '{values_col}' does not exist in the expression list or the columns for table '{str(parent.this.this)}'")
|
|
82
|
+
|
|
83
|
+
column_index = column_names.index(values_col)
|
|
84
|
+
select_expr = parent_insert_expr.selects[column_index]
|
|
85
|
+
right_expr.replace(select_expr.copy())
|
|
86
|
+
|
|
87
|
+
for col in right_expr.find_all(exp.Column):
|
|
88
|
+
# Transform any aliased columns into their correct expressions.
|
|
89
|
+
# Examples:
|
|
90
|
+
# - ON CONFLICT DO UPDATE ... EXCLUDED.col
|
|
91
|
+
# - ON DUPLICATE KEY UPDATE ... new_alias.col
|
|
92
|
+
table_name = (col.table or "").upper()
|
|
93
|
+
if table_name == "EXCLUDED" or (values_alias_name and col.table == values_alias_name):
|
|
94
|
+
if col.name not in parent_insert_expr.named_selects:
|
|
95
|
+
# The outer INSERT did not provide a value for this column; reference the existing column
|
|
96
|
+
col.replace(exp.column(col.name))
|
|
97
|
+
else:
|
|
98
|
+
# Set it to the unaliased expression from the outer INSERT/SELECT
|
|
99
|
+
select_expr = [
|
|
100
|
+
alias_expr for alias_expr in parent_insert_expr.selects if alias_expr.alias == col.name
|
|
101
|
+
][0]
|
|
102
|
+
new_expr = select_expr.unalias().copy()
|
|
103
|
+
|
|
104
|
+
if isinstance(new_expr, exp.Column) and parent_table:
|
|
105
|
+
new_expr.set("table", exp.to_identifier(parent_table.alias_or_name))
|
|
106
|
+
|
|
107
|
+
col.replace(new_expr)
|
|
108
|
+
|
|
109
|
+
update_expr.set("expressions", statement.expressions)
|
|
110
|
+
|
|
111
|
+
return update_expr
|
|
112
|
+
|
|
113
|
+
def _convert_update_defaults_to_values(self, statement: exp.Update) -> exp.Update:
|
|
114
|
+
"""
|
|
115
|
+
Transform the query:
|
|
116
|
+
UPDATE x SET a = DEFAULT
|
|
117
|
+
into:
|
|
118
|
+
UPDATE x SET a = 42
|
|
119
|
+
according to the table's default column values.
|
|
120
|
+
"""
|
|
121
|
+
query = self.query
|
|
122
|
+
child_table = query.get_target_as_table()
|
|
123
|
+
table_query = query.object_mapping.lookup_table_query(table=child_table)
|
|
124
|
+
if not table_query:
|
|
125
|
+
return statement
|
|
126
|
+
|
|
127
|
+
table_columns = table_query.get_column_defs()
|
|
128
|
+
|
|
129
|
+
for expr in statement.expressions:
|
|
130
|
+
if isinstance(expr, exp.EQ) and isinstance(expr.left, exp.Column):
|
|
131
|
+
if (isinstance(expr.right, exp.Var) and expr.right.name.upper() == "DEFAULT") or (
|
|
132
|
+
isinstance(expr.right, exp.Column) and expr.right.name.upper() == "DEFAULT"
|
|
133
|
+
):
|
|
134
|
+
# Replace 'DEFAULT' with the associated column's default expression
|
|
135
|
+
self._replace_default_with_value(
|
|
136
|
+
expression=expr.right,
|
|
137
|
+
column_name=expr.left.name,
|
|
138
|
+
table_columns=table_columns,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
return statement
|
|
142
|
+
|
|
143
|
+
def _add_information_from_merge(self, statement: exp.Insert | exp.Update) -> exp.Insert | exp.Update:
|
|
144
|
+
"""
|
|
145
|
+
UPDATE branch of _add_information_from_merge.
|
|
146
|
+
|
|
147
|
+
Transform a nested UPDATE (from a MERGE WHEN MATCHED branch) into a
|
|
148
|
+
fully qualified UPDATE so it can be processed independently.
|
|
149
|
+
"""
|
|
150
|
+
if not isinstance(statement, exp.Update):
|
|
151
|
+
return statement
|
|
152
|
+
|
|
153
|
+
ctx = self._extract_merge_context(statement)
|
|
154
|
+
if ctx is None:
|
|
155
|
+
return statement
|
|
156
|
+
|
|
157
|
+
using = ctx["using"]
|
|
158
|
+
on = ctx["on"]
|
|
159
|
+
returning = ctx["returning"]
|
|
160
|
+
new_ctes = [{"alias": cte.alias_or_name, "as_": cte.this} for cte in ctx["ctes"]]
|
|
161
|
+
|
|
162
|
+
# Add the missing information to the UPDATE statement
|
|
163
|
+
target = self.query.target_info.expression
|
|
164
|
+
if isinstance(target, exp.Table):
|
|
165
|
+
self.query.only = target.args.get("only", False) # type: ignore
|
|
166
|
+
|
|
167
|
+
update_expr = statement.table(self.query.target_info.expression).from_(using).where(on)
|
|
168
|
+
update_expr.set("returning", returning)
|
|
169
|
+
|
|
170
|
+
for cte in new_ctes:
|
|
171
|
+
update_expr = update_expr.with_(alias=cte["alias"], as_=cte["as_"])
|
|
172
|
+
|
|
173
|
+
statement.replace(update_expr)
|
|
174
|
+
return update_expr
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from sqlglot import exp
|
|
2
|
+
|
|
3
|
+
from sqlleaf.processors.transformer.base import BaseQueryTransformer
|
|
4
|
+
from sqlleaf.processors.transformer.expressions import normalize_values
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ValuesTransformer(BaseQueryTransformer):
|
|
8
|
+
def transform(self, statement: exp.Values) -> exp.Values:
|
|
9
|
+
statement = normalize_values(self.query, statement)
|
|
10
|
+
return statement
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
from functools import cache
|
|
3
|
+
|
|
4
|
+
from sqlglot import exp
|
|
5
|
+
|
|
6
|
+
from sqlleaf.settings.athena import AthenaSettings as AthenaSettings
|
|
7
|
+
from sqlleaf.settings.base import BaseDialectSettings as BaseDialectSettings
|
|
8
|
+
from sqlleaf.settings.postgres import PostgresSettings as PostgresSettings
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@cache
|
|
12
|
+
def system_columns(dialect: str) -> t.List[exp.ColumnDef]:
|
|
13
|
+
return BaseDialectSettings.from_dialect(dialect).system_columns
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@cache
|
|
17
|
+
def system_functions(dialect: str) -> t.Dict[str, t.List[str]]:
|
|
18
|
+
# TODO: refactor this into a set of objects that can be loaded into the mapping
|
|
19
|
+
return BaseDialectSettings.from_dialect(dialect).system_functions
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
|
|
3
|
+
from sqlglot import exp
|
|
4
|
+
|
|
5
|
+
from sqlleaf.settings.base import BaseDialectSettings
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AthenaSettings(BaseDialectSettings):
|
|
9
|
+
DIALECT = "athena"
|
|
10
|
+
PSEUDOCOLUMNS = {
|
|
11
|
+
"$bucket": "BIGINT",
|
|
12
|
+
"$file_modified_time": "TIMESTAMP",
|
|
13
|
+
"$file_size": "BIGINT",
|
|
14
|
+
"$partition": "VARCHAR",
|
|
15
|
+
"$path": "VARCHAR",
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def system_columns(self) -> t.List[exp.ColumnDef]:
|
|
20
|
+
return [
|
|
21
|
+
exp.ColumnDef(this=exp.to_identifier(name), kind=exp.DataType.build(kind, self.DIALECT))
|
|
22
|
+
for name, kind in self.PSEUDOCOLUMNS.items()
|
|
23
|
+
]
|
sqlleaf/settings/base.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
|
|
3
|
+
from sqlglot import exp
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseDialectSettings:
|
|
7
|
+
# A registry to store subclasses
|
|
8
|
+
_dialects = {}
|
|
9
|
+
DIALECT = ""
|
|
10
|
+
|
|
11
|
+
def __init_subclass__(cls, **kwargs):
|
|
12
|
+
"""Automatically registers subclasses when they are defined."""
|
|
13
|
+
super().__init_subclass__(**kwargs)
|
|
14
|
+
if cls.DIALECT:
|
|
15
|
+
BaseDialectSettings._dialects[cls.DIALECT] = cls
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def from_dialect(cls, dialect: str) -> BaseDialectSettings:
|
|
19
|
+
"""Instantiates a class from the registry by name."""
|
|
20
|
+
target_class = cls._dialects.get(dialect)
|
|
21
|
+
if not target_class:
|
|
22
|
+
return BaseDialectSettings()
|
|
23
|
+
return target_class()
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def system_columns(self) -> t.List[exp.ColumnDef]:
|
|
27
|
+
"""
|
|
28
|
+
Represents system columns.
|
|
29
|
+
"""
|
|
30
|
+
return []
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def system_functions(self) -> t.Dict[str, t.List[str]]:
|
|
34
|
+
"""
|
|
35
|
+
Represents system functions.
|
|
36
|
+
"""
|
|
37
|
+
return {}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
|
|
3
|
+
from sqlglot import exp
|
|
4
|
+
from sqlglot.dialects import postgres
|
|
5
|
+
|
|
6
|
+
from sqlleaf.settings.base import BaseDialectSettings
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PostgresSettings(BaseDialectSettings):
|
|
10
|
+
DIALECT = "postgres"
|
|
11
|
+
PSEUDOCOLUMNS = {
|
|
12
|
+
"cmax": "OID",
|
|
13
|
+
"cmin": "OID",
|
|
14
|
+
"ctid": "OID",
|
|
15
|
+
"oid": "OID",
|
|
16
|
+
"tableoid": "OID",
|
|
17
|
+
"xmax": "OID",
|
|
18
|
+
"xmin": "OID",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def system_columns(self) -> t.List[exp.ColumnDef]:
|
|
23
|
+
return [
|
|
24
|
+
exp.ColumnDef(this=exp.to_identifier(name), kind=exp.DataType.build(kind, self.DIALECT))
|
|
25
|
+
for name, kind in self.PSEUDOCOLUMNS.items()
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def system_functions(self) -> t.Dict[str, t.List[str]]:
|
|
30
|
+
"""
|
|
31
|
+
Map function names to their returned column names.
|
|
32
|
+
"""
|
|
33
|
+
return {
|
|
34
|
+
"aclexplode": ["grantor, grantee, privilege_type, is_grantable"],
|
|
35
|
+
"json_array_elements": ["value"],
|
|
36
|
+
"json_array_elements_text": ["value"],
|
|
37
|
+
"json_each": ["key", "value"],
|
|
38
|
+
"json_each_text": ["key", "value"],
|
|
39
|
+
"jsonb_array_elements": ["value"],
|
|
40
|
+
"jsonb_array_elements_text": ["value"],
|
|
41
|
+
"jsonb_each": ["key", "value"],
|
|
42
|
+
"jsonb_each_text": ["key", "value"],
|
|
43
|
+
"ts_debug": ["alias, description, token, dictionaries, dictionary, lexemes"],
|
|
44
|
+
"ts_parse": ["tokid", "token"],
|
|
45
|
+
"ts_stat": ["word", "ndoc", "nentry"],
|
|
46
|
+
"ts_token_type": ["tokid", "alias", "description"],
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# sqlglot is missing pseudocolumns for Postgres
|
|
51
|
+
postgres.Postgres.PSEUDOCOLUMNS = {c.upper() for c in PostgresSettings.PSEUDOCOLUMNS.keys()}
|
|
52
|
+
postgres.Postgres.EXCLUDES_PSEUDOCOLUMNS_FROM_STAR = True
|
|
File without changes
|
sqlleaf/typing.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from enum import StrEnum, auto
|
|
4
|
+
|
|
5
|
+
from sqlglot import exp
|
|
6
|
+
from sqlglot.optimizer import Scope
|
|
7
|
+
|
|
8
|
+
E = t.TypeVar("E", bound=exp.Expr)
|
|
9
|
+
|
|
10
|
+
"""
|
|
11
|
+
Represents a source and a target object of an SQL query.
|
|
12
|
+
|
|
13
|
+
For example,
|
|
14
|
+
INSERT INTO <target> SELECT * FROM <source>
|
|
15
|
+
COPY <target> FROM <source>
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
TargetExprType = exp.Table | exp.Literal | exp.Identifier | exp.Schema
|
|
19
|
+
SourceExprType = exp.Table | exp.Literal | exp.Identifier | exp.Select | exp.Schema | exp.Values
|
|
20
|
+
|
|
21
|
+
TableOrScopeType = exp.Table | Scope
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SqlObjectType(StrEnum):
|
|
25
|
+
"""
|
|
26
|
+
The types of models that represent a 'source' or 'target' in an SQL statement.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
NONE = auto()
|
|
30
|
+
|
|
31
|
+
# Targets and sources
|
|
32
|
+
DATABASE = auto()
|
|
33
|
+
DYNAMODB = auto()
|
|
34
|
+
FILE = auto()
|
|
35
|
+
PREPARED_STATEMENT = auto()
|
|
36
|
+
PROCEDURE = auto()
|
|
37
|
+
PROGRAM = auto()
|
|
38
|
+
SCHEMA = auto()
|
|
39
|
+
STAGE = auto()
|
|
40
|
+
STREAM = auto()
|
|
41
|
+
TABLE = auto()
|
|
42
|
+
# Sources
|
|
43
|
+
DML = auto()
|
|
44
|
+
SELECT = auto()
|
|
45
|
+
SET = auto() # Temporary
|
|
46
|
+
TUPLE = auto() # Temporary
|
|
47
|
+
VALUES = auto()
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def type_has_no_column_defs(cls, *clses: t.Iterable) -> bool:
|
|
51
|
+
"""
|
|
52
|
+
Return all the types that do not define any columns of their own.
|
|
53
|
+
"""
|
|
54
|
+
return not set(clses).isdisjoint({
|
|
55
|
+
cls.DYNAMODB,
|
|
56
|
+
cls.FILE,
|
|
57
|
+
cls.PROGRAM,
|
|
58
|
+
cls.STAGE,
|
|
59
|
+
cls.STREAM,
|
|
60
|
+
cls.PROCEDURE,
|
|
61
|
+
cls.PREPARED_STATEMENT,
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class SourceInfo:
|
|
67
|
+
expression: SourceExprType
|
|
68
|
+
type: SqlObjectType
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class TargetInfo:
|
|
73
|
+
expression: TargetExprType
|
|
74
|
+
type: SqlObjectType
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# TODO: merge this with SqlObjectType?
|
|
78
|
+
class TableType(StrEnum):
|
|
79
|
+
TABLE = auto()
|
|
80
|
+
VIEW = auto()
|
|
81
|
+
CTE = auto()
|
|
82
|
+
DERIVED_TABLE = auto()
|
|
83
|
+
STAGE = auto()
|
|
84
|
+
FILE = auto()
|
|
85
|
+
UDTF = auto() # LATERAL hello() as hello(col1, col2)
|
|
86
|
+
PROCEDURE = auto()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class TableSubtype(StrEnum):
|
|
90
|
+
RECURSIVE = auto()
|
|
91
|
+
TEMPORARY = auto()
|
|
92
|
+
EXTERNAL = auto()
|
|
93
|
+
MATERIALIZED = auto()
|