sqlassert 0.1.2__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlassert/__init__.py +34 -9
- sqlassert/__main__.py +138 -0
- sqlassert/check.py +444 -0
- sqlassert/diagnostics.py +57 -0
- sqlassert/dialects.py +211 -0
- sqlassert/discovery/__init__.py +30 -0
- sqlassert/discovery/bigquery.py +71 -0
- sqlassert/discovery/databricks.py +18 -0
- sqlassert/discovery/duckdb.py +163 -0
- sqlassert/discovery/fetcher.py +402 -0
- sqlassert/discovery/information_schema.py +326 -0
- sqlassert/discovery/mysql.py +70 -0
- sqlassert/discovery/postgres.py +143 -0
- sqlassert/discovery/redshift.py +36 -0
- sqlassert/discovery/snowflake.py +63 -0
- sqlassert/discovery/trino.py +37 -0
- sqlassert/improvements.py +528 -0
- sqlassert/ir/__init__.py +56 -0
- sqlassert/ir/convert.py +2377 -0
- sqlassert/ir/model.py +676 -0
- sqlassert/main.py +57 -0
- sqlassert/properties.py +121 -0
- sqlassert/provenance.py +73 -0
- sqlassert/python_engine.py +299 -0
- sqlassert/reporting.py +102 -0
- sqlassert/rules/STYLEGUIDE.md +19 -0
- sqlassert/rules/__init__.py +11 -0
- sqlassert/rules/column_set.py +34 -0
- sqlassert/rules/domain.py +376 -0
- sqlassert/rules/equality.py +233 -0
- sqlassert/rules/exactly_one.py +75 -0
- sqlassert/rules/foreign_key.py +316 -0
- sqlassert/rules/join_cardinality.py +94 -0
- sqlassert/rules/nullability.py +418 -0
- sqlassert/rules/uniqueness.py +601 -0
- sqlassert/smells.py +306 -0
- sqlassert/sql_parse.py +974 -0
- sqlassert/suggest.py +339 -0
- sqlassert-0.3.1.dist-info/METADATA +725 -0
- sqlassert-0.3.1.dist-info/RECORD +44 -0
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/WHEEL +1 -1
- sqlassert-0.3.1.dist-info/entry_points.txt +2 -0
- sqlassert-0.3.1.dist-info/licenses/LICENSE +21 -0
- sqlassert/unique.py +0 -605
- sqlassert-0.1.2.dist-info/METADATA +0 -173
- sqlassert-0.1.2.dist-info/RECORD +0 -6
- {sqlassert-0.1.2.dist-info → sqlassert-0.3.1.dist-info}/top_level.txt +0 -0
sqlassert/ir/convert.py
ADDED
|
@@ -0,0 +1,2377 @@
|
|
|
1
|
+
"""Link a parsed SQL program into the framework-independent relational IR.
|
|
2
|
+
|
|
3
|
+
Declaration and scope discovery happens before lowering. This gives table,
|
|
4
|
+
view, and CTE references compiler-like symbol resolution: every occurrence is
|
|
5
|
+
a fresh Alias, while all occurrences share one memoized NamedRelation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable, Iterable, Sequence, Set
|
|
11
|
+
from dataclasses import dataclass, field, replace
|
|
12
|
+
from functools import cache
|
|
13
|
+
from typing import Literal, assert_never
|
|
14
|
+
|
|
15
|
+
from sqlglot import Expr, exp
|
|
16
|
+
|
|
17
|
+
from sqlassert import diagnostics as diag
|
|
18
|
+
from sqlassert import ir
|
|
19
|
+
from sqlassert.dialects import DialectSemantics, semantics
|
|
20
|
+
from sqlassert.diagnostics import Diagnostic
|
|
21
|
+
from sqlassert.properties import (
|
|
22
|
+
EnumeratedDomain, ExactlyOneJoin, ExhaustiveCase, ForeignKey, FunctionalDependency, ManyToOneJoin, NonNullColumn, NonNullColumns, NonNullElements, NonNullUniqueSet,
|
|
23
|
+
OneToManyJoin, OneToOneJoin, Property, ReferentialCoverage, UniqueJoin,
|
|
24
|
+
UniqueSet,
|
|
25
|
+
)
|
|
26
|
+
from sqlassert.provenance import AttachmentSite, Origin, SQL
|
|
27
|
+
from sqlassert.sql_parse import (
|
|
28
|
+
JOIN_MARKER_NOUNS, MARKER_NOUNS, ColumnMarker, ParsedProgram, assertion_span, column_markers,
|
|
29
|
+
exhaustive_span, grouping, is_ambiguous_pipes, join_marker_kind, join_origin, literal_value, marker_position,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class IrConversionResult:
|
|
35
|
+
program: ir.Program
|
|
36
|
+
knowledge: tuple[Property, ...]
|
|
37
|
+
diagnostics: tuple[Diagnostic, ...] = ()
|
|
38
|
+
warnings: tuple[Diagnostic, ...] = ()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(eq=False)
|
|
42
|
+
class _Symbol:
|
|
43
|
+
name: str
|
|
44
|
+
role: ir.RelationRole
|
|
45
|
+
origin: Origin
|
|
46
|
+
body: exp.Query | None = None
|
|
47
|
+
statement: exp.Create | None = None
|
|
48
|
+
required_columns: set[str] = field(default_factory=set)
|
|
49
|
+
# Table-level constraints added after CREATE TABLE (ALTER TABLE, CREATE
|
|
50
|
+
# UNIQUE INDEX), shaped like the CREATE TABLE's own, and columns set NOT NULL.
|
|
51
|
+
constraints: list[Expr] = field(default_factory=list)
|
|
52
|
+
not_null: set[str] = field(default_factory=set)
|
|
53
|
+
# Each unique index with an expression key part: its parts and statement.
|
|
54
|
+
expression_keys: list[tuple[list[Expr], exp.Create]] = field(default_factory=list)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class _Binding:
|
|
59
|
+
qualifier: str
|
|
60
|
+
name: str
|
|
61
|
+
column: ir.OutputColumn
|
|
62
|
+
# The schema (and catalog) of an unaliased table, as `s1` in `s1.t.k`.
|
|
63
|
+
schema: str = ""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class _MarkerReference:
|
|
68
|
+
"""A Foreign Key marker whose referencing columns are lowered, awaiting its referenced relation."""
|
|
69
|
+
|
|
70
|
+
marker: ColumnMarker
|
|
71
|
+
kind: Literal["foreign-key", "declared-foreign-key"]
|
|
72
|
+
origin: Origin
|
|
73
|
+
columns: tuple[ir.OutputColumn, ...]
|
|
74
|
+
target: _Symbol
|
|
75
|
+
position: int
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class _ConstraintReference:
|
|
80
|
+
"""A `REFERENCES` constraint of a declared table onto a relation the program reaches."""
|
|
81
|
+
|
|
82
|
+
table: _Symbol
|
|
83
|
+
columns: tuple[str, ...]
|
|
84
|
+
target: _Symbol
|
|
85
|
+
target_columns: tuple[str, ...] | None
|
|
86
|
+
origin: Origin
|
|
87
|
+
# The `REFERENCES` or `FOREIGN KEY` constraint itself.
|
|
88
|
+
node: Expr
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class _Lowered:
|
|
93
|
+
relation: ir.RelationExpr
|
|
94
|
+
bindings: tuple[_Binding, ...]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass
|
|
98
|
+
class IrParser:
|
|
99
|
+
dialect: str
|
|
100
|
+
# Unenforced keys and foreign keys become Property Declarations, as `RELY` makes them.
|
|
101
|
+
trust_unenforced_keys: bool = True
|
|
102
|
+
_global_symbols: dict[str, _Symbol] = field(default_factory=dict)
|
|
103
|
+
_symbols: list[_Symbol] = field(default_factory=list)
|
|
104
|
+
_query_scopes: dict[int, dict[str, _Symbol]] = field(default_factory=dict)
|
|
105
|
+
_table_symbols: dict[int, _Symbol] = field(default_factory=dict)
|
|
106
|
+
_source_required: dict[int, set[str]] = field(default_factory=dict)
|
|
107
|
+
_source_expressions: dict[int, Expr] = field(default_factory=dict)
|
|
108
|
+
_named_relations: dict[_Symbol, ir.NamedRelation] = field(default_factory=dict)
|
|
109
|
+
_resolving: set[_Symbol] = field(default_factory=set)
|
|
110
|
+
_knowledge: list[Property] = field(default_factory=list)
|
|
111
|
+
_assertions: list[ir.Assertion] = field(default_factory=list)
|
|
112
|
+
_declarations: list[Property] = field(default_factory=list)
|
|
113
|
+
_diagnostics: list[Diagnostic] = field(default_factory=list)
|
|
114
|
+
_warnings: list[Diagnostic] = field(default_factory=list)
|
|
115
|
+
_handled_column_markers: set[int] = field(default_factory=set)
|
|
116
|
+
_handled_joins: set[int] = field(default_factory=set)
|
|
117
|
+
_handled_cases: set[int] = field(default_factory=set)
|
|
118
|
+
_reported_cycles: set[_Symbol] = field(default_factory=set)
|
|
119
|
+
_attachment_sites: list[tuple[AttachmentSite, ir.RelationExpr | ir.SimpleCase]] = field(default_factory=list)
|
|
120
|
+
_sited_cases: set[int] = field(default_factory=set)
|
|
121
|
+
# Each scalar subquery lowered as a join, by id, and the qualifier of the
|
|
122
|
+
# binding that holds its value.
|
|
123
|
+
_scalar_values: dict[int, str] = field(default_factory=dict)
|
|
124
|
+
_scalar_subqueries: list[tuple[AttachmentSite, ir.Join]] = field(default_factory=list)
|
|
125
|
+
_sort_keys: dict[int, tuple[ir.OutputColumn, ...] | None] = field(default_factory=dict)
|
|
126
|
+
_site_sort_keys: list[tuple[AttachmentSite, tuple[ir.OutputColumn, ...] | None]] = field(default_factory=list)
|
|
127
|
+
_set_operations: list[tuple[AttachmentSite, ir.SetOperation]] = field(default_factory=list)
|
|
128
|
+
# Columns made up for names a query uses: of a table with no CREATE, or an
|
|
129
|
+
# opaque relation (`_guessed`), and those a declared table does not have (`_absent`).
|
|
130
|
+
_guessed: set[ir.OutputColumn] = field(default_factory=set)
|
|
131
|
+
# A scalar subquery's columns, each bound to the scope that owns it.
|
|
132
|
+
_scoped_columns: dict[int, ir.OutputColumn] = field(default_factory=dict)
|
|
133
|
+
_absent: set[ir.OutputColumn] = field(default_factory=set)
|
|
134
|
+
# Each declared table column's type, as its CREATE TABLE writes it.
|
|
135
|
+
_declared_types: dict[ir.OutputColumn, exp.DataType] = field(default_factory=dict)
|
|
136
|
+
# The comparisons already checked for a zoned timestamp's text, by id.
|
|
137
|
+
_zone_checked: set[int] = field(default_factory=set)
|
|
138
|
+
# Each composite column or CAST a comparison reads, by id, with the
|
|
139
|
+
# comparison: a warning unless its elements are declared non-null.
|
|
140
|
+
_composite_comparisons: dict[int, tuple[Origin, str, str, ir.ScalarExpr]] = field(default_factory=dict)
|
|
141
|
+
_marker_targets: dict[int, _Symbol] = field(default_factory=dict)
|
|
142
|
+
_marker_references: list[_MarkerReference] = field(default_factory=list)
|
|
143
|
+
_constraint_references: list[_ConstraintReference] = field(default_factory=list)
|
|
144
|
+
# Constraints an ALTER TABLE ... NOT VALID added: rows already there were never checked.
|
|
145
|
+
_unvalidated: set[int] = field(default_factory=set)
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def semantics(self) -> DialectSemantics:
|
|
149
|
+
return semantics(self.dialect)
|
|
150
|
+
|
|
151
|
+
def parse(self, ast: ParsedProgram) -> IrConversionResult:
|
|
152
|
+
for statement in ast.create_statements:
|
|
153
|
+
match statement:
|
|
154
|
+
case exp.Create(kind="INDEX"):
|
|
155
|
+
self._declare_index(statement)
|
|
156
|
+
case exp.Create(kind="SCHEMA"):
|
|
157
|
+
pass # A schema states nothing about rows, like a plain index.
|
|
158
|
+
case exp.Create():
|
|
159
|
+
self._declare(statement)
|
|
160
|
+
case exp.Alter():
|
|
161
|
+
self._alter(statement)
|
|
162
|
+
case _:
|
|
163
|
+
raise TypeError(f"Unhandled schema statement: {type(statement).__name__}")
|
|
164
|
+
|
|
165
|
+
for symbol in tuple(self._symbols):
|
|
166
|
+
if symbol.role is ir.RelationRole.VIEW and symbol.body is not None:
|
|
167
|
+
self._discover_query(symbol.body, {})
|
|
168
|
+
if ast.root_select is not None:
|
|
169
|
+
self._discover_query(ast.root_select, {})
|
|
170
|
+
self._discover_constraint_references()
|
|
171
|
+
self._propagate_required_columns()
|
|
172
|
+
|
|
173
|
+
root = self._lower_query(ast.root_select) if ast.root_select is not None else None
|
|
174
|
+
# Every view is a query of its own: its markers are checked whether or
|
|
175
|
+
# not the root reads it, so a file of views (a project's models) needs
|
|
176
|
+
# no root SELECT.
|
|
177
|
+
for symbol in tuple(self._symbols):
|
|
178
|
+
if symbol.role is ir.RelationRole.VIEW and symbol not in self._named_relations:
|
|
179
|
+
self._resolve_symbol(symbol)
|
|
180
|
+
# Referenced relations are linked last: a marker on a view may name
|
|
181
|
+
# the view itself, or one that reads it.
|
|
182
|
+
self._link_constraint_references()
|
|
183
|
+
self._link_marker_references()
|
|
184
|
+
self._report_unanalyzed(ast)
|
|
185
|
+
|
|
186
|
+
named_relations = tuple(self._named_relations.get(symbol) or self._unresolved_named_relation(symbol) for symbol in self._symbols)
|
|
187
|
+
warnings = [*self._undeclared_table_warnings(), *self._warnings, *self._composite_warnings()]
|
|
188
|
+
return IrConversionResult(
|
|
189
|
+
ir.Program(named_relations=named_relations, root=root, assertions=tuple(self._assertions), declarations=tuple(self._declarations), attachment_sites=tuple(self._attachment_sites),
|
|
190
|
+
scalar_subqueries=tuple(self._scalar_subqueries), sort_keys=tuple(self._site_sort_keys),
|
|
191
|
+
set_operations=tuple(self._set_operations)),
|
|
192
|
+
tuple(self._knowledge), tuple(self._diagnostics), tuple(dict.fromkeys(warnings)),
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def _undeclared_table_warnings(self) -> list[Diagnostic]:
|
|
196
|
+
return [
|
|
197
|
+
Diagnostic(
|
|
198
|
+
diag.UNDECLARED_TABLE,
|
|
199
|
+
f"table {symbol.name} has no CREATE TABLE in the program, so its keys, nullability and column types are unknown: "
|
|
200
|
+
"declare it with CREATE TABLE, or analyze with a database connection (analyze_database)",
|
|
201
|
+
symbol.origin,
|
|
202
|
+
)
|
|
203
|
+
for symbol in self._symbols
|
|
204
|
+
if symbol.role is ir.RelationRole.TABLE and symbol.statement is None
|
|
205
|
+
]
|
|
206
|
+
|
|
207
|
+
# Declaration and symbol discovery ---------------------------------------
|
|
208
|
+
|
|
209
|
+
def _declare(self, statement: exp.Create) -> None:
|
|
210
|
+
kind = (statement.kind or "").upper()
|
|
211
|
+
if kind not in _SUPPORTED_CREATE_KINDS:
|
|
212
|
+
self._diagnostics.append(
|
|
213
|
+
Diagnostic(
|
|
214
|
+
diag.UNSUPPORTED_CREATE_STATEMENT,
|
|
215
|
+
f"create statement kind {kind or '<unknown>'} is not supported; only TABLE, VIEW, INDEX, and SCHEMA are analyzed: {statement.sql(dialect=self.dialect)}",
|
|
216
|
+
)
|
|
217
|
+
)
|
|
218
|
+
return
|
|
219
|
+
|
|
220
|
+
table = _created_table(statement)
|
|
221
|
+
if table is None:
|
|
222
|
+
return
|
|
223
|
+
name = _qualified_name(table)
|
|
224
|
+
key = name.lower()
|
|
225
|
+
origin = Origin(SQL, statement.sql(dialect=self.dialect))
|
|
226
|
+
if key in self._global_symbols:
|
|
227
|
+
self._diagnostics.append(Diagnostic(diag.DUPLICATE_DECLARATION, f"relation {name} is declared more than once", origin))
|
|
228
|
+
return
|
|
229
|
+
|
|
230
|
+
role = ir.RelationRole.TABLE if kind == "TABLE" else ir.RelationRole.VIEW
|
|
231
|
+
body = statement.args.get("expression")
|
|
232
|
+
query = _unwrap_query(body)
|
|
233
|
+
symbol = _Symbol(name=name, role=role, origin=origin, body=query, statement=statement)
|
|
234
|
+
self._global_symbols[key] = symbol
|
|
235
|
+
self._symbols.append(symbol)
|
|
236
|
+
|
|
237
|
+
def _declare_index(self, statement: exp.Create) -> None:
|
|
238
|
+
"""A unique index over plain columns is a Unique Set, like UNIQUE; one
|
|
239
|
+
with an expression key part may be (`_expression_key`); a partial one,
|
|
240
|
+
or any other index, states nothing about the data."""
|
|
241
|
+
index = statement.this
|
|
242
|
+
table = index.args.get("table") if isinstance(index, exp.Index) else None
|
|
243
|
+
if not isinstance(table, exp.Table):
|
|
244
|
+
self._unsupported_create(statement, "an index without a table")
|
|
245
|
+
return
|
|
246
|
+
symbol = self._created_before(table, statement)
|
|
247
|
+
if symbol is None or not statement.args.get("unique"):
|
|
248
|
+
return
|
|
249
|
+
params = index.args.get("params")
|
|
250
|
+
columns = params.args.get("columns") or () if isinstance(params, exp.IndexParameters) else ()
|
|
251
|
+
plain = [
|
|
252
|
+
column.this for column in columns
|
|
253
|
+
if isinstance(column, exp.Ordered) and isinstance(column.this, exp.Column) and not column.this.table
|
|
254
|
+
]
|
|
255
|
+
if not columns or params.args.get("where") is not None:
|
|
256
|
+
# Uniqueness of only some rows is not a Unique Set.
|
|
257
|
+
self._unsupported_create(statement, "a partial unique index")
|
|
258
|
+
return
|
|
259
|
+
if len(plain) != len(columns):
|
|
260
|
+
parts = [column.this if isinstance(column, exp.Ordered) else column for column in columns]
|
|
261
|
+
symbol.expression_keys.append(([part.unnest() for part in parts], statement))
|
|
262
|
+
return
|
|
263
|
+
schema = exp.Schema(expressions=[exp.to_identifier(column.name) for column in plain])
|
|
264
|
+
symbol.constraints.append(exp.UniqueColumnConstraint(this=schema))
|
|
265
|
+
|
|
266
|
+
def _alter(self, statement: exp.Alter) -> None:
|
|
267
|
+
"""ALTER TABLE adding PRIMARY KEY, UNIQUE, or FOREIGN KEY constraints, or
|
|
268
|
+
setting columns NOT NULL. Any other action could invalidate what is
|
|
269
|
+
known of the table, so a statement with one states nothing."""
|
|
270
|
+
actions = statement.args.get("actions") or ()
|
|
271
|
+
added: list[Expr] = []
|
|
272
|
+
not_null: list[str] = []
|
|
273
|
+
for action in actions:
|
|
274
|
+
match action:
|
|
275
|
+
case exp.AddConstraint():
|
|
276
|
+
added.extend(_unwrap_constraints(action.expressions))
|
|
277
|
+
case exp.AlterColumn() if action.args.get("allow_null") is False and {
|
|
278
|
+
key for key, value in action.args.items() if value is not None and value is not False
|
|
279
|
+
} == {"this"}:
|
|
280
|
+
not_null.append(action.name)
|
|
281
|
+
case _:
|
|
282
|
+
added.append(action)
|
|
283
|
+
if not actions or not all(_is_added_constraint(constraint) for constraint in added):
|
|
284
|
+
self._diagnostics.append(Diagnostic(
|
|
285
|
+
diag.UNSUPPORTED_STATEMENT,
|
|
286
|
+
f"only ALTER TABLE adding PRIMARY KEY, UNIQUE, or FOREIGN KEY constraints, or setting NOT NULL, "
|
|
287
|
+
f"is analyzed: {statement.sql(dialect=self.dialect)}",
|
|
288
|
+
self._origin(statement),
|
|
289
|
+
))
|
|
290
|
+
return
|
|
291
|
+
table = statement.this
|
|
292
|
+
symbol = self._created_before(table, statement) if isinstance(table, exp.Table) else None
|
|
293
|
+
if symbol is None:
|
|
294
|
+
return
|
|
295
|
+
symbol.constraints.extend(added)
|
|
296
|
+
symbol.not_null.update(not_null)
|
|
297
|
+
if statement.args.get("not_valid"):
|
|
298
|
+
self._unvalidated.update(id(constraint) for constraint in added)
|
|
299
|
+
|
|
300
|
+
def _created_before(self, table: exp.Table, statement: Expr) -> _Symbol | None:
|
|
301
|
+
"""The table `statement` amends, which a CREATE TABLE earlier in the
|
|
302
|
+
program must declare; otherwise the statement is diagnosed."""
|
|
303
|
+
name = _qualified_name(table)
|
|
304
|
+
symbol = self._global_symbols.get(name.lower())
|
|
305
|
+
if symbol is None or symbol.role is not ir.RelationRole.TABLE or symbol.statement is None:
|
|
306
|
+
self._diagnostics.append(Diagnostic(
|
|
307
|
+
diag.CONSTRAINT_WITHOUT_TABLE,
|
|
308
|
+
f"{name} is not a table created earlier in the program, so this statement's constraints are ignored: "
|
|
309
|
+
f"{statement.sql(dialect=self.dialect)}",
|
|
310
|
+
self._origin(statement),
|
|
311
|
+
))
|
|
312
|
+
return None
|
|
313
|
+
return symbol
|
|
314
|
+
|
|
315
|
+
def _unsupported_create(self, statement: exp.Create, what: str) -> None:
|
|
316
|
+
self._diagnostics.append(Diagnostic(
|
|
317
|
+
diag.UNSUPPORTED_CREATE_STATEMENT,
|
|
318
|
+
f"{what} is not analyzed: {statement.sql(dialect=self.dialect)}",
|
|
319
|
+
self._origin(statement),
|
|
320
|
+
))
|
|
321
|
+
|
|
322
|
+
def _discover_query(self, query: exp.Query, inherited_scope: dict[str, _Symbol]) -> None:
|
|
323
|
+
if id(query) in self._query_scopes:
|
|
324
|
+
return
|
|
325
|
+
|
|
326
|
+
scope = dict(inherited_scope)
|
|
327
|
+
with_clause = _arg(query, "with")
|
|
328
|
+
if with_clause is not None:
|
|
329
|
+
for cte in with_clause.expressions:
|
|
330
|
+
body = _unwrap_query(cte.this)
|
|
331
|
+
symbol = _Symbol(
|
|
332
|
+
name=cte.alias,
|
|
333
|
+
role=ir.RelationRole.CTE,
|
|
334
|
+
origin=Origin(SQL, cte.sql(dialect=self.dialect)),
|
|
335
|
+
body=body,
|
|
336
|
+
)
|
|
337
|
+
scope[cte.alias.lower()] = symbol
|
|
338
|
+
self._symbols.append(symbol)
|
|
339
|
+
|
|
340
|
+
self._query_scopes[id(query)] = scope
|
|
341
|
+
if isinstance(query, exp.SetOperation):
|
|
342
|
+
# SQLGlot 30 wraps a nested set operation in a Subquery.
|
|
343
|
+
for side in (query.this, query.expression):
|
|
344
|
+
inner = _unwrap_query(side)
|
|
345
|
+
if inner is not None:
|
|
346
|
+
self._discover_query(inner, scope)
|
|
347
|
+
return
|
|
348
|
+
if not isinstance(query, exp.Select):
|
|
349
|
+
return
|
|
350
|
+
|
|
351
|
+
sources = _direct_sources(query)
|
|
352
|
+
by_qualifier: dict[str, list[Expr]] = {}
|
|
353
|
+
for source in sources:
|
|
354
|
+
self._source_required.setdefault(id(source), set())
|
|
355
|
+
self._source_expressions[id(source)] = source
|
|
356
|
+
qualifier = source.alias_or_name.lower()
|
|
357
|
+
if qualifier:
|
|
358
|
+
by_qualifier.setdefault(qualifier, []).append(source)
|
|
359
|
+
if isinstance(source, exp.Table):
|
|
360
|
+
symbol = self._symbol_for_table(source, scope)
|
|
361
|
+
self._table_symbols[id(source)] = symbol
|
|
362
|
+
elif isinstance(source, exp.Subquery):
|
|
363
|
+
body = _unwrap_query(source)
|
|
364
|
+
if body is not None:
|
|
365
|
+
self._discover_query(body, scope)
|
|
366
|
+
|
|
367
|
+
for column in query.find_all(exp.Column):
|
|
368
|
+
if column.table:
|
|
369
|
+
matching = by_qualifier.get(column.table.lower(), ())
|
|
370
|
+
for source in matching:
|
|
371
|
+
self._require(source, column.name)
|
|
372
|
+
elif len(sources) == 1 and column.find_ancestor(exp.Select) is query:
|
|
373
|
+
# An unqualified column in a nested query belongs to that query's scope.
|
|
374
|
+
self._require(sources[0], column.name)
|
|
375
|
+
|
|
376
|
+
for join in query.args.get("joins") or ():
|
|
377
|
+
for identifier in join.args.get("using") or ():
|
|
378
|
+
for source in sources:
|
|
379
|
+
self._require(source, identifier.name)
|
|
380
|
+
|
|
381
|
+
if len(sources) == 1 and _star_only(query):
|
|
382
|
+
for marker in column_markers(query):
|
|
383
|
+
for column in (*marker.columns, *marker.dependent):
|
|
384
|
+
self._require(sources[0], column)
|
|
385
|
+
|
|
386
|
+
for marker in column_markers(query):
|
|
387
|
+
if marker.referenced_relation is not None:
|
|
388
|
+
origin = Origin.marked(marker.referenced_relation, marker.span)
|
|
389
|
+
target = self._symbol_for_name(marker.referenced_relation, scope, origin)
|
|
390
|
+
self._marker_targets[id(marker)] = target
|
|
391
|
+
self._require_undeclared(target, marker.referenced_columns)
|
|
392
|
+
|
|
393
|
+
where = query.args.get("where")
|
|
394
|
+
for conjunct in _conjuncts(where.this) if where is not None else ():
|
|
395
|
+
matched = _subquery_conjunct(conjunct)
|
|
396
|
+
if matched is not None:
|
|
397
|
+
self._discover_query(matched[1], scope)
|
|
398
|
+
for expression in (*(_conjuncts(where.this) if where is not None else ()), *query.expressions):
|
|
399
|
+
for subquery in _scalar_subqueries(expression):
|
|
400
|
+
if isinstance(subquery.this, exp.Select):
|
|
401
|
+
self._discover_query(subquery.this, scope)
|
|
402
|
+
|
|
403
|
+
if with_clause is not None:
|
|
404
|
+
for cte in with_clause.expressions:
|
|
405
|
+
body = _unwrap_query(cte.this)
|
|
406
|
+
if body is not None:
|
|
407
|
+
self._discover_query(body, scope)
|
|
408
|
+
|
|
409
|
+
def _symbol_for_table(self, table: exp.Table, scope: dict[str, _Symbol]) -> _Symbol:
|
|
410
|
+
return self._symbol_for_name(_qualified_name(table), scope, Origin(SQL, table.sql(dialect=self.dialect)))
|
|
411
|
+
|
|
412
|
+
def _symbol_for_name(self, name: str, scope: dict[str, _Symbol], origin: Origin) -> _Symbol:
|
|
413
|
+
key = name.lower()
|
|
414
|
+
if key in scope:
|
|
415
|
+
return scope[key]
|
|
416
|
+
symbol = self._global_symbols.get(key)
|
|
417
|
+
if symbol is None:
|
|
418
|
+
symbol = _Symbol(name=name, role=ir.RelationRole.TABLE, origin=origin)
|
|
419
|
+
self._global_symbols[key] = symbol
|
|
420
|
+
self._symbols.append(symbol)
|
|
421
|
+
return symbol
|
|
422
|
+
|
|
423
|
+
def _require_undeclared(self, symbol: _Symbol, columns: Iterable[str]) -> None:
|
|
424
|
+
"""Referenced columns become columns of an undeclared table only: a
|
|
425
|
+
declared relation's schema is never widened by a reference to it."""
|
|
426
|
+
if symbol.role is ir.RelationRole.TABLE and symbol.statement is None:
|
|
427
|
+
symbol.required_columns.update(columns)
|
|
428
|
+
|
|
429
|
+
def _discover_constraint_references(self) -> None:
|
|
430
|
+
"""`REFERENCES` constraints onto relations the program reaches; one onto
|
|
431
|
+
a relation outside the program can never be used, so it is not linked."""
|
|
432
|
+
for symbol in tuple(self._symbols):
|
|
433
|
+
if symbol.role is not ir.RelationRole.TABLE or symbol.statement is None:
|
|
434
|
+
continue
|
|
435
|
+
for columns, table, target_columns, node in _references(symbol):
|
|
436
|
+
target = self._global_symbols.get(_qualified_name(table).lower())
|
|
437
|
+
if target is None or target.role is not ir.RelationRole.TABLE:
|
|
438
|
+
continue
|
|
439
|
+
self._require_undeclared(target, target_columns or ())
|
|
440
|
+
self._constraint_references.append(
|
|
441
|
+
_ConstraintReference(symbol, columns, target, target_columns, self._origin(node.find_ancestor(exp.ColumnDef) or node), node)
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
def _require(self, source: Expr, column: str) -> None:
|
|
445
|
+
self._source_required.setdefault(id(source), set()).add(column)
|
|
446
|
+
if isinstance(source, exp.Table):
|
|
447
|
+
self._table_symbols[id(source)].required_columns.add(column)
|
|
448
|
+
|
|
449
|
+
def _propagate_required_columns(self) -> None:
|
|
450
|
+
"""Push requested output columns through star-only view/CTE/subquery bodies."""
|
|
451
|
+
changed = True
|
|
452
|
+
while changed:
|
|
453
|
+
before = sum(len(symbol.required_columns) for symbol in self._symbols) + sum(len(columns) for columns in self._source_required.values())
|
|
454
|
+
for symbol in self._symbols:
|
|
455
|
+
if isinstance(symbol.body, exp.Select) and _star_only(symbol.body):
|
|
456
|
+
sources = _direct_sources(symbol.body)
|
|
457
|
+
if len(sources) == 1:
|
|
458
|
+
for column in symbol.required_columns:
|
|
459
|
+
self._require(sources[0], column)
|
|
460
|
+
for source_id, columns in tuple(self._source_required.items()):
|
|
461
|
+
source = self._source_expressions[source_id]
|
|
462
|
+
body = _unwrap_query(source) if isinstance(source, exp.Subquery) else None
|
|
463
|
+
if isinstance(body, exp.Select) and _star_only(body):
|
|
464
|
+
nested_sources = _direct_sources(body)
|
|
465
|
+
if len(nested_sources) == 1:
|
|
466
|
+
for column in columns:
|
|
467
|
+
self._require(nested_sources[0], column)
|
|
468
|
+
after = sum(len(symbol.required_columns) for symbol in self._symbols) + sum(len(columns) for columns in self._source_required.values())
|
|
469
|
+
changed = after != before
|
|
470
|
+
|
|
471
|
+
# Lowering ---------------------------------------------------------------
|
|
472
|
+
|
|
473
|
+
def _lower_query(self, query: exp.Query) -> ir.RelationExpr | None:
|
|
474
|
+
"""A Root Select, view, CTE, and FROM-subquery body all lower the same way."""
|
|
475
|
+
unwrapped = _unwrap_query(query)
|
|
476
|
+
if unwrapped is None:
|
|
477
|
+
return None
|
|
478
|
+
query = unwrapped
|
|
479
|
+
if isinstance(query, exp.SetOperation):
|
|
480
|
+
left = self._lower_query(query.this)
|
|
481
|
+
right = self._lower_query(query.expression)
|
|
482
|
+
if left is None:
|
|
483
|
+
left = self._opaque_relation(query.this)
|
|
484
|
+
if right is None:
|
|
485
|
+
right = self._opaque_relation(query.expression)
|
|
486
|
+
output_columns = _pass_output_columns(left.output_columns, self._origin(query))
|
|
487
|
+
relation = ir.SetOperation(
|
|
488
|
+
origin=self._origin(query), output_columns=output_columns, is_schema_complete=left.is_schema_complete and right.is_schema_complete,
|
|
489
|
+
operator=_SET_OPERATORS[type(query)], left=left, right=right, distinct=bool(query.args.get("distinct")),
|
|
490
|
+
by_name=bool(query.args.get("by_name")),
|
|
491
|
+
)
|
|
492
|
+
last = _last_select(query.this)
|
|
493
|
+
position = marker_position(last) if last is not None else None
|
|
494
|
+
if relation.operator != "union" and position is not None:
|
|
495
|
+
self._set_operations.append((AttachmentSite(relation.operator.upper(), position[1], position[0], True), relation))
|
|
496
|
+
return self._lower_limit(relation, query)
|
|
497
|
+
if not isinstance(query, exp.Select):
|
|
498
|
+
return None
|
|
499
|
+
return self._lower_select(query)
|
|
500
|
+
|
|
501
|
+
def _lower_select(self, select: exp.Select) -> ir.RelationExpr | None:
|
|
502
|
+
source = _from_source(select)
|
|
503
|
+
if source is None:
|
|
504
|
+
# No FROM: the select list is evaluated once, over one row of no
|
|
505
|
+
# columns, which an opaque relation limited to one row stands for.
|
|
506
|
+
origin = self._origin(select)
|
|
507
|
+
row = ir.OpaqueRelation(origin=origin, output_columns=(), is_schema_complete=True, description="no FROM")
|
|
508
|
+
one_row = ir.Limit(origin=origin, output_columns=(), is_schema_complete=True, input=row, count=1)
|
|
509
|
+
relation = self._lower_select_tail(_Lowered(one_row, ()), select)
|
|
510
|
+
elif any(join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()):
|
|
511
|
+
relation = self._opaque_relation(select, _selected_output_names(select))
|
|
512
|
+
else:
|
|
513
|
+
lowered = self._lower_source(source)
|
|
514
|
+
for join in select.args.get("joins") or ():
|
|
515
|
+
lowered = self._lower_join(lowered, join)
|
|
516
|
+
relation = self._lower_select_tail(lowered, select)
|
|
517
|
+
return self._finish_select(relation, select)
|
|
518
|
+
|
|
519
|
+
def _finish_select(self, relation: ir.RelationExpr, select: exp.Select) -> ir.RelationExpr:
|
|
520
|
+
relation = self._lower_limit(relation, select)
|
|
521
|
+
self._register_column_markers(relation, select)
|
|
522
|
+
self._add_attachment_site(select, _select_label(select), relation, trailing=True)
|
|
523
|
+
position = marker_position(select)
|
|
524
|
+
if position is not None:
|
|
525
|
+
offset, line = position
|
|
526
|
+
self._site_sort_keys.append((AttachmentSite(_select_label(select), line, offset, True), self._sort_keys.get(id(select))))
|
|
527
|
+
return relation
|
|
528
|
+
|
|
529
|
+
def _lower_limit(self, relation: ir.RelationExpr, query: exp.Query) -> ir.RelationExpr:
|
|
530
|
+
limit = query.args.get("limit")
|
|
531
|
+
if limit is None and query.args.get("offset") is None:
|
|
532
|
+
return relation
|
|
533
|
+
count = None
|
|
534
|
+
if limit is not None:
|
|
535
|
+
expression = limit.args.get("expression") if isinstance(limit, exp.Limit) else limit.args.get("count")
|
|
536
|
+
options = limit.args.get("limit_options")
|
|
537
|
+
hard_bound = options is None or not any(options.args.get(key) for key in ("percent", "with_ties"))
|
|
538
|
+
if hard_bound and isinstance(expression, exp.Literal) and expression.is_int:
|
|
539
|
+
count = int(expression.this)
|
|
540
|
+
origin = self._origin(query)
|
|
541
|
+
return ir.Limit(origin=origin, output_columns=_pass_output_columns(relation.output_columns, origin), is_schema_complete=relation.is_schema_complete, input=relation, count=count)
|
|
542
|
+
|
|
543
|
+
def _lower_source(self, source: Expr) -> _Lowered:
|
|
544
|
+
if isinstance(source, exp.Table):
|
|
545
|
+
symbol = self._table_symbols[id(source)]
|
|
546
|
+
alias_name = source.alias or source.name
|
|
547
|
+
# An unaliased table may be named with its schema: `s1.t.k`.
|
|
548
|
+
schema = "" if source.alias else _schema_path(source)
|
|
549
|
+
if symbol in self._resolving:
|
|
550
|
+
if symbol.role is ir.RelationRole.VIEW:
|
|
551
|
+
self._report_cycle(symbol, source)
|
|
552
|
+
recursive = self._recursive_relation(symbol, source)
|
|
553
|
+
return self._alias(recursive, alias_name, self._origin(source), schema)
|
|
554
|
+
return self._alias(self._resolve_symbol(symbol), alias_name, self._origin(source), schema)
|
|
555
|
+
|
|
556
|
+
if isinstance(source, exp.Subquery):
|
|
557
|
+
relation = self._lower_query(source)
|
|
558
|
+
if relation is not None:
|
|
559
|
+
# DuckDB allows an unaliased FROM subquery; nothing can qualify
|
|
560
|
+
# a column by its name, so a placeholder serves.
|
|
561
|
+
return self._alias(relation, source.alias or "subquery", self._origin(source))
|
|
562
|
+
|
|
563
|
+
opaque = self._opaque_relation(source, self._source_required.get(id(source), set()))
|
|
564
|
+
return self._alias(opaque, source.alias_or_name or "opaque", self._origin(source))
|
|
565
|
+
|
|
566
|
+
def _resolve_symbol(self, symbol: _Symbol) -> ir.NamedRelation:
|
|
567
|
+
existing = self._named_relations.get(symbol)
|
|
568
|
+
if existing is not None:
|
|
569
|
+
return existing
|
|
570
|
+
|
|
571
|
+
if symbol.role is ir.RelationRole.TABLE:
|
|
572
|
+
output_columns, complete = self._table_output_columns(symbol)
|
|
573
|
+
named = ir.NamedRelation(
|
|
574
|
+
origin=symbol.origin, output_columns=output_columns, is_schema_complete=complete, name=symbol.name, role=symbol.role
|
|
575
|
+
)
|
|
576
|
+
self._named_relations[symbol] = named
|
|
577
|
+
self._table_knowledge(named, symbol)
|
|
578
|
+
return named
|
|
579
|
+
|
|
580
|
+
self._resolving.add(symbol)
|
|
581
|
+
try:
|
|
582
|
+
body = self._lower_query(symbol.body) if symbol.body is not None else None
|
|
583
|
+
finally:
|
|
584
|
+
self._resolving.discard(symbol)
|
|
585
|
+
if body is None:
|
|
586
|
+
body = self._opaque_relation(symbol.body or exp.Table(this=symbol.name), symbol.required_columns)
|
|
587
|
+
output_columns = _pass_output_columns(body.output_columns, symbol.origin)
|
|
588
|
+
named = ir.NamedRelation(
|
|
589
|
+
origin=symbol.origin,
|
|
590
|
+
output_columns=output_columns,
|
|
591
|
+
is_schema_complete=body.is_schema_complete,
|
|
592
|
+
name=symbol.name,
|
|
593
|
+
role=symbol.role,
|
|
594
|
+
body=body,
|
|
595
|
+
)
|
|
596
|
+
self._named_relations[symbol] = named
|
|
597
|
+
return named
|
|
598
|
+
|
|
599
|
+
def _unresolved_named_relation(self, symbol: _Symbol) -> ir.NamedRelation:
|
|
600
|
+
output_columns, complete = self._table_output_columns(symbol) if symbol.role is ir.RelationRole.TABLE else (self._opaque_output_columns(symbol.required_columns, symbol.origin), False)
|
|
601
|
+
named = ir.NamedRelation(
|
|
602
|
+
origin=symbol.origin, output_columns=output_columns, is_schema_complete=complete, name=symbol.name, role=symbol.role
|
|
603
|
+
)
|
|
604
|
+
self._named_relations[symbol] = named
|
|
605
|
+
if symbol.role is ir.RelationRole.TABLE:
|
|
606
|
+
self._table_knowledge(named, symbol)
|
|
607
|
+
return named
|
|
608
|
+
|
|
609
|
+
def _table_output_columns(self, symbol: _Symbol) -> tuple[tuple[ir.OutputColumn, ...], bool]:
|
|
610
|
+
definitions = list(symbol.statement.this.find_all(exp.ColumnDef)) if symbol.statement is not None else []
|
|
611
|
+
names = [definition.name for definition in definitions]
|
|
612
|
+
seen = {name.lower() for name in names}
|
|
613
|
+
names.extend(column for column in sorted(symbol.required_columns, key=str.lower) if column.lower() not in seen)
|
|
614
|
+
complete = symbol.statement is not None and {column.lower() for column in symbol.required_columns} <= seen
|
|
615
|
+
columns = self._opaque_output_columns(names, symbol.origin)
|
|
616
|
+
by_name = {definition.name: definition for definition in definitions}
|
|
617
|
+
if symbol.statement is not None:
|
|
618
|
+
self._absent.update(column for column in columns if column.name not in by_name)
|
|
619
|
+
typed = tuple(
|
|
620
|
+
replace(
|
|
621
|
+
column, binary_text=self._is_binary_text(by_name[column.name]),
|
|
622
|
+
comparison_type=self._comparison_type(by_name[column.name]),
|
|
623
|
+
identity_comparison=self._identity_comparison(by_name[column.name]),
|
|
624
|
+
exact_numeric=self._exact_numeric(by_name[column.name]),
|
|
625
|
+
)
|
|
626
|
+
if column.name in by_name else column for column in columns
|
|
627
|
+
)
|
|
628
|
+
for column in typed:
|
|
629
|
+
kind = by_name[column.name].args.get("kind") if column.name in by_name else None
|
|
630
|
+
if isinstance(kind, exp.DataType):
|
|
631
|
+
self._declared_types[column] = kind
|
|
632
|
+
return typed, complete
|
|
633
|
+
|
|
634
|
+
def _comparison_type(self, definition: exp.ColumnDef) -> str | None:
|
|
635
|
+
kind = definition.args.get("kind")
|
|
636
|
+
if not isinstance(kind, exp.DataType):
|
|
637
|
+
return None
|
|
638
|
+
if kind.this in exp.DataType.TEXT_TYPES:
|
|
639
|
+
return kind.sql(dialect=self.dialect) if self._is_binary_text(definition) else None
|
|
640
|
+
if _collated(definition) or self._unscaled_decimal(kind):
|
|
641
|
+
return None
|
|
642
|
+
return kind.sql(dialect=self.dialect)
|
|
643
|
+
|
|
644
|
+
def _identity_comparison(self, definition: exp.ColumnDef) -> bool:
|
|
645
|
+
"""Declared with a type whose equal values are identical (text only
|
|
646
|
+
when compared by identity, which `_comparison_type` already checks)."""
|
|
647
|
+
kind = definition.args.get("kind")
|
|
648
|
+
if not (self._comparison_type(definition) is not None and isinstance(kind, exp.DataType)):
|
|
649
|
+
return False
|
|
650
|
+
if kind.this == exp.DataType.Type.TIMESTAMPTZ:
|
|
651
|
+
return self.semantics.zoned_timestamp_is_instant
|
|
652
|
+
return kind.this in _IDENTITY_TYPES
|
|
653
|
+
|
|
654
|
+
def _exact_numeric(self, definition: exp.ColumnDef) -> bool:
|
|
655
|
+
kind = definition.args.get("kind")
|
|
656
|
+
return isinstance(kind, exp.DataType) and kind.this in _EXACT_NUMERIC_TYPES and not self._unscaled_decimal(kind)
|
|
657
|
+
|
|
658
|
+
def _unscaled_decimal(self, kind: exp.DataType) -> bool:
|
|
659
|
+
"""A Postgres NUMERIC with no scale: 1.0 and 1.00 are equal values that differ."""
|
|
660
|
+
return self.semantics.unscaled_decimal and kind.this == exp.DataType.Type.DECIMAL and not kind.expressions
|
|
661
|
+
|
|
662
|
+
def _is_binary_text(self, definition: exp.ColumnDef) -> bool:
|
|
663
|
+
"""Declared VARCHAR or TEXT, with no COLLATE, in a dialect whose
|
|
664
|
+
default collation is binary: compared by identity.
|
|
665
|
+
|
|
666
|
+
CHAR is not: it ignores trailing spaces when compared. Any COLLATE,
|
|
667
|
+
even a binary one, is not modelled (issue 49)."""
|
|
668
|
+
kind = definition.args.get("kind")
|
|
669
|
+
return (
|
|
670
|
+
isinstance(kind, exp.DataType) and kind.this in _TEXT_TYPES
|
|
671
|
+
and not _collated(definition) and self.semantics.text_is_binary
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
def _opaque_output_columns(self, names: Iterable[str], origin: Origin) -> tuple[ir.OutputColumn, ...]:
|
|
675
|
+
if isinstance(names, Set):
|
|
676
|
+
names = sorted(names, key=lambda name: (name.lower(), name))
|
|
677
|
+
columns = tuple(
|
|
678
|
+
ir.OutputColumn(
|
|
679
|
+
origin=origin,
|
|
680
|
+
name=name,
|
|
681
|
+
scalar_expr=ir.OpaqueExpression(origin=origin, description=f"source column {name}"),
|
|
682
|
+
)
|
|
683
|
+
for name in names
|
|
684
|
+
)
|
|
685
|
+
self._guessed.update(columns)
|
|
686
|
+
return columns
|
|
687
|
+
|
|
688
|
+
def _recursive_relation(self, symbol: _Symbol, source: Expr) -> ir.RecursiveRelation:
|
|
689
|
+
origin = self._origin(source)
|
|
690
|
+
return ir.RecursiveRelation(
|
|
691
|
+
origin=origin,
|
|
692
|
+
output_columns=self._opaque_output_columns(symbol.required_columns, origin),
|
|
693
|
+
is_schema_complete=False,
|
|
694
|
+
description=source.sql(dialect=self.dialect),
|
|
695
|
+
relation_name=symbol.name,
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
def _alias(self, source: ir.RelationExpr, name: str, origin: Origin, schema: str = "") -> _Lowered:
|
|
699
|
+
output_columns = _pass_output_columns(source.output_columns, origin)
|
|
700
|
+
alias = ir.Alias(origin=origin, output_columns=output_columns, is_schema_complete=source.is_schema_complete, source=source, name=name)
|
|
701
|
+
return _Lowered(alias, tuple(_Binding(name.lower(), output.name.lower(), output, schema) for output in output_columns))
|
|
702
|
+
|
|
703
|
+
def _lower_join(self, left: _Lowered, join: exp.Join) -> _Lowered:
|
|
704
|
+
right = self._lower_source(join.this)
|
|
705
|
+
marked = assertion_span(join)
|
|
706
|
+
origin = self._origin(join, marked.line if marked is not None else None)
|
|
707
|
+
kind = _join_kind(join, self.dialect)
|
|
708
|
+
if kind is None:
|
|
709
|
+
return self._opaque_join(left, right, join, origin)
|
|
710
|
+
predicate = self._lower_join_predicate(join, left, right)
|
|
711
|
+
if kind == "semi" or kind == "anti":
|
|
712
|
+
# RIGHT SEMI and RIGHT ANTI keep right rows: the same join, inputs swapped.
|
|
713
|
+
kept, other = (right, left) if (join.args.get("side") or "").lower() == "right" else (left, right)
|
|
714
|
+
filtered = self._filtering_join(kept, other.relation, kind, predicate, origin)
|
|
715
|
+
assert isinstance(filtered.relation, ir.Join)
|
|
716
|
+
self._join_marker(join, filtered.relation)
|
|
717
|
+
self._add_attachment_site(join, _join_label(join, self.dialect), filtered.relation, trailing=False)
|
|
718
|
+
return filtered
|
|
719
|
+
if _is_positional(join):
|
|
720
|
+
# A FULL JOIN on row position: an opaque condition, as no column holds it.
|
|
721
|
+
predicate = ir.OpaqueExpression(origin=origin, description="row position")
|
|
722
|
+
input_columns = left.relation.output_columns + right.relation.output_columns
|
|
723
|
+
output_columns = _pass_output_columns(input_columns, origin)
|
|
724
|
+
remapped = {source: output for source, output in zip(input_columns, output_columns)}
|
|
725
|
+
relation = ir.Join(
|
|
726
|
+
origin=origin,
|
|
727
|
+
output_columns=output_columns,
|
|
728
|
+
is_schema_complete=left.relation.is_schema_complete and right.relation.is_schema_complete,
|
|
729
|
+
kind=kind,
|
|
730
|
+
left=left.relation,
|
|
731
|
+
right=right.relation,
|
|
732
|
+
predicate=predicate,
|
|
733
|
+
)
|
|
734
|
+
bindings = tuple(
|
|
735
|
+
replace(binding, column=remapped[binding.column])
|
|
736
|
+
for binding in left.bindings + right.bindings
|
|
737
|
+
)
|
|
738
|
+
self._join_marker(join, relation)
|
|
739
|
+
self._add_attachment_site(join, _join_label(join, self.dialect), relation, trailing=False)
|
|
740
|
+
return _Lowered(relation, bindings)
|
|
741
|
+
|
|
742
|
+
def _opaque_join(self, left: _Lowered, right: _Lowered, join: exp.Join, origin: Origin) -> _Lowered:
|
|
743
|
+
"""A join whose kind is not modelled: every column it outputs is opaque,
|
|
744
|
+
and a marker on it is reported as unanalyzed."""
|
|
745
|
+
description = join.sql(dialect=self.dialect)
|
|
746
|
+
input_columns = left.relation.output_columns + right.relation.output_columns
|
|
747
|
+
opaque = {
|
|
748
|
+
column: ir.OutputColumn(origin=origin, name=column.name, scalar_expr=ir.OpaqueExpression(origin=origin, description=description))
|
|
749
|
+
for column in input_columns
|
|
750
|
+
}
|
|
751
|
+
relation = ir.OpaqueRelation(origin=origin, output_columns=tuple(opaque.values()), is_schema_complete=False, description=description)
|
|
752
|
+
return _Lowered(relation, tuple(
|
|
753
|
+
replace(binding, column=opaque[binding.column]) for binding in left.bindings + right.bindings
|
|
754
|
+
))
|
|
755
|
+
|
|
756
|
+
def _join_marker(self, join: exp.Join, relation: ir.Join) -> None:
|
|
757
|
+
kind = join_marker_kind(join)
|
|
758
|
+
if kind is not None:
|
|
759
|
+
self._handled_joins.add(id(join))
|
|
760
|
+
cardinality = {"unique-join": UniqueJoin, "many-to-one-join": ManyToOneJoin, "one-to-many-join": OneToManyJoin, "one-to-one-join": OneToOneJoin}
|
|
761
|
+
if kind in cardinality and relation.kind in {"semi", "anti"}:
|
|
762
|
+
# It keeps or drops left rows whole and outputs no right row: cardinality does not apply.
|
|
763
|
+
self._diagnostics.append(Diagnostic(
|
|
764
|
+
diag.CARDINALITY_ON_FILTERING_JOIN,
|
|
765
|
+
f"the {JOIN_MARKER_NOUNS[kind]} marker on line {join_origin(join, self.dialect).line} is on a {relation.kind} join, "
|
|
766
|
+
"which outputs no right rows: join cardinality does not apply to it",
|
|
767
|
+
join_origin(join, self.dialect),
|
|
768
|
+
))
|
|
769
|
+
return
|
|
770
|
+
match kind:
|
|
771
|
+
case "unique-join" | "many-to-one-join" | "one-to-many-join" | "one-to-one-join":
|
|
772
|
+
self._assertions.append(ir.Assertion(origin=join_origin(join, self.dialect), property=cardinality[kind](join=relation)))
|
|
773
|
+
case "exactly-one-join":
|
|
774
|
+
self._assertions.append(
|
|
775
|
+
ir.Assertion(origin=join_origin(join, self.dialect), property=ExactlyOneJoin(join=relation))
|
|
776
|
+
)
|
|
777
|
+
case None:
|
|
778
|
+
pass
|
|
779
|
+
case _:
|
|
780
|
+
assert_never(kind)
|
|
781
|
+
|
|
782
|
+
def _lower_join_predicate(self, join: exp.Join, left: _Lowered, right: _Lowered) -> ir.ScalarExpr | None:
|
|
783
|
+
"""The ON condition; USING is an AND of one equality per column."""
|
|
784
|
+
using = join.args.get("using")
|
|
785
|
+
if using:
|
|
786
|
+
return _and(tuple(
|
|
787
|
+
ir.Equality(
|
|
788
|
+
origin=self._origin(identifier, _source_line(identifier)),
|
|
789
|
+
left=self._lower_expression(exp.column(identifier.name), left.bindings),
|
|
790
|
+
right=self._lower_expression(exp.column(identifier.name), right.bindings),
|
|
791
|
+
)
|
|
792
|
+
for identifier in using
|
|
793
|
+
), self._origin(join))
|
|
794
|
+
predicate = join.args.get("on")
|
|
795
|
+
if predicate is None:
|
|
796
|
+
return None
|
|
797
|
+
complete = left.relation.is_schema_complete and right.relation.is_schema_complete
|
|
798
|
+
return self._lower_condition(predicate, _conjuncts(predicate), left.bindings + right.bindings, schema_complete=complete)
|
|
799
|
+
|
|
800
|
+
def _lower_condition(
|
|
801
|
+
self, source: Expr, conjuncts: list[Expr], bindings: tuple[_Binding, ...], *, schema_complete: bool
|
|
802
|
+
) -> ir.ScalarExpr:
|
|
803
|
+
"""The AND of `conjuncts`, the top-level conjuncts of `source`."""
|
|
804
|
+
assert conjuncts, "a condition has at least one conjunct"
|
|
805
|
+
lowered = _and(
|
|
806
|
+
tuple(self._lower_conjunct(conjunct, bindings, schema_complete=schema_complete) for conjunct in conjuncts),
|
|
807
|
+
self._origin(source, _source_line(source)),
|
|
808
|
+
)
|
|
809
|
+
assert lowered is not None
|
|
810
|
+
return lowered
|
|
811
|
+
|
|
812
|
+
def _lower_conjunct(self, conjunct: Expr, bindings: tuple[_Binding, ...], *, schema_complete: bool) -> ir.ScalarExpr:
|
|
813
|
+
return self._lower_expression(conjunct, bindings, schema_complete=schema_complete, line=_source_line(conjunct))
|
|
814
|
+
|
|
815
|
+
def _lower_select_tail(self, lowered: _Lowered, select: exp.Select) -> ir.RelationExpr:
|
|
816
|
+
keys = grouping(select)
|
|
817
|
+
if keys is not None:
|
|
818
|
+
relation = self._lower_aggregate(lowered, select, keys)
|
|
819
|
+
elif select.args.get("distinct") is not None and select.args["distinct"].args.get("on") is not None:
|
|
820
|
+
relation = self._lower_distinct_on(lowered, select, select.args["distinct"].args["on"])
|
|
821
|
+
elif select.args.get("distinct") is not None:
|
|
822
|
+
relation = self._lower_distinct(lowered, select)
|
|
823
|
+
elif isinstance(select.args.get("qualify"), exp.Qualify):
|
|
824
|
+
relation = self._lower_qualify(lowered, select, select.args["qualify"])
|
|
825
|
+
else:
|
|
826
|
+
if select.args.get("where") is not None:
|
|
827
|
+
lowered = self._filter(lowered, select.args["where"])
|
|
828
|
+
if _star_only(select):
|
|
829
|
+
relation = lowered.relation
|
|
830
|
+
else:
|
|
831
|
+
lowered, _ = self._join_scalar_subqueries(lowered, select.expressions)
|
|
832
|
+
relation = self._project(lowered, select.expressions, self._origin(select))
|
|
833
|
+
if relation is None:
|
|
834
|
+
return self._opaque_relation(select, _selected_output_names(select))
|
|
835
|
+
self._sort_keys[id(select)] = self._resolve_order(select, lowered.bindings, relation)
|
|
836
|
+
return relation
|
|
837
|
+
|
|
838
|
+
def _resolve_order(
|
|
839
|
+
self, select: exp.Select, bindings: tuple[_Binding, ...], relation: ir.RelationExpr
|
|
840
|
+
) -> tuple[ir.OutputColumn, ...] | None:
|
|
841
|
+
"""Each ORDER BY key as a column of the stage it sorts (see
|
|
842
|
+
`Program.sort_keys`): a selected column by its name or position, else
|
|
843
|
+
the column a name binds, qualifier and schema included."""
|
|
844
|
+
order = select.args.get("order")
|
|
845
|
+
if order is None:
|
|
846
|
+
return ()
|
|
847
|
+
distinct = select.args.get("distinct")
|
|
848
|
+
if distinct is not None and distinct.args.get("on") is not None:
|
|
849
|
+
return None # Its ORDER BY picks the row each group keeps.
|
|
850
|
+
scope = relation.input if isinstance(relation, ir.Project) else relation
|
|
851
|
+
outputs = relation.output_columns
|
|
852
|
+
items = [item for item in select.expressions if not isinstance(item, exp.Star)]
|
|
853
|
+
keys: list[ir.OutputColumn] = []
|
|
854
|
+
for ordered in order.expressions:
|
|
855
|
+
expression = ordered.this if isinstance(ordered, exp.Ordered) else ordered
|
|
856
|
+
key: ir.OutputColumn | None = None
|
|
857
|
+
match expression:
|
|
858
|
+
case exp.Column(table="") if len(items) == len(outputs) and [item.alias_or_name.lower() for item in items].count(expression.name.lower()) == 1:
|
|
859
|
+
names = [item.alias_or_name.lower() for item in items]
|
|
860
|
+
key = _in_scope(outputs[names.index(expression.name.lower())], scope)
|
|
861
|
+
case exp.Literal(is_int=True) if len(items) == len(outputs) and 1 <= int(expression.this) <= len(outputs):
|
|
862
|
+
key = _in_scope(outputs[int(expression.this) - 1], scope)
|
|
863
|
+
case exp.Column():
|
|
864
|
+
schema, name = _schema_path(expression), expression.name.lower()
|
|
865
|
+
bound = [
|
|
866
|
+
binding for binding in bindings
|
|
867
|
+
if binding.name == name and (not expression.table or binding.qualifier == expression.table.lower())
|
|
868
|
+
and (not schema or binding.schema == schema)
|
|
869
|
+
]
|
|
870
|
+
if len(bound) == 1:
|
|
871
|
+
key = _carrying(bound[0].column, scope)
|
|
872
|
+
case _:
|
|
873
|
+
pass
|
|
874
|
+
if key is None:
|
|
875
|
+
return None
|
|
876
|
+
keys.append(key)
|
|
877
|
+
return tuple(keys)
|
|
878
|
+
|
|
879
|
+
def _filter(self, lowered: _Lowered, where: exp.Where) -> _Lowered:
|
|
880
|
+
"""WHERE: each `EXISTS`, `NOT EXISTS`, or `IN (subquery)` conjunct as a
|
|
881
|
+
semi or anti join (Paulley §4.4.2), the other conjuncts as one Filter."""
|
|
882
|
+
remaining = []
|
|
883
|
+
for conjunct in _conjuncts(where.this):
|
|
884
|
+
joined = self._subquery_join(lowered, conjunct)
|
|
885
|
+
if joined is None:
|
|
886
|
+
remaining.append(conjunct)
|
|
887
|
+
else:
|
|
888
|
+
lowered = joined
|
|
889
|
+
if not remaining:
|
|
890
|
+
return lowered
|
|
891
|
+
before = lowered.relation.output_columns
|
|
892
|
+
lowered, carried = self._join_scalar_subqueries(lowered, remaining)
|
|
893
|
+
origin = self._origin(where)
|
|
894
|
+
output_columns = _pass_output_columns(lowered.relation.output_columns, origin)
|
|
895
|
+
remapped = {source: output for source, output in zip(lowered.relation.output_columns, output_columns)}
|
|
896
|
+
complete = lowered.relation.is_schema_complete
|
|
897
|
+
relation = ir.Filter(
|
|
898
|
+
origin=origin, output_columns=output_columns, is_schema_complete=complete, input=lowered.relation,
|
|
899
|
+
predicate=self._lower_condition(where.this, remaining, lowered.bindings, schema_complete=complete),
|
|
900
|
+
)
|
|
901
|
+
bindings = tuple(
|
|
902
|
+
replace(binding, column=remapped[binding.column]) for binding in lowered.bindings
|
|
903
|
+
)
|
|
904
|
+
if all(carried[column] is column for column in before):
|
|
905
|
+
return _Lowered(relation, bindings)
|
|
906
|
+
# Scalar subqueries were joined in: keep only the columns from before,
|
|
907
|
+
# so their values stay inside this WHERE.
|
|
908
|
+
kept = tuple(remapped[carried[column]] for column in before)
|
|
909
|
+
project = ir.Project(
|
|
910
|
+
origin=origin, output_columns=_pass_output_columns(kept, origin), is_schema_complete=complete, input=relation,
|
|
911
|
+
)
|
|
912
|
+
projected = dict(zip(kept, project.output_columns))
|
|
913
|
+
return _Lowered(project, tuple(
|
|
914
|
+
replace(binding, column=projected[binding.column])
|
|
915
|
+
for binding in bindings if binding.column in projected
|
|
916
|
+
))
|
|
917
|
+
|
|
918
|
+
def _correlated(self, select: exp.Select, inner: _Lowered, outer: _Lowered) -> bool | None:
|
|
919
|
+
"""Whether `select` reads the outer scope, only in its WHERE; None when
|
|
920
|
+
it reads it anywhere else, or when a column may belong to either scope."""
|
|
921
|
+
where = select.args.get("where")
|
|
922
|
+
correlated = False
|
|
923
|
+
for column in select.find_all(exp.Column):
|
|
924
|
+
found = self._scope_of(column, inner, outer)
|
|
925
|
+
if found is None:
|
|
926
|
+
return None
|
|
927
|
+
scope, binding = found
|
|
928
|
+
self._scoped_columns[id(column)] = binding.column
|
|
929
|
+
if scope == "outer":
|
|
930
|
+
if where is None or column.find_ancestor(exp.Where) is not where:
|
|
931
|
+
return None # Only a WHERE becomes the dependent join's ON.
|
|
932
|
+
correlated = True
|
|
933
|
+
return correlated
|
|
934
|
+
|
|
935
|
+
def _scope_of(
|
|
936
|
+
self, column: exp.Column, inner: _Lowered, outer: _Lowered
|
|
937
|
+
) -> tuple[Literal["inner", "outer"], _Binding] | None:
|
|
938
|
+
"""The scope a subquery's column belongs to, the innermost that has it,
|
|
939
|
+
and its binding there. None when that is unknown (a table with no
|
|
940
|
+
CREATE may have the column) or the name binds more than one column."""
|
|
941
|
+
name = column.name.lower()
|
|
942
|
+
|
|
943
|
+
def one(scope: Literal["inner", "outer"], bindings: list[_Binding]) -> tuple[Literal["inner", "outer"], _Binding] | None:
|
|
944
|
+
return (scope, bindings[0]) if len(bindings) == 1 else None
|
|
945
|
+
|
|
946
|
+
if column.table:
|
|
947
|
+
qualifier, schema = column.table.lower(), _schema_path(column)
|
|
948
|
+
for scope, bindings in (("inner", inner.bindings), ("outer", outer.bindings)):
|
|
949
|
+
if any(binding.qualifier == qualifier and (not schema or binding.schema == schema) for binding in bindings):
|
|
950
|
+
return one(scope, [
|
|
951
|
+
binding for binding in bindings
|
|
952
|
+
if binding.qualifier == qualifier and (not schema or binding.schema == schema) and binding.name == name
|
|
953
|
+
])
|
|
954
|
+
return None
|
|
955
|
+
inner_named = [binding for binding in inner.bindings if binding.name == name and self._owns(binding) != "no"]
|
|
956
|
+
outer_named = [binding for binding in outer.bindings if binding.name == name and self._owns(binding) != "no"]
|
|
957
|
+
owners = {self._owns(binding) for binding in inner_named}
|
|
958
|
+
if "yes" in owners:
|
|
959
|
+
return one("inner", inner_named)
|
|
960
|
+
outer_has = bool(outer_named)
|
|
961
|
+
if "unknown" in owners or (not inner_named and not inner.relation.is_schema_complete and not any(binding.name == name for binding in inner.bindings)):
|
|
962
|
+
if not outer_has:
|
|
963
|
+
return one("inner", inner_named)
|
|
964
|
+
self._warnings.append(Diagnostic(
|
|
965
|
+
diag.AMBIGUOUS_COLUMN,
|
|
966
|
+
f"column {column.name} in a subquery may belong to its own FROM or to the outer query, "
|
|
967
|
+
"since some of its tables have no declared columns: qualify it, declare those tables with CREATE TABLE, "
|
|
968
|
+
"or analyze with a database connection (analyze_database)",
|
|
969
|
+
self._origin(column, _source_line(column)),
|
|
970
|
+
))
|
|
971
|
+
return None
|
|
972
|
+
return one("outer", outer_named) if outer_has else None
|
|
973
|
+
|
|
974
|
+
def _owns(self, binding: _Binding) -> Literal["yes", "no", "unknown"]:
|
|
975
|
+
"""Whether the relation behind `binding` has its column: a declared or
|
|
976
|
+
computed column it has, one its declared table lacks it does not, and
|
|
977
|
+
one made up for a table with no CREATE it may have."""
|
|
978
|
+
base = binding.column
|
|
979
|
+
while isinstance(base.scalar_expr, ir.ColumnRef):
|
|
980
|
+
base = base.scalar_expr.column
|
|
981
|
+
if base in self._absent:
|
|
982
|
+
return "no"
|
|
983
|
+
return "unknown" if base in self._guessed else "yes"
|
|
984
|
+
|
|
985
|
+
def _join_scalar_subqueries(
|
|
986
|
+
self, lowered: _Lowered, expressions: Sequence[Expr]
|
|
987
|
+
) -> tuple[_Lowered, dict[ir.OutputColumn, ir.OutputColumn]]:
|
|
988
|
+
"""Each scalar subquery in `expressions` joined onto `lowered`, and
|
|
989
|
+
where each of `lowered`'s columns went."""
|
|
990
|
+
carried = {column: column for column in lowered.relation.output_columns}
|
|
991
|
+
for expression in expressions:
|
|
992
|
+
for subquery in _scalar_subqueries(expression):
|
|
993
|
+
joined = self._scalar_subquery_join(lowered, subquery)
|
|
994
|
+
if joined is None:
|
|
995
|
+
continue
|
|
996
|
+
lowered, remapped = joined
|
|
997
|
+
carried = {column: remapped[current] for column, current in carried.items()}
|
|
998
|
+
return lowered, carried
|
|
999
|
+
|
|
1000
|
+
def _scalar_subquery_join(
|
|
1001
|
+
self, lowered: _Lowered, subquery: exp.Subquery
|
|
1002
|
+
) -> tuple[_Lowered, dict[ir.OutputColumn, ir.OutputColumn]] | None:
|
|
1003
|
+
"""A scalar subquery with body B as a left join of `lowered` with B: no
|
|
1004
|
+
row makes it NULL, which is a left join's padding. More than one row
|
|
1005
|
+
is a runtime error, so the join is many to one, or the query fails.
|
|
1006
|
+
|
|
1007
|
+
Uncorrelated, B is lowered whole under an implied LIMIT 1, sound since
|
|
1008
|
+
it changes the result only when the query would fail: its value is
|
|
1009
|
+
the same on every row. Correlated, B's WHERE becomes the ON of a
|
|
1010
|
+
dependent join (as for EXISTS), so only a plain select-from-where
|
|
1011
|
+
whose value is one of its columns is lowered. Anything else, or a
|
|
1012
|
+
column that may belong to either scope, stays opaque."""
|
|
1013
|
+
select = subquery.this
|
|
1014
|
+
if not isinstance(select, exp.Select) or len(select.expressions) != 1 or _star_only(select):
|
|
1015
|
+
return None
|
|
1016
|
+
source = _from_source(select)
|
|
1017
|
+
if source is None or select.args.get("with") or any(
|
|
1018
|
+
join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()
|
|
1019
|
+
) or any(isinstance(node, (exp.Subquery, exp.Exists, exp.Query)) and node is not select for node in select.walk()):
|
|
1020
|
+
return None
|
|
1021
|
+
right = self._lower_source(source)
|
|
1022
|
+
for join in select.args.get("joins") or ():
|
|
1023
|
+
right = self._lower_join(right, join)
|
|
1024
|
+
correlated = self._correlated(select, right, lowered)
|
|
1025
|
+
origin = self._origin(subquery)
|
|
1026
|
+
if correlated is None:
|
|
1027
|
+
return None
|
|
1028
|
+
if not correlated:
|
|
1029
|
+
body = self._finish_select(self._lower_select_tail(right, select), select)
|
|
1030
|
+
right_relation: ir.RelationExpr = ir.Limit(
|
|
1031
|
+
origin=origin, output_columns=_pass_output_columns(body.output_columns, origin),
|
|
1032
|
+
is_schema_complete=body.is_schema_complete, input=body, count=1,
|
|
1033
|
+
)
|
|
1034
|
+
value = right_relation.output_columns[0]
|
|
1035
|
+
predicate = None
|
|
1036
|
+
else:
|
|
1037
|
+
if grouping(select) is not None or any(select.args.get(key) for key in ("having", "qualify", "distinct", "limit", "offset", "windows")):
|
|
1038
|
+
return None
|
|
1039
|
+
item = select.expressions[0]
|
|
1040
|
+
lowered_value = self._lower_expression(
|
|
1041
|
+
item.this if isinstance(item, exp.Alias) else item, right.bindings, schema_complete=right.relation.is_schema_complete,
|
|
1042
|
+
)
|
|
1043
|
+
if not (isinstance(lowered_value, ir.ColumnRef) and lowered_value.column in right.relation.output_columns):
|
|
1044
|
+
return None
|
|
1045
|
+
inner = select.args.get("where")
|
|
1046
|
+
predicate = None if inner is None else self._lower_condition(
|
|
1047
|
+
inner.this, _conjuncts(inner.this), lowered.bindings + right.bindings,
|
|
1048
|
+
schema_complete=lowered.relation.is_schema_complete and right.relation.is_schema_complete,
|
|
1049
|
+
)
|
|
1050
|
+
right_relation, value = right.relation, lowered_value.column
|
|
1051
|
+
input_columns = lowered.relation.output_columns + right_relation.output_columns
|
|
1052
|
+
output_columns = _pass_output_columns(input_columns, origin)
|
|
1053
|
+
remapped = dict(zip(input_columns, output_columns))
|
|
1054
|
+
join = ir.Join(
|
|
1055
|
+
origin=origin, output_columns=output_columns,
|
|
1056
|
+
is_schema_complete=lowered.relation.is_schema_complete and right_relation.is_schema_complete,
|
|
1057
|
+
kind="left", left=lowered.relation, right=right_relation, predicate=predicate, scalar_subquery=True,
|
|
1058
|
+
)
|
|
1059
|
+
position = marker_position(select)
|
|
1060
|
+
if position is not None:
|
|
1061
|
+
self._scalar_subqueries.append((AttachmentSite("scalar subquery", position[1], position[0], True), join))
|
|
1062
|
+
qualifier = f"{_SCALAR_QUALIFIER}{len(self._scalar_values)}"
|
|
1063
|
+
self._scalar_values[id(subquery)] = qualifier
|
|
1064
|
+
bindings = (
|
|
1065
|
+
*(replace(binding, column=remapped[binding.column]) for binding in lowered.bindings),
|
|
1066
|
+
_Binding(qualifier, _SCALAR_QUALIFIER, remapped[value]),
|
|
1067
|
+
)
|
|
1068
|
+
return _Lowered(join, bindings), remapped
|
|
1069
|
+
|
|
1070
|
+
def _subquery_join(self, lowered: _Lowered, conjunct: Expr) -> _Lowered | None:
|
|
1071
|
+
"""`EXISTS (SELECT ... FROM r WHERE p)` as a semi join of r on p, and
|
|
1072
|
+
`NOT EXISTS` as an anti join; `p` may read the outer row (a dependent
|
|
1073
|
+
join, so the IR stays a DAG). `x IN (SELECT a ...)` as a semi join on
|
|
1074
|
+
`x = a`. Not `NOT IN`: a NULL among the values makes it reject every row."""
|
|
1075
|
+
matched = _subquery_conjunct(conjunct)
|
|
1076
|
+
if matched is None:
|
|
1077
|
+
return None
|
|
1078
|
+
kind, select, operand = matched
|
|
1079
|
+
if operand is None:
|
|
1080
|
+
source = _from_source(select)
|
|
1081
|
+
if source is None or grouping(select) is not None or any(
|
|
1082
|
+
select.args.get(key) for key in ("having", "qualify", "distinct", "limit", "offset", "with", "windows")
|
|
1083
|
+
) or any(join.args.get("method") == "NATURAL" for join in select.args.get("joins") or ()):
|
|
1084
|
+
# Grouped by nothing (`grouping` is []), it returns one row whatever its WHERE: EXISTS is always true.
|
|
1085
|
+
return None
|
|
1086
|
+
right = self._lower_source(source)
|
|
1087
|
+
for join in select.args.get("joins") or ():
|
|
1088
|
+
right = self._lower_join(right, join)
|
|
1089
|
+
inner = select.args.get("where")
|
|
1090
|
+
predicate = None if inner is None else self._lower_condition(
|
|
1091
|
+
inner.this, _conjuncts(inner.this), lowered.bindings + right.bindings,
|
|
1092
|
+
schema_complete=lowered.relation.is_schema_complete and right.relation.is_schema_complete,
|
|
1093
|
+
)
|
|
1094
|
+
relation = right.relation
|
|
1095
|
+
else:
|
|
1096
|
+
subquery = self._lower_query(select)
|
|
1097
|
+
if subquery is None or len(subquery.output_columns) != 1:
|
|
1098
|
+
return None
|
|
1099
|
+
relation = subquery
|
|
1100
|
+
predicate = ir.Equality(
|
|
1101
|
+
origin=self._origin(conjunct, _source_line(conjunct)),
|
|
1102
|
+
left=self._lower_expression(operand, lowered.bindings, schema_complete=lowered.relation.is_schema_complete),
|
|
1103
|
+
right=ir.ColumnRef(origin=subquery.origin, column=subquery.output_columns[0]),
|
|
1104
|
+
)
|
|
1105
|
+
return self._filtering_join(lowered, relation, kind, predicate, self._origin(conjunct))
|
|
1106
|
+
|
|
1107
|
+
def _filtering_join(
|
|
1108
|
+
self, left: _Lowered, right: ir.RelationExpr, kind: ir.JoinKind, predicate: ir.ScalarExpr | None, origin: Origin
|
|
1109
|
+
) -> _Lowered:
|
|
1110
|
+
"""A semi or anti join: it keeps or drops left rows, and outputs only theirs."""
|
|
1111
|
+
output_columns = _pass_output_columns(left.relation.output_columns, origin)
|
|
1112
|
+
remapped = {source: output for source, output in zip(left.relation.output_columns, output_columns)}
|
|
1113
|
+
relation = ir.Join(
|
|
1114
|
+
origin=origin, output_columns=output_columns, is_schema_complete=left.relation.is_schema_complete,
|
|
1115
|
+
kind=kind, left=left.relation, right=right, predicate=predicate,
|
|
1116
|
+
)
|
|
1117
|
+
return _Lowered(relation, tuple(replace(binding, column=remapped[binding.column]) for binding in left.bindings))
|
|
1118
|
+
|
|
1119
|
+
def _project(self, lowered: _Lowered, items: list[Expr], origin: Origin) -> ir.Project:
|
|
1120
|
+
output_columns = tuple(
|
|
1121
|
+
ir.OutputColumn(
|
|
1122
|
+
origin=self._origin(item),
|
|
1123
|
+
name=item.alias_or_name,
|
|
1124
|
+
scalar_expr=self._lower_expression(
|
|
1125
|
+
item.this if isinstance(item, exp.Alias) else item,
|
|
1126
|
+
lowered.bindings,
|
|
1127
|
+
schema_complete=lowered.relation.is_schema_complete,
|
|
1128
|
+
),
|
|
1129
|
+
)
|
|
1130
|
+
for item in items
|
|
1131
|
+
)
|
|
1132
|
+
return ir.Project(origin=origin, output_columns=output_columns, is_schema_complete=True, input=lowered.relation)
|
|
1133
|
+
|
|
1134
|
+
def _lower_aggregate(self, lowered: _Lowered, select: exp.Select, keys: list[Expr]) -> ir.Aggregate | None:
|
|
1135
|
+
"""`select` grouped by `keys`; [] is one group, one row however many rows its input has."""
|
|
1136
|
+
group = select.args.get("group")
|
|
1137
|
+
if select.args.get("having") is not None or (
|
|
1138
|
+
group is not None and any(group.args.get(key) for key in ("grouping_sets", "rollup", "cube", "totals"))
|
|
1139
|
+
):
|
|
1140
|
+
return None
|
|
1141
|
+
if select.args.get("where") is not None:
|
|
1142
|
+
lowered = self._filter(lowered, select.args["where"])
|
|
1143
|
+
|
|
1144
|
+
grouped_items: set[int] = set()
|
|
1145
|
+
for group_expression in keys:
|
|
1146
|
+
item = _matching_grouping_output(group_expression, select.expressions)
|
|
1147
|
+
if item is None:
|
|
1148
|
+
return None
|
|
1149
|
+
grouped_items.add(id(item))
|
|
1150
|
+
|
|
1151
|
+
output_columns = self._aggregate_output_columns(select.expressions, lowered, grouped_items)
|
|
1152
|
+
grouping_outputs: list[ir.OutputColumn] = []
|
|
1153
|
+
for group_expression in keys:
|
|
1154
|
+
item = _matching_grouping_output(group_expression, select.expressions)
|
|
1155
|
+
grouping_outputs.append(output_columns[select.expressions.index(item)])
|
|
1156
|
+
calls = tuple(
|
|
1157
|
+
ir.AggregateCall(
|
|
1158
|
+
origin=self._origin(call), description=call.sql(dialect=self.dialect),
|
|
1159
|
+
arguments=tuple(
|
|
1160
|
+
self._lower_expression(argument, lowered.bindings, schema_complete=lowered.relation.is_schema_complete)
|
|
1161
|
+
for argument in _aggregate_arguments(call)
|
|
1162
|
+
),
|
|
1163
|
+
)
|
|
1164
|
+
for item in select.expressions for call in item.find_all(exp.AggFunc)
|
|
1165
|
+
if call.find_ancestor(exp.Window, exp.Select) is select
|
|
1166
|
+
)
|
|
1167
|
+
return ir.Aggregate(
|
|
1168
|
+
origin=self._origin(select), output_columns=output_columns, is_schema_complete=True, input=lowered.relation,
|
|
1169
|
+
grouping_outputs=tuple(grouping_outputs), calls=calls,
|
|
1170
|
+
)
|
|
1171
|
+
|
|
1172
|
+
def _lower_distinct(self, lowered: _Lowered, select: exp.Select) -> ir.Distinct | None:
|
|
1173
|
+
distinct = select.args.get("distinct")
|
|
1174
|
+
if distinct is None or select.args.get("having") is not None or _star_only(select):
|
|
1175
|
+
return None
|
|
1176
|
+
if select.args.get("where") is not None:
|
|
1177
|
+
lowered = self._filter(lowered, select.args["where"])
|
|
1178
|
+
return ir.Distinct(
|
|
1179
|
+
origin=self._origin(select), output_columns=self._selected_output_columns(select.expressions, lowered),
|
|
1180
|
+
is_schema_complete=True, input=lowered.relation,
|
|
1181
|
+
)
|
|
1182
|
+
|
|
1183
|
+
def _lower_qualify(self, lowered: _Lowered, select: exp.Select, qualify: exp.Qualify) -> ir.QualifyByPartition | None:
|
|
1184
|
+
window = _row_number_equals_one(qualify.this)
|
|
1185
|
+
if window is None:
|
|
1186
|
+
return None
|
|
1187
|
+
order = window.args.get("order")
|
|
1188
|
+
return self._one_row_per_partition(
|
|
1189
|
+
lowered, select, window.args.get("partition_by") or (), order.expressions if order else (),
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
def _lower_distinct_on(self, lowered: _Lowered, select: exp.Select, on: Expr) -> ir.QualifyByPartition | None:
|
|
1193
|
+
"""`DISTINCT ON (k...)` keeps one row per distinct `k...`, like
|
|
1194
|
+
`QUALIFY ROW_NUMBER() OVER (PARTITION BY k...) = 1`. Its ORDER BY only
|
|
1195
|
+
picks which row survives, and may name outputs, so it is not lowered."""
|
|
1196
|
+
if select.args.get("qualify") is not None:
|
|
1197
|
+
return None
|
|
1198
|
+
keys = on.expressions if isinstance(on, exp.Tuple) else [on]
|
|
1199
|
+
return self._one_row_per_partition(lowered, select, keys, ())
|
|
1200
|
+
|
|
1201
|
+
def _one_row_per_partition(
|
|
1202
|
+
self, lowered: _Lowered, select: exp.Select, partitions: Sequence[Expr], order: Sequence[exp.Ordered],
|
|
1203
|
+
) -> ir.QualifyByPartition | None:
|
|
1204
|
+
"""Every partition key must be a selected output: the Unique Set is stated on outputs."""
|
|
1205
|
+
if not partitions or select.args.get("having") is not None or _star_only(select):
|
|
1206
|
+
return None
|
|
1207
|
+
if select.args.get("where") is not None:
|
|
1208
|
+
lowered = self._filter(lowered, select.args["where"])
|
|
1209
|
+
|
|
1210
|
+
output_columns = self._selected_output_columns(select.expressions, lowered)
|
|
1211
|
+
partition_outputs: list[ir.OutputColumn] = []
|
|
1212
|
+
for partition in partitions:
|
|
1213
|
+
item = _matching_output(partition, select.expressions)
|
|
1214
|
+
if item is None:
|
|
1215
|
+
return None
|
|
1216
|
+
partition_outputs.append(output_columns[select.expressions.index(item)])
|
|
1217
|
+
ordering = tuple(
|
|
1218
|
+
self._lower_expression(ordered.this, lowered.bindings, schema_complete=lowered.relation.is_schema_complete)
|
|
1219
|
+
for ordered in order
|
|
1220
|
+
)
|
|
1221
|
+
return ir.QualifyByPartition(
|
|
1222
|
+
origin=self._origin(select), output_columns=output_columns, is_schema_complete=True, input=lowered.relation,
|
|
1223
|
+
partition_outputs=tuple(partition_outputs), ordering=ordering,
|
|
1224
|
+
)
|
|
1225
|
+
|
|
1226
|
+
def _selected_output_columns(self, items: list[Expr], lowered: _Lowered) -> tuple[ir.OutputColumn, ...]:
|
|
1227
|
+
return tuple(
|
|
1228
|
+
ir.OutputColumn(
|
|
1229
|
+
origin=self._origin(item),
|
|
1230
|
+
name=item.alias_or_name,
|
|
1231
|
+
scalar_expr=self._lower_expression(
|
|
1232
|
+
item.this if isinstance(item, exp.Alias) else item,
|
|
1233
|
+
lowered.bindings,
|
|
1234
|
+
schema_complete=lowered.relation.is_schema_complete,
|
|
1235
|
+
),
|
|
1236
|
+
)
|
|
1237
|
+
for item in items
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
def _aggregate_output_columns(self, items: list[Expr], source: _Lowered, grouped_items: set[int]) -> tuple[ir.OutputColumn, ...]:
|
|
1241
|
+
output_columns: list[ir.OutputColumn] = []
|
|
1242
|
+
for item in items:
|
|
1243
|
+
expression = item.this if isinstance(item, exp.Alias) else item
|
|
1244
|
+
lowered = self._lower_expression(expression, source.bindings, schema_complete=source.relation.is_schema_complete)
|
|
1245
|
+
if id(item) not in grouped_items and _is_bare_aggregate_expression(expression):
|
|
1246
|
+
# Grouped by nothing (no grouped items), the one row exists even
|
|
1247
|
+
# over no input rows, so a bare column, where a dialect allows
|
|
1248
|
+
# one, may be NULL.
|
|
1249
|
+
lowered = (
|
|
1250
|
+
ir.AnyAggregate(origin=self._origin(expression), input=lowered) if grouped_items
|
|
1251
|
+
else ir.OpaqueExpression(origin=self._origin(expression), description=expression.sql(dialect=self.dialect))
|
|
1252
|
+
)
|
|
1253
|
+
output_columns.append(ir.OutputColumn(origin=self._origin(item), name=item.alias_or_name, scalar_expr=lowered))
|
|
1254
|
+
return tuple(output_columns)
|
|
1255
|
+
|
|
1256
|
+
def _lower_expression(
|
|
1257
|
+
self, expression: Expr, bindings: tuple[_Binding, ...], *, schema_complete: bool = False, line: int | None = None
|
|
1258
|
+
) -> ir.ScalarExpr:
|
|
1259
|
+
"""An unqualified column resolves by name across several sources only
|
|
1260
|
+
when every source's schema is complete: otherwise the one visible match
|
|
1261
|
+
might be shadowed by a column of a source whose columns are unknown.
|
|
1262
|
+
|
|
1263
|
+
A Builtin becomes a Call; any other function or operator an
|
|
1264
|
+
OpaquePureFunction of its lowered arguments, unless its value is not a
|
|
1265
|
+
function of them (`_NOT_A_FUNCTION_OF_ITS_COLUMNS`)."""
|
|
1266
|
+
origin = self._origin(expression, line)
|
|
1267
|
+
|
|
1268
|
+
def lower(node: Expr) -> ir.ScalarExpr:
|
|
1269
|
+
return self._lower_expression(node, bindings, schema_complete=schema_complete)
|
|
1270
|
+
|
|
1271
|
+
if isinstance(expression, exp.Subquery) and id(expression) in self._scalar_values:
|
|
1272
|
+
qualifier = self._scalar_values[id(expression)]
|
|
1273
|
+
value = next((binding.column for binding in bindings if binding.qualifier == qualifier), None)
|
|
1274
|
+
if value is not None:
|
|
1275
|
+
return ir.ColumnRef(origin=origin, column=value)
|
|
1276
|
+
if isinstance(expression, exp.Column) and id(expression) in self._scoped_columns:
|
|
1277
|
+
column = self._scoped_columns[id(expression)]
|
|
1278
|
+
if any(binding.column is column for binding in bindings):
|
|
1279
|
+
return ir.ColumnRef(origin=origin, column=column)
|
|
1280
|
+
if isinstance(expression, exp.Column):
|
|
1281
|
+
if expression.table:
|
|
1282
|
+
schema = _schema_path(expression)
|
|
1283
|
+
matches = [
|
|
1284
|
+
binding for binding in bindings
|
|
1285
|
+
if binding.qualifier == expression.table.lower() and binding.name == expression.name.lower() and (not schema or binding.schema == schema)
|
|
1286
|
+
]
|
|
1287
|
+
elif schema_complete or len({binding.qualifier for binding in bindings if not binding.qualifier.startswith(_SCALAR_QUALIFIER)}) == 1:
|
|
1288
|
+
matches = [binding for binding in bindings if binding.name == expression.name.lower()]
|
|
1289
|
+
else:
|
|
1290
|
+
matches = []
|
|
1291
|
+
sources = ", ".join(dict.fromkeys(binding.qualifier for binding in bindings))
|
|
1292
|
+
self._warnings.append(Diagnostic(
|
|
1293
|
+
diag.AMBIGUOUS_COLUMN,
|
|
1294
|
+
f"column {expression.name} is unqualified and more than one source may own it ({sources}), "
|
|
1295
|
+
"since some have no declared columns: qualify it, declare those tables with CREATE TABLE, "
|
|
1296
|
+
"or analyze with a database connection (analyze_database)",
|
|
1297
|
+
origin,
|
|
1298
|
+
))
|
|
1299
|
+
if len(matches) == 1:
|
|
1300
|
+
return ir.ColumnRef(origin=origin, column=matches[0].column)
|
|
1301
|
+
return ir.OpaqueExpression(origin=origin, description=expression.sql(dialect=self.dialect))
|
|
1302
|
+
if isinstance(expression, _COMPARISONS):
|
|
1303
|
+
self._check_zoned_text(expression, lower, self._origin(expression, line or _source_line(expression)))
|
|
1304
|
+
description = expression.sql(dialect=self.dialect)
|
|
1305
|
+
quantified = isinstance(expression, exp.Binary) and any(
|
|
1306
|
+
isinstance(operand, (exp.Any, exp.All)) for operand in (expression.this, expression.expression)
|
|
1307
|
+
) # `n > ANY (...)` is FALSE over no rows, even for a NULL `n`.
|
|
1308
|
+
match expression:
|
|
1309
|
+
case exp.Paren():
|
|
1310
|
+
return self._lower_expression(expression.this, bindings, schema_complete=schema_complete, line=line)
|
|
1311
|
+
case exp.Null():
|
|
1312
|
+
return ir.NullLiteral(origin=origin)
|
|
1313
|
+
case exp.Count():
|
|
1314
|
+
return ir.Count(origin=origin, description=description)
|
|
1315
|
+
case exp.Coalesce():
|
|
1316
|
+
return ir.Coalesce(origin=origin, arguments=tuple(lower(argument) for argument in (expression.this, *expression.expressions)))
|
|
1317
|
+
case exp.Case() if expression.this is not None:
|
|
1318
|
+
return self._lower_case(expression, bindings, schema_complete=schema_complete)
|
|
1319
|
+
case exp.And():
|
|
1320
|
+
return self._lower_condition(expression, _conjuncts(expression), bindings, schema_complete=schema_complete)
|
|
1321
|
+
case exp.Or() if is_ambiguous_pipes(expression):
|
|
1322
|
+
pipes = f"{expression.this.sql(dialect=self.dialect)} || {expression.expression.sql(dialect=self.dialect)}"
|
|
1323
|
+
self._warnings.append(Diagnostic(
|
|
1324
|
+
diag.AMBIGUOUS_OPERATOR,
|
|
1325
|
+
f"`||` is OR in {self.dialect}, or concatenation under sql_mode PIPES_AS_CONCAT, which also groups "
|
|
1326
|
+
f"its operands differently, so {pipes} is an unknown function of its columns: write OR or CONCAT()",
|
|
1327
|
+
self._origin(expression, line or _source_line(expression)),
|
|
1328
|
+
))
|
|
1329
|
+
return ir.OpaquePureFunction(origin=origin, arguments=tuple(lower(child) for child in expression.iter_expressions()), description=pipes)
|
|
1330
|
+
case exp.Or():
|
|
1331
|
+
return ir.Or(origin=origin, parts=tuple(
|
|
1332
|
+
self._lower_condition(branch, _conjuncts(branch), bindings, schema_complete=schema_complete)
|
|
1333
|
+
for branch in _disjuncts(expression)
|
|
1334
|
+
))
|
|
1335
|
+
case exp.EQ() if not quantified:
|
|
1336
|
+
return ir.Equality(origin=origin, left=lower(expression.this), right=lower(expression.expression))
|
|
1337
|
+
case exp.Is(expression=exp.Null()):
|
|
1338
|
+
return ir.IsNull(origin=origin, operand=lower(expression.this))
|
|
1339
|
+
case exp.Not(this=exp.Is(expression=exp.Null()) as test):
|
|
1340
|
+
return ir.IsNotNull(origin=origin, operand=lower(test.this))
|
|
1341
|
+
case exp.In() if expression.expressions and not any(expression.args.get(key) for key in ("query", "unnest", "field")):
|
|
1342
|
+
return ir.InList(origin=origin, operand=lower(expression.this), values=tuple(lower(value) for value in expression.expressions))
|
|
1343
|
+
case _:
|
|
1344
|
+
pass
|
|
1345
|
+
value = literal_value(expression)
|
|
1346
|
+
if value == ir.LiteralValue(type="string", text="") and self.semantics.empty_string_is_null:
|
|
1347
|
+
return ir.NullLiteral(origin=origin)
|
|
1348
|
+
if value is not None or isinstance(expression, exp.Literal):
|
|
1349
|
+
return ir.Constant(origin=origin, value=value)
|
|
1350
|
+
builtin = dialect_builtins(self.semantics).get(type(expression))
|
|
1351
|
+
if builtin is not None and not quantified:
|
|
1352
|
+
function, keys = builtin
|
|
1353
|
+
if type(expression) is exp.Cast and function.total is not None and self._is_json(expression.this, lower):
|
|
1354
|
+
# `CAST(JSON 'null' AS BIGINT)` is NULL.
|
|
1355
|
+
function = replace(function, total=None)
|
|
1356
|
+
values = [value for key in keys for value in _as_list(expression.args[key])]
|
|
1357
|
+
if function.accepts(len(values)):
|
|
1358
|
+
arguments = tuple(lower(value) for value in values)
|
|
1359
|
+
composites = [
|
|
1360
|
+
(value, argument) for value, argument in zip(values, arguments)
|
|
1361
|
+
if function.name in _ORDER_COMPARISONS and function.total is not None and self._is_composite(value, lower)
|
|
1362
|
+
]
|
|
1363
|
+
if composites:
|
|
1364
|
+
function, arguments = self._composite_comparison(expression, function, arguments, composites, self._origin(expression, line or _source_line(expression)))
|
|
1365
|
+
return ir.Call(origin=origin, function=function, arguments=arguments, description=description)
|
|
1366
|
+
if isinstance(expression, _NOT_A_FUNCTION_OF_ITS_COLUMNS):
|
|
1367
|
+
return ir.OpaqueExpression(origin=origin, description=description)
|
|
1368
|
+
return ir.OpaquePureFunction(origin=origin, arguments=tuple(lower(child) for child in expression.iter_expressions()), description=description)
|
|
1369
|
+
|
|
1370
|
+
def _check_zoned_text(self, comparison: Expr, lower: Callable[[Expr], ir.ScalarExpr], origin: Origin) -> None:
|
|
1371
|
+
"""Warn of a comparison with a CAST of a zoned timestamp, where it keeps
|
|
1372
|
+
its zone: equal instants cast to different text (or dates, or local
|
|
1373
|
+
times), so the comparison depends on the zone each value was given."""
|
|
1374
|
+
if self.semantics.zoned_timestamp_is_instant or id(comparison) in self._zone_checked:
|
|
1375
|
+
return
|
|
1376
|
+
self._zone_checked.add(id(comparison))
|
|
1377
|
+
operands = [comparison.this, *_as_list(comparison.args.get("expression") or []), *comparison.expressions]
|
|
1378
|
+
operands += [comparison.args[key] for key in ("low", "high") if comparison.args.get(key) is not None]
|
|
1379
|
+
for operand in operands:
|
|
1380
|
+
while isinstance(operand, exp.Paren):
|
|
1381
|
+
operand = operand.this
|
|
1382
|
+
if not isinstance(operand, exp.Cast) or operand.to.this in _ZONED_TYPES or not isinstance(operand.this, exp.Column):
|
|
1383
|
+
continue
|
|
1384
|
+
kind = self._declared_type(lower(operand.this))
|
|
1385
|
+
if kind is None or kind.this not in _ZONED_TYPES:
|
|
1386
|
+
continue
|
|
1387
|
+
cast = operand.sql(dialect=self.dialect)
|
|
1388
|
+
self._warnings.append(Diagnostic(
|
|
1389
|
+
diag.ZONED_TIMESTAMP_CAST,
|
|
1390
|
+
f"{cast} is compared, but a {kind.sql(dialect=self.dialect)} keeps the zone it was given in {self.dialect}, "
|
|
1391
|
+
"so equal instants cast to different values and the comparison depends on each value's zone: "
|
|
1392
|
+
"compare the timestamps themselves, or convert to one zone first (AT TIME ZONE 'UTC')",
|
|
1393
|
+
origin,
|
|
1394
|
+
))
|
|
1395
|
+
|
|
1396
|
+
def _composite_comparison(
|
|
1397
|
+
self, comparison: Expr, function: ir.Builtin, arguments: tuple[ir.ScalarExpr, ...],
|
|
1398
|
+
composites: list[tuple[Expr, ir.ScalarExpr]], origin: Origin,
|
|
1399
|
+
) -> tuple[ir.Builtin, tuple[ir.ScalarExpr, ...]]:
|
|
1400
|
+
"""A comparison of composites (ARRAYs, ROWs, MAPs), non-null only when
|
|
1401
|
+
no element is NULL either: `ARRAY[1, NULL] <> ARRAY[1, NULL]` is NULL in
|
|
1402
|
+
Trino. So it also takes each composite operand's Elements, as a
|
|
1403
|
+
non-strict, total argument. A composite column or CAST, whose elements
|
|
1404
|
+
the SQL does not show, is noted for a warning."""
|
|
1405
|
+
assert function.total is not None
|
|
1406
|
+
extra = len(composites)
|
|
1407
|
+
widened = ir.Builtin(
|
|
1408
|
+
name=function.name, strict=function.strict + (False,) * extra, total=function.total + (True,) * extra, injective=False,
|
|
1409
|
+
)
|
|
1410
|
+
elements = tuple(ir.Elements(origin=argument.origin, operand=argument) for _, argument in composites)
|
|
1411
|
+
for value, argument in composites:
|
|
1412
|
+
while isinstance(value, exp.Paren):
|
|
1413
|
+
value = value.this
|
|
1414
|
+
if isinstance(value, (exp.Column, exp.Cast)) and id(value) not in self._composite_comparisons:
|
|
1415
|
+
self._composite_comparisons[id(value)] = (origin, comparison.sql(dialect=self.dialect), value.sql(dialect=self.dialect), argument)
|
|
1416
|
+
return widened, arguments + elements
|
|
1417
|
+
|
|
1418
|
+
def _composite_warnings(self) -> list[Diagnostic]:
|
|
1419
|
+
"""A warning for each composite operand whose elements no NOT NULL
|
|
1420
|
+
ELEMENTS declaration covers, on it or on a column it passes on from."""
|
|
1421
|
+
declared = {column for item in self._declarations if isinstance(item, NonNullElements) for column in item.columns}
|
|
1422
|
+
warnings = []
|
|
1423
|
+
for origin, comparison, operand, argument in self._composite_comparisons.values():
|
|
1424
|
+
expression, covered = argument, False
|
|
1425
|
+
while isinstance(expression, ir.ColumnRef) and not covered:
|
|
1426
|
+
covered = expression.column in declared
|
|
1427
|
+
expression = expression.column.scalar_expr
|
|
1428
|
+
if not covered:
|
|
1429
|
+
warnings.append(Diagnostic(
|
|
1430
|
+
diag.COMPOSITE_COMPARISON,
|
|
1431
|
+
f"{comparison} compares {operand}, which may hold a NULL element, and a comparison of composites with "
|
|
1432
|
+
"a NULL element may be NULL (it is in Trino), so it is not taken to be non-null: if its elements are "
|
|
1433
|
+
"never NULL, say so with /**DECLARE NOT NULL ELEMENTS(...)**/ on a Select Expression that outputs it",
|
|
1434
|
+
origin,
|
|
1435
|
+
))
|
|
1436
|
+
return warnings
|
|
1437
|
+
|
|
1438
|
+
def _is_composite(self, node: Expr, lower: Callable[[Expr], ir.ScalarExpr]) -> bool:
|
|
1439
|
+
"""Whether `node`'s value is known to be an ARRAY, ROW or MAP. A column
|
|
1440
|
+
of unknown type is taken to be a scalar: composites are rare."""
|
|
1441
|
+
match node:
|
|
1442
|
+
case exp.Paren() | exp.Nullif():
|
|
1443
|
+
return self._is_composite(node.this, lower)
|
|
1444
|
+
case exp.Cast():
|
|
1445
|
+
return node.to.this in _COMPOSITE_TYPES
|
|
1446
|
+
case exp.Column():
|
|
1447
|
+
kind = self._declared_type(lower(node))
|
|
1448
|
+
return kind is not None and kind.this in _COMPOSITE_TYPES
|
|
1449
|
+
case exp.Array() | exp.Struct() | exp.Map() | exp.VarMap():
|
|
1450
|
+
return True
|
|
1451
|
+
case exp.Coalesce():
|
|
1452
|
+
return any(self._is_composite(each, lower) for each in (node.this, *node.expressions))
|
|
1453
|
+
case exp.Case():
|
|
1454
|
+
arms = [arm.args["true"] for arm in node.args.get("ifs") or ()]
|
|
1455
|
+
default = node.args.get("default")
|
|
1456
|
+
return any(self._is_composite(each, lower) for each in (*arms, *([default] if default is not None else [])))
|
|
1457
|
+
case exp.If():
|
|
1458
|
+
false = node.args.get("false")
|
|
1459
|
+
return any(self._is_composite(each, lower) for each in (node.args["true"], *([false] if false is not None else [])))
|
|
1460
|
+
case _:
|
|
1461
|
+
return False
|
|
1462
|
+
|
|
1463
|
+
def _is_json(self, node: Expr, lower: Callable[[Expr], ir.ScalarExpr]) -> bool:
|
|
1464
|
+
"""Whether `node`'s value is known to be JSON (or a VARIANT), whose
|
|
1465
|
+
`null` a CAST turns into NULL. A column of unknown type is taken not to be."""
|
|
1466
|
+
match node:
|
|
1467
|
+
case exp.Paren() | exp.Nullif():
|
|
1468
|
+
return self._is_json(node.this, lower)
|
|
1469
|
+
case exp.Cast():
|
|
1470
|
+
return node.to.this in _JSON_TYPES
|
|
1471
|
+
case exp.Column():
|
|
1472
|
+
kind = self._declared_type(lower(node))
|
|
1473
|
+
return kind is not None and kind.this in _JSON_TYPES
|
|
1474
|
+
case exp.ParseJSON() | exp.JSONExtract() | exp.JSONBExtract() | exp.JSONObject() | exp.JSONArray() | exp.JSONObjectAgg() | exp.JSONArrayAgg():
|
|
1475
|
+
return True
|
|
1476
|
+
case exp.Coalesce():
|
|
1477
|
+
return any(self._is_json(each, lower) for each in (node.this, *node.expressions))
|
|
1478
|
+
case exp.Case():
|
|
1479
|
+
arms = [arm.args["true"] for arm in node.args.get("ifs") or ()]
|
|
1480
|
+
default = node.args.get("default")
|
|
1481
|
+
return any(self._is_json(each, lower) for each in (*arms, *([default] if default is not None else [])))
|
|
1482
|
+
case exp.If():
|
|
1483
|
+
false = node.args.get("false")
|
|
1484
|
+
return any(self._is_json(each, lower) for each in (node.args["true"], *([false] if false is not None else [])))
|
|
1485
|
+
case _:
|
|
1486
|
+
return False
|
|
1487
|
+
|
|
1488
|
+
def _declared_type(self, expression: ir.ScalarExpr) -> exp.DataType | None:
|
|
1489
|
+
"""The type a CREATE TABLE declares for the column `expression` reads,
|
|
1490
|
+
through every column that passes it on."""
|
|
1491
|
+
while isinstance(expression, ir.ColumnRef):
|
|
1492
|
+
column = expression.column
|
|
1493
|
+
if column in self._declared_types:
|
|
1494
|
+
return self._declared_types[column]
|
|
1495
|
+
expression = column.scalar_expr
|
|
1496
|
+
return None
|
|
1497
|
+
|
|
1498
|
+
def _lower_case(self, case: exp.Case, bindings: tuple[_Binding, ...], *, schema_complete: bool) -> ir.SimpleCase:
|
|
1499
|
+
"""A simple CASE, and the assertion its marker states, once per CASE."""
|
|
1500
|
+
|
|
1501
|
+
def lower(expression: Expr) -> ir.ScalarExpr:
|
|
1502
|
+
return self._lower_expression(expression, bindings, schema_complete=schema_complete)
|
|
1503
|
+
|
|
1504
|
+
arms = case.args.get("ifs") or ()
|
|
1505
|
+
default = case.args.get("default")
|
|
1506
|
+
lowered = ir.SimpleCase(
|
|
1507
|
+
origin=self._origin(case),
|
|
1508
|
+
operand=lower(case.this),
|
|
1509
|
+
whens=tuple(lower(arm.this) for arm in arms),
|
|
1510
|
+
thens=tuple(lower(arm.args["true"]) for arm in arms),
|
|
1511
|
+
default=lower(default) if default is not None else ir.NullLiteral(origin=self._origin(case)),
|
|
1512
|
+
)
|
|
1513
|
+
if id(case) not in self._sited_cases:
|
|
1514
|
+
self._sited_cases.add(id(case))
|
|
1515
|
+
self._add_attachment_site(case, f"CASE {case.this.sql(dialect=self.dialect)}", lowered, trailing=False)
|
|
1516
|
+
span = exhaustive_span(case)
|
|
1517
|
+
if span is not None and id(case) not in self._handled_cases:
|
|
1518
|
+
self._handled_cases.add(id(case))
|
|
1519
|
+
origin = Origin.marked(case.sql(dialect=self.dialect), span)
|
|
1520
|
+
self._assertions.append(ir.Assertion(origin=origin, property=ExhaustiveCase(case=lowered)))
|
|
1521
|
+
return lowered
|
|
1522
|
+
|
|
1523
|
+
def _opaque_relation(self, source: Expr, names=()) -> ir.OpaqueRelation:
|
|
1524
|
+
origin = self._origin(source)
|
|
1525
|
+
return ir.OpaqueRelation(
|
|
1526
|
+
origin=origin,
|
|
1527
|
+
output_columns=self._opaque_output_columns(names, origin),
|
|
1528
|
+
is_schema_complete=False,
|
|
1529
|
+
description=source.sql(dialect=self.dialect),
|
|
1530
|
+
)
|
|
1531
|
+
|
|
1532
|
+
# Assertions and diagnostics --------------------------------------------
|
|
1533
|
+
|
|
1534
|
+
def _register_column_markers(self, relation: ir.RelationExpr, select: exp.Select) -> None:
|
|
1535
|
+
by_name = {output.name.lower(): output for output in relation.output_columns}
|
|
1536
|
+
for marker in column_markers(select):
|
|
1537
|
+
kind, columns, line = marker.kind, marker.columns, marker.span.line
|
|
1538
|
+
self._handled_column_markers.add(id(marker))
|
|
1539
|
+
origin = Origin.marked(select.sql(dialect=self.dialect), marker.span)
|
|
1540
|
+
unknown = [column for column in (*columns, *marker.dependent) if column.lower() not in by_name]
|
|
1541
|
+
if unknown and relation.is_schema_complete:
|
|
1542
|
+
self._diagnostics.append(
|
|
1543
|
+
Diagnostic(
|
|
1544
|
+
diag.UNKNOWN_ASSERTED_COLUMN,
|
|
1545
|
+
f"the {MARKER_NOUNS[kind]} on line {line} names {', '.join(unknown)}, which is not among this Select Expression's own output columns",
|
|
1546
|
+
origin,
|
|
1547
|
+
)
|
|
1548
|
+
)
|
|
1549
|
+
continue
|
|
1550
|
+
if unknown:
|
|
1551
|
+
# The column may exist in a source whose columns are unknown,
|
|
1552
|
+
# so it is neither absent nor something to state a Property of.
|
|
1553
|
+
self._diagnostics.append(
|
|
1554
|
+
Diagnostic(
|
|
1555
|
+
diag.UNRESOLVED_ASSERTED_COLUMN,
|
|
1556
|
+
f"the {MARKER_NOUNS[kind]} on line {line} names {', '.join(unknown)}, which cannot be resolved: "
|
|
1557
|
+
"this Select Expression reads a relation whose columns are unknown (declare it with CREATE TABLE)",
|
|
1558
|
+
origin,
|
|
1559
|
+
)
|
|
1560
|
+
)
|
|
1561
|
+
continue
|
|
1562
|
+
members = frozenset(by_name[column.lower()] for column in columns)
|
|
1563
|
+
dependent = frozenset(by_name[column.lower()] for column in marker.dependent)
|
|
1564
|
+
match kind:
|
|
1565
|
+
case "foreign-key" | "declared-foreign-key":
|
|
1566
|
+
self._marker_references.append(_MarkerReference(
|
|
1567
|
+
marker, kind, origin, tuple(by_name[column.lower()] for column in columns),
|
|
1568
|
+
self._marker_targets[id(marker)], len(self._assertions),
|
|
1569
|
+
))
|
|
1570
|
+
case "unique":
|
|
1571
|
+
self._assertions.append(ir.Assertion(origin=origin, property=UniqueSet(columns=members)))
|
|
1572
|
+
case "key":
|
|
1573
|
+
self._assertions.append(ir.Assertion(origin=origin, property=NonNullUniqueSet(columns=members)))
|
|
1574
|
+
case "not-null":
|
|
1575
|
+
self._assertions.append(ir.Assertion(origin=origin, property=NonNullColumns(columns=members)))
|
|
1576
|
+
case "declared-unique":
|
|
1577
|
+
self._declarations.append(UniqueSet(columns=members))
|
|
1578
|
+
case "declared-key":
|
|
1579
|
+
# Non-nullness is declared per column too, so it outlives
|
|
1580
|
+
# projections that drop part of a composite key.
|
|
1581
|
+
self._declarations.append(NonNullUniqueSet(columns=members))
|
|
1582
|
+
self._declarations.extend(NonNullColumn(column=column) for column in members)
|
|
1583
|
+
case "declared-not-null":
|
|
1584
|
+
self._declarations.extend(NonNullColumn(column=column) for column in members)
|
|
1585
|
+
case "not-null-elements":
|
|
1586
|
+
self._assertions.append(ir.Assertion(origin=origin, property=NonNullElements(columns=members)))
|
|
1587
|
+
case "declared-not-null-elements":
|
|
1588
|
+
# One per column, so each column's elements are known on their own.
|
|
1589
|
+
self._declarations.extend(NonNullElements(columns=frozenset({column})) for column in members)
|
|
1590
|
+
case "determines":
|
|
1591
|
+
property = FunctionalDependency(determinant=members, dependent=dependent)
|
|
1592
|
+
self._assertions.append(ir.Assertion(origin=origin, property=property))
|
|
1593
|
+
case "declared-determines":
|
|
1594
|
+
self._declarations.append(FunctionalDependency(determinant=members, dependent=dependent))
|
|
1595
|
+
case "enum" | "declared-enum":
|
|
1596
|
+
column, = members
|
|
1597
|
+
values = frozenset(marker.values)
|
|
1598
|
+
domain = EnumeratedDomain(column=column, values=values)
|
|
1599
|
+
if kind == "enum":
|
|
1600
|
+
self._assertions.append(ir.Assertion(origin=origin, property=domain))
|
|
1601
|
+
else:
|
|
1602
|
+
self._declarations.append(domain)
|
|
1603
|
+
case _:
|
|
1604
|
+
assert_never(kind)
|
|
1605
|
+
|
|
1606
|
+
def _add_attachment_site(
|
|
1607
|
+
self, node: exp.Join | exp.Select | exp.Case, label: str, relation: ir.RelationExpr | ir.SimpleCase, *, trailing: bool
|
|
1608
|
+
) -> None:
|
|
1609
|
+
position = marker_position(node)
|
|
1610
|
+
if position is not None:
|
|
1611
|
+
offset, line = position
|
|
1612
|
+
self._attachment_sites.append((AttachmentSite(label, line, offset, trailing), relation))
|
|
1613
|
+
|
|
1614
|
+
def _link_marker_references(self) -> None:
|
|
1615
|
+
"""Lower each Foreign Key marker's referenced relation, then state the
|
|
1616
|
+
marker's Property; assertions keep the place their marker was written."""
|
|
1617
|
+
placed: list[tuple[int, ir.Assertion]] = []
|
|
1618
|
+
while self._marker_references:
|
|
1619
|
+
pending = self._marker_references.pop(0)
|
|
1620
|
+
target = self._resolve_symbol(pending.target)
|
|
1621
|
+
marker = pending.marker
|
|
1622
|
+
referenced = self._columns_named(target, marker.referenced_columns)
|
|
1623
|
+
if referenced is None:
|
|
1624
|
+
self._diagnostics.append(Diagnostic(
|
|
1625
|
+
diag.UNKNOWN_REFERENCED_COLUMN,
|
|
1626
|
+
f"the {MARKER_NOUNS[marker.kind]} on line {marker.span.line} references "
|
|
1627
|
+
f"{target.name}({', '.join(marker.referenced_columns)}), but {target.name} has no column "
|
|
1628
|
+
f"{', '.join(_missing(target, marker.referenced_columns))}",
|
|
1629
|
+
pending.origin,
|
|
1630
|
+
))
|
|
1631
|
+
continue
|
|
1632
|
+
match pending.kind:
|
|
1633
|
+
case "foreign-key":
|
|
1634
|
+
placed.append((pending.position, ir.Assertion(
|
|
1635
|
+
origin=pending.origin, property=ForeignKey(columns=pending.columns, referenced_columns=referenced),
|
|
1636
|
+
)))
|
|
1637
|
+
case "declared-foreign-key":
|
|
1638
|
+
self._declarations.append(ReferentialCoverage(columns=pending.columns, referenced_columns=referenced))
|
|
1639
|
+
case _:
|
|
1640
|
+
assert_never(pending.kind)
|
|
1641
|
+
# Descending, so earlier positions stay valid; ties keep written order.
|
|
1642
|
+
for position, assertion in reversed(placed):
|
|
1643
|
+
self._assertions.insert(position, assertion)
|
|
1644
|
+
|
|
1645
|
+
def _link_constraint_references(self) -> None:
|
|
1646
|
+
for reference in self._constraint_references:
|
|
1647
|
+
table = self._resolve_symbol(reference.table)
|
|
1648
|
+
target = self._resolve_symbol(reference.target)
|
|
1649
|
+
target_names = reference.target_columns
|
|
1650
|
+
if target_names is None:
|
|
1651
|
+
if reference.target.statement is None:
|
|
1652
|
+
continue # An undeclared table's primary key is unknown.
|
|
1653
|
+
target_names = _primary_key(reference.target)
|
|
1654
|
+
if target_names is None:
|
|
1655
|
+
self._diagnostics.append(Diagnostic(
|
|
1656
|
+
diag.UNKNOWN_REFERENCED_COLUMN,
|
|
1657
|
+
f"a foreign key of {table.name} references {target.name} without columns, "
|
|
1658
|
+
f"and {target.name} declares no primary key",
|
|
1659
|
+
reference.origin,
|
|
1660
|
+
))
|
|
1661
|
+
continue
|
|
1662
|
+
columns = self._columns_named(table, reference.columns)
|
|
1663
|
+
if columns is None or len(reference.columns) != len(target_names):
|
|
1664
|
+
self._diagnostics.append(Diagnostic(
|
|
1665
|
+
diag.INVALID_FOREIGN_KEY_CONSTRAINT,
|
|
1666
|
+
f"a foreign key of {table.name} must pair columns of its own with equally many columns of "
|
|
1667
|
+
f"{target.name}: ({', '.join(reference.columns)}) references ({', '.join(target_names)})",
|
|
1668
|
+
reference.origin,
|
|
1669
|
+
))
|
|
1670
|
+
continue
|
|
1671
|
+
referenced = self._columns_named(target, target_names)
|
|
1672
|
+
if referenced is None:
|
|
1673
|
+
self._diagnostics.append(Diagnostic(
|
|
1674
|
+
diag.UNKNOWN_REFERENCED_COLUMN,
|
|
1675
|
+
f"a foreign key of {table.name} references {target.name}({', '.join(target_names)}), "
|
|
1676
|
+
f"but {target.name} has no column {', '.join(_missing(target, target_names))}",
|
|
1677
|
+
reference.origin,
|
|
1678
|
+
))
|
|
1679
|
+
continue
|
|
1680
|
+
trust = self._key_trust(reference.node, reference.table, reference.columns)
|
|
1681
|
+
if trust is not None:
|
|
1682
|
+
(self._knowledge if trust == "enforced" else self._declarations).append(
|
|
1683
|
+
ReferentialCoverage(columns=columns, referenced_columns=referenced)
|
|
1684
|
+
)
|
|
1685
|
+
|
|
1686
|
+
def _columns_named(self, relation: ir.RelationExpr, names: Iterable[str]) -> tuple[ir.OutputColumn, ...] | None:
|
|
1687
|
+
by_name = {column.name.lower(): column for column in relation.output_columns}
|
|
1688
|
+
columns = [by_name.get(name.lower()) for name in names]
|
|
1689
|
+
found = tuple(column for column in columns if column is not None)
|
|
1690
|
+
return found if len(found) == len(columns) else None
|
|
1691
|
+
|
|
1692
|
+
def _report_cycle(self, symbol: _Symbol, table: exp.Table) -> None:
|
|
1693
|
+
if symbol in self._reported_cycles:
|
|
1694
|
+
return
|
|
1695
|
+
self._reported_cycles.add(symbol)
|
|
1696
|
+
self._diagnostics.append(
|
|
1697
|
+
Diagnostic(
|
|
1698
|
+
diag.RECURSIVE_VIEW_DEFINITION,
|
|
1699
|
+
f"view {symbol.name.lower()} is defined recursively, which is not supported",
|
|
1700
|
+
Origin(SQL, table.sql(dialect=self.dialect)),
|
|
1701
|
+
)
|
|
1702
|
+
)
|
|
1703
|
+
|
|
1704
|
+
def _report_unanalyzed(self, ast: ParsedProgram) -> None:
|
|
1705
|
+
for join in _asserted_joins(ast):
|
|
1706
|
+
origin, kind = join_origin(join, self.dialect), join_marker_kind(join)
|
|
1707
|
+
if id(join) in self._handled_joins:
|
|
1708
|
+
continue
|
|
1709
|
+
assert kind is not None, "every asserted join records its marker kind"
|
|
1710
|
+
self._diagnostics.append(
|
|
1711
|
+
Diagnostic(
|
|
1712
|
+
diag.UNANALYZED_ASSERTION,
|
|
1713
|
+
f"the {JOIN_MARKER_NOUNS[kind]} assertion on line {origin.line} "
|
|
1714
|
+
"is in a part of the program this analysis does not model",
|
|
1715
|
+
origin,
|
|
1716
|
+
)
|
|
1717
|
+
)
|
|
1718
|
+
for case in _asserted_cases(ast):
|
|
1719
|
+
if id(case) in self._handled_cases:
|
|
1720
|
+
continue
|
|
1721
|
+
span = exhaustive_span(case)
|
|
1722
|
+
assert span is not None, "every asserted case records its marker span"
|
|
1723
|
+
self._diagnostics.append(
|
|
1724
|
+
Diagnostic(
|
|
1725
|
+
diag.UNANALYZED_ASSERTION,
|
|
1726
|
+
f"the exhaustive assertion on line {span.line} is in a part of the program this analysis does not model",
|
|
1727
|
+
Origin.marked(case.sql(dialect=self.dialect), span),
|
|
1728
|
+
)
|
|
1729
|
+
)
|
|
1730
|
+
for select in _asserted_selects(ast):
|
|
1731
|
+
for marker in column_markers(select):
|
|
1732
|
+
if id(marker) in self._handled_column_markers:
|
|
1733
|
+
continue
|
|
1734
|
+
self._diagnostics.append(
|
|
1735
|
+
Diagnostic(
|
|
1736
|
+
diag.UNANALYZED_ASSERTION,
|
|
1737
|
+
f"the {MARKER_NOUNS[marker.kind]} on line {marker.span.line} is in a part of the program this analysis does not model",
|
|
1738
|
+
Origin.marked(select.sql(dialect=self.dialect), marker.span),
|
|
1739
|
+
)
|
|
1740
|
+
)
|
|
1741
|
+
|
|
1742
|
+
def _table_knowledge(self, relation: ir.NamedRelation, symbol: _Symbol) -> None:
|
|
1743
|
+
"""A declared table's ENUM types, NOT NULL columns and keys: Knowledge
|
|
1744
|
+
where the dialect enforces them, Property Declarations where they are
|
|
1745
|
+
trusted (`_key_trust`)."""
|
|
1746
|
+
statement = symbol.statement
|
|
1747
|
+
if statement is None:
|
|
1748
|
+
return
|
|
1749
|
+
columns = {column.name.lower(): column for column in relation.output_columns}
|
|
1750
|
+
non_null = {column.lower() for column in symbol.not_null}
|
|
1751
|
+
# Each key's columns, its constraint, and whether it is a primary key.
|
|
1752
|
+
keys: list[tuple[tuple[str, ...], Expr, bool]] = []
|
|
1753
|
+
for definition in statement.this.find_all(exp.ColumnDef):
|
|
1754
|
+
domain = _enum_type_values(definition, self.semantics)
|
|
1755
|
+
if domain is not None and definition.name.lower() in columns:
|
|
1756
|
+
self._knowledge.append(EnumeratedDomain(column=columns[definition.name.lower()], values=domain))
|
|
1757
|
+
for constraint in definition.constraints:
|
|
1758
|
+
match constraint.args.get("kind"):
|
|
1759
|
+
case exp.NotNullColumnConstraint() as not_null if not not_null.args.get("allow_null"):
|
|
1760
|
+
non_null.add(definition.name.lower())
|
|
1761
|
+
case exp.PrimaryKeyColumnConstraint() as key:
|
|
1762
|
+
keys.append(((definition.name,), key, True))
|
|
1763
|
+
case exp.UniqueColumnConstraint() as key:
|
|
1764
|
+
keys.append(((definition.name,), key, False))
|
|
1765
|
+
case _:
|
|
1766
|
+
pass # Other column constraints state nothing about keys or NULL.
|
|
1767
|
+
for constraint in _table_constraints(symbol):
|
|
1768
|
+
constrained = _table_level_unique_columns(constraint)
|
|
1769
|
+
if constrained is not None:
|
|
1770
|
+
keys.append((constrained, constraint, isinstance(constraint, exp.PrimaryKey)))
|
|
1771
|
+
self._knowledge.extend(NonNullColumn(column=columns[name]) for name in sorted(non_null) if name in columns)
|
|
1772
|
+
for column_names, constraint, primary in keys:
|
|
1773
|
+
members = tuple(columns[name.lower()] for name in column_names if name.lower() in columns)
|
|
1774
|
+
if len(members) != len(column_names):
|
|
1775
|
+
continue
|
|
1776
|
+
trust = self._key_trust(constraint, symbol, column_names)
|
|
1777
|
+
if trust is None:
|
|
1778
|
+
continue
|
|
1779
|
+
facts: list[Property] = [UniqueSet(columns=frozenset(members))]
|
|
1780
|
+
if primary and (trust == "declared" or self.semantics.primary_key_not_null):
|
|
1781
|
+
facts.extend(NonNullColumn(column=member) for member in members if member.name.lower() not in non_null)
|
|
1782
|
+
(self._knowledge if trust == "enforced" else self._declarations).extend(facts)
|
|
1783
|
+
for parts, index in symbol.expression_keys:
|
|
1784
|
+
members = self._expression_key(relation, parts, index)
|
|
1785
|
+
if members is None:
|
|
1786
|
+
continue
|
|
1787
|
+
names = tuple(member.name for member in members)
|
|
1788
|
+
trust = self._key_trust(exp.UniqueColumnConstraint(), symbol, names)
|
|
1789
|
+
if trust is not None:
|
|
1790
|
+
(self._knowledge if trust == "enforced" else self._declarations).append(UniqueSet(columns=frozenset(members)))
|
|
1791
|
+
|
|
1792
|
+
def _expression_key(
|
|
1793
|
+
self, relation: ir.NamedRelation, parts: list[Expr], index: exp.Create
|
|
1794
|
+
) -> tuple[ir.OutputColumn, ...] | None:
|
|
1795
|
+
"""The columns a unique index over `parts` makes a Unique Set, or None
|
|
1796
|
+
(diagnosed) when it makes none.
|
|
1797
|
+
|
|
1798
|
+
Two rows equal and non-null on the columns the parts read would give
|
|
1799
|
+
equal index keys, which the index forbids, when every part is a pure
|
|
1800
|
+
function of its columns (equal inputs, equal keys: so only over columns
|
|
1801
|
+
whose equal values are identical; under a case-insensitive collation
|
|
1802
|
+
`'A' = 'a'` yet their bytes differ) and total in them (a NULL key part
|
|
1803
|
+
exempts the row: `nullif(email, '')` lets every '' through). No
|
|
1804
|
+
injectivity is needed: `lower(email)` unique makes `email` unique."""
|
|
1805
|
+
bindings = tuple(_Binding(relation.name.lower(), column.name.lower(), column) for column in relation.output_columns)
|
|
1806
|
+
found: dict[ir.OutputColumn, None] = {}
|
|
1807
|
+
for part in parts:
|
|
1808
|
+
lowered = self._lower_expression(part, bindings, schema_complete=relation.is_schema_complete)
|
|
1809
|
+
inputs = ir.determinants(lowered)
|
|
1810
|
+
text = part.sql(dialect=self.dialect)
|
|
1811
|
+
if inputs is None:
|
|
1812
|
+
problem = f"{text} is not a function of the table's columns"
|
|
1813
|
+
elif not ir.total(lowered):
|
|
1814
|
+
problem = f"{text} may be NULL where its columns are not, which exempts the row from the index"
|
|
1815
|
+
elif not all(column in relation.output_columns and column.identity_comparison for column in inputs):
|
|
1816
|
+
problem = f"{text} reads a column whose equal values may differ (a float, or text under a collation)"
|
|
1817
|
+
else:
|
|
1818
|
+
found.update(dict.fromkeys(inputs))
|
|
1819
|
+
continue
|
|
1820
|
+
self._unsupported_create(index, f"a unique index whose key states no Unique Set ({problem})")
|
|
1821
|
+
return None
|
|
1822
|
+
return tuple(found)
|
|
1823
|
+
|
|
1824
|
+
def _key_trust(self, constraint: Expr, symbol: _Symbol, columns: tuple[str, ...]) -> Literal["enforced", "declared"] | None:
|
|
1825
|
+
"""Whether a key or foreign key constraint holds: enforced by the
|
|
1826
|
+
database (Knowledge), trusted by its author (a Property Declaration:
|
|
1827
|
+
marked `RELY`, or any unenforced one under `trust_unenforced_keys`),
|
|
1828
|
+
or neither, which a warning reports. An unenforced one is never
|
|
1829
|
+
Knowledge, so never proves anything by itself (ADR 0005)."""
|
|
1830
|
+
assert symbol.statement is not None
|
|
1831
|
+
foreign = isinstance(constraint, exp.ForeignKey | exp.Reference)
|
|
1832
|
+
reference = constraint.args.get("reference") if isinstance(constraint, exp.ForeignKey) else constraint
|
|
1833
|
+
options = {str(option).upper() for option in (reference.args.get("options") or ())} if isinstance(reference, Expr) else set()
|
|
1834
|
+
meaning = self.semantics
|
|
1835
|
+
reason = None
|
|
1836
|
+
if "NOT ENFORCED" in options:
|
|
1837
|
+
reason = "it is declared NOT ENFORCED"
|
|
1838
|
+
elif id(constraint) in self._unvalidated:
|
|
1839
|
+
reason = "ALTER TABLE ... NOT VALID added it, so rows already in the table were never checked"
|
|
1840
|
+
elif "DEFERRABLE" in options or "INITIALLY DEFERRED" in options:
|
|
1841
|
+
reason = "it is DEFERRABLE, so a transaction may see it violated"
|
|
1842
|
+
elif (meaning.foreign_keys if foreign else meaning.unique_keys) != "enforced":
|
|
1843
|
+
reason = f"{self.dialect} does not enforce it"
|
|
1844
|
+
elif isinstance(constraint, exp.Reference) and not meaning.inline_references:
|
|
1845
|
+
reason = f"{self.dialect} before 9.0 ignores a column-level REFERENCES"
|
|
1846
|
+
elif foreign and meaning.foreign_key_engines is not None and (engine := _engine(symbol)) not in meaning.foreign_key_engines:
|
|
1847
|
+
reason = (
|
|
1848
|
+
f"ENGINE={engine} does not enforce foreign keys" if engine is not None
|
|
1849
|
+
else "its table names no ENGINE, and the server's default_storage_engine may not enforce foreign keys"
|
|
1850
|
+
)
|
|
1851
|
+
if reason is None:
|
|
1852
|
+
return "enforced"
|
|
1853
|
+
if ("RELY" in options and meaning.rely) or self.trust_unenforced_keys:
|
|
1854
|
+
return "declared"
|
|
1855
|
+
kind = "foreign key" if foreign else "primary key" if isinstance(constraint, exp.PrimaryKey | exp.PrimaryKeyColumnConstraint) else "unique constraint"
|
|
1856
|
+
fixes = ["mark it RELY"] if meaning.rely else []
|
|
1857
|
+
if foreign and meaning.foreign_key_engines is not None and _engine(symbol) is None:
|
|
1858
|
+
fixes.append("write ENGINE=InnoDB")
|
|
1859
|
+
fixes.append("analyze with trust_unenforced_keys=True")
|
|
1860
|
+
self._warnings.append(Diagnostic(
|
|
1861
|
+
diag.UNENFORCED_CONSTRAINT,
|
|
1862
|
+
f"the {kind} ({', '.join(columns)}) of {symbol.name} is not Knowledge, since {reason}: "
|
|
1863
|
+
f"if the data satisfies it, {' or '.join(fixes)}",
|
|
1864
|
+
self._origin(constraint, _source_line(constraint) or _source_line(symbol.statement)),
|
|
1865
|
+
))
|
|
1866
|
+
return None
|
|
1867
|
+
|
|
1868
|
+
def _origin(self, node: Expr, line: int | None = None) -> Origin:
|
|
1869
|
+
return Origin(SQL, node.sql(dialect=self.dialect), line)
|
|
1870
|
+
|
|
1871
|
+
|
|
1872
|
+
def _pass_output_columns(inputs: tuple[ir.OutputColumn, ...], origin: Origin) -> tuple[ir.OutputColumn, ...]:
|
|
1873
|
+
return tuple(
|
|
1874
|
+
ir.OutputColumn(origin=origin, name=column.name, scalar_expr=ir.ColumnRef(origin=origin, column=column))
|
|
1875
|
+
for column in inputs
|
|
1876
|
+
)
|
|
1877
|
+
|
|
1878
|
+
|
|
1879
|
+
def _source_line(node: Expr) -> int | None:
|
|
1880
|
+
"""The first source line of `node`, from the positions SQLGlot keeps on its tokens."""
|
|
1881
|
+
lines = [descendant.meta["line"] for descendant in node.walk() if "line" in descendant.meta]
|
|
1882
|
+
return min(lines, default=None)
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
def _subquery_conjunct(conjunct: Expr) -> tuple[ir.JoinKind, exp.Select, Expr | None] | None:
|
|
1886
|
+
"""A WHERE conjunct lowered as a filtering join: its kind, the subquery,
|
|
1887
|
+
and the IN operand (None for EXISTS)."""
|
|
1888
|
+
match conjunct:
|
|
1889
|
+
case exp.Exists(this=exp.Select() as select):
|
|
1890
|
+
return "semi", select, None
|
|
1891
|
+
case exp.Not(this=exp.Exists(this=exp.Select() as select)):
|
|
1892
|
+
return "anti", select, None
|
|
1893
|
+
case exp.In(this=operand, args={"query": exp.Subquery() as subquery}) if not isinstance(operand, exp.Tuple):
|
|
1894
|
+
body = _unwrap_query(subquery)
|
|
1895
|
+
return ("semi", body, operand) if isinstance(body, exp.Select) else None
|
|
1896
|
+
case _:
|
|
1897
|
+
return None
|
|
1898
|
+
|
|
1899
|
+
|
|
1900
|
+
@cache
|
|
1901
|
+
def dialect_builtins(meaning: DialectSemantics) -> dict[type[Expr], tuple[ir.Builtin, tuple[str, ...]]]:
|
|
1902
|
+
"""Each modelled Builtin, by the SQLGlot node it lowers from (exactly that
|
|
1903
|
+
type: TRY_CAST, a subclass of CAST, may be NULL for a value), with the
|
|
1904
|
+
argument keys its parameters come from, a list key in place of a variadic
|
|
1905
|
+
Builtin's repeated parameter, as a dialect's semantics define it
|
|
1906
|
+
(`dialects.py`; `scripts/certify_dialects.py` probes each Builtin fact).
|
|
1907
|
+
|
|
1908
|
+
Comparisons, LIKE, NOT, BETWEEN and the bitwise operators are never NULL
|
|
1909
|
+
for non-null arguments where a failed implicit cast raises (`casts_raise`;
|
|
1910
|
+
in Spark without ANSI mode `1 < 'x'` is NULL). Arithmetic and CAST only
|
|
1911
|
+
where they raise an error rather than return NULL (`checked_arithmetic`:
|
|
1912
|
+
SQLite turns a NaN into NULL, MySQL a bad date CAST), and division and
|
|
1913
|
+
remainder only where division by zero raises too (`checked_division`).
|
|
1914
|
+
A CAST from JSON is not total anywhere (`convert._is_json`): JSON `null`
|
|
1915
|
+
is a value that casts to NULL. `+`, `-` and negation are injective where exact arithmetic overflows into
|
|
1916
|
+
an error. `||`, `concat()` and `concat_ws()` as each dialect treats NULL;
|
|
1917
|
+
`||` not at all where a setting decides what it means (MySQL: OR, or
|
|
1918
|
+
concatenation)."""
|
|
1919
|
+
checked = meaning.checked_arithmetic
|
|
1920
|
+
casts = meaning.casts_raise
|
|
1921
|
+
|
|
1922
|
+
def builtin(name: str, arity: int, *, total: bool = True, injective: bool = False) -> ir.Builtin:
|
|
1923
|
+
return ir.Builtin(name=name, strict=(True,) * arity, total=(True,) * arity if total else None, injective=injective)
|
|
1924
|
+
|
|
1925
|
+
binary = {
|
|
1926
|
+
exp.NEQ: "<>", exp.GT: ">", exp.GTE: ">=", exp.LT: "<", exp.LTE: "<=", exp.Like: "LIKE", exp.ILike: "ILIKE",
|
|
1927
|
+
exp.BitwiseAnd: "&", exp.BitwiseOr: "|", exp.BitwiseXor: "^", exp.BitwiseLeftShift: "<<", exp.BitwiseRightShift: ">>",
|
|
1928
|
+
}
|
|
1929
|
+
division = {exp.Div: "/", exp.IntDiv: "//", exp.Mod: "%"}
|
|
1930
|
+
builtins: dict[type[Expr], tuple[ir.Builtin, tuple[str, ...]]] = {
|
|
1931
|
+
**{node: (builtin(name, 2, total=casts), ("this", "expression")) for node, name in binary.items()},
|
|
1932
|
+
exp.Mul: (builtin("*", 2, total=checked), ("this", "expression")),
|
|
1933
|
+
**{node: (builtin(name, 2, total=meaning.checked_division), ("this", "expression")) for node, name in division.items()},
|
|
1934
|
+
**{node: (builtin(name, 2, total=checked, injective=checked), ("this", "expression")) for node, name in {exp.Add: "+", exp.Sub: "-"}.items()},
|
|
1935
|
+
exp.Neg: (builtin("unary -", 1, total=checked, injective=checked), ("this",)),
|
|
1936
|
+
exp.Not: (builtin("NOT", 1, total=casts), ("this",)),
|
|
1937
|
+
exp.BitwiseNot: (builtin("~", 1, total=casts), ("this",)),
|
|
1938
|
+
exp.Cast: (builtin("CAST", 1, total=checked), ("this",)),
|
|
1939
|
+
# `1 BETWEEN NULL AND 0` is FALSE.
|
|
1940
|
+
exp.Between: (ir.Builtin(name="BETWEEN", strict=(True, False, False), total=(True, True, True) if casts else None, injective=False), ("this", "low", "high")),
|
|
1941
|
+
}
|
|
1942
|
+
match meaning.pipes:
|
|
1943
|
+
case "strict":
|
|
1944
|
+
builtins[exp.DPipe] = (builtin("||", 2), ("this", "expression"))
|
|
1945
|
+
case "lenient":
|
|
1946
|
+
builtins[exp.DPipe] = (ir.Builtin(name="||", strict=(False, False), total=(True, True), injective=False), ("this", "expression"))
|
|
1947
|
+
case "ambiguous" | None:
|
|
1948
|
+
pass
|
|
1949
|
+
case _:
|
|
1950
|
+
assert_never(meaning.pipes)
|
|
1951
|
+
match meaning.concat:
|
|
1952
|
+
case "strict":
|
|
1953
|
+
builtins[exp.Concat] = (
|
|
1954
|
+
ir.BuiltinVariadic(name="concat", strict=(), total=(), injective=False, repeated_strict=True, repeated_total=True), ("expressions",),
|
|
1955
|
+
)
|
|
1956
|
+
case "skips-null":
|
|
1957
|
+
# `concat(NULL)` is ''.
|
|
1958
|
+
builtins[exp.Concat] = (
|
|
1959
|
+
ir.BuiltinVariadic(name="concat", strict=(), total=(), injective=False, repeated_strict=False, repeated_total=False), ("expressions",),
|
|
1960
|
+
)
|
|
1961
|
+
case None:
|
|
1962
|
+
pass
|
|
1963
|
+
case _:
|
|
1964
|
+
assert_never(meaning.concat)
|
|
1965
|
+
match meaning.concat_ws:
|
|
1966
|
+
case "strict":
|
|
1967
|
+
builtins[exp.ConcatWs] = (
|
|
1968
|
+
ir.BuiltinVariadic(name="concat_ws", strict=(True,), total=(True,), injective=False, repeated_strict=True, repeated_total=True),
|
|
1969
|
+
("expressions",),
|
|
1970
|
+
)
|
|
1971
|
+
case "skips-null":
|
|
1972
|
+
# NULL values are skipped: `concat_ws(',', NULL)` is ''.
|
|
1973
|
+
builtins[exp.ConcatWs] = (
|
|
1974
|
+
ir.BuiltinVariadic(name="concat_ws", strict=(True,), total=(True,), injective=False, repeated_strict=False, repeated_total=False),
|
|
1975
|
+
("expressions",),
|
|
1976
|
+
)
|
|
1977
|
+
case None:
|
|
1978
|
+
pass
|
|
1979
|
+
case _:
|
|
1980
|
+
assert_never(meaning.concat_ws)
|
|
1981
|
+
return builtins
|
|
1982
|
+
|
|
1983
|
+
|
|
1984
|
+
def _as_list(value: Expr | list[Expr]) -> list[Expr]:
|
|
1985
|
+
return value if isinstance(value, list) else [value]
|
|
1986
|
+
|
|
1987
|
+
|
|
1988
|
+
def _disjuncts(predicate: Expr) -> list[Expr]:
|
|
1989
|
+
"""The operands of a predicate's top-level ORs, parentheses removed."""
|
|
1990
|
+
while isinstance(predicate, exp.Paren):
|
|
1991
|
+
predicate = predicate.this
|
|
1992
|
+
if isinstance(predicate, exp.Or) and not is_ambiguous_pipes(predicate):
|
|
1993
|
+
return _disjuncts(predicate.this) + _disjuncts(predicate.expression)
|
|
1994
|
+
return [predicate]
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
def _and(parts: tuple[ir.ScalarExpr, ...], origin: Origin) -> ir.ScalarExpr | None:
|
|
1998
|
+
"""The AND of `parts`: None for none, the part itself for one."""
|
|
1999
|
+
match parts:
|
|
2000
|
+
case ():
|
|
2001
|
+
return None
|
|
2002
|
+
case (part,):
|
|
2003
|
+
return part
|
|
2004
|
+
case _:
|
|
2005
|
+
return ir.And(origin=origin, parts=parts)
|
|
2006
|
+
|
|
2007
|
+
|
|
2008
|
+
def _conjuncts(predicate: Expr) -> list[Expr]:
|
|
2009
|
+
"""The operands of a predicate's top-level ANDs, parentheses removed."""
|
|
2010
|
+
while isinstance(predicate, exp.Paren):
|
|
2011
|
+
predicate = predicate.this
|
|
2012
|
+
if isinstance(predicate, exp.And):
|
|
2013
|
+
return _conjuncts(predicate.this) + _conjuncts(predicate.expression)
|
|
2014
|
+
return [predicate]
|
|
2015
|
+
|
|
2016
|
+
|
|
2017
|
+
def _direct_sources(select: exp.Select) -> list[Expr]:
|
|
2018
|
+
source = _from_source(select)
|
|
2019
|
+
return ([] if source is None else [source]) + [join.this for join in select.args.get("joins") or ()]
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
def _asserted_joins(ast: ParsedProgram) -> list[exp.Join]:
|
|
2023
|
+
statements = (*ast.create_statements, ast.root_select)
|
|
2024
|
+
return [
|
|
2025
|
+
join for statement in statements if statement is not None
|
|
2026
|
+
for join in statement.find_all(exp.Join) if assertion_span(join) is not None
|
|
2027
|
+
]
|
|
2028
|
+
|
|
2029
|
+
|
|
2030
|
+
def _asserted_selects(ast: ParsedProgram) -> list[exp.Select]:
|
|
2031
|
+
statements = (*ast.create_statements, ast.root_select)
|
|
2032
|
+
return [
|
|
2033
|
+
select for statement in statements if statement is not None
|
|
2034
|
+
for select in statement.find_all(exp.Select) if column_markers(select)
|
|
2035
|
+
]
|
|
2036
|
+
|
|
2037
|
+
|
|
2038
|
+
|
|
2039
|
+
def _asserted_cases(ast: ParsedProgram) -> list[exp.Case]:
|
|
2040
|
+
statements = (*ast.create_statements, ast.root_select)
|
|
2041
|
+
return [
|
|
2042
|
+
case for statement in statements if statement is not None
|
|
2043
|
+
for case in statement.find_all(exp.Case) if exhaustive_span(case) is not None
|
|
2044
|
+
]
|
|
2045
|
+
|
|
2046
|
+
|
|
2047
|
+
def _table_level_unique_columns(constraint: Expr) -> tuple[str, ...] | None:
|
|
2048
|
+
if isinstance(constraint, exp.PrimaryKey):
|
|
2049
|
+
return tuple(identifier.name for identifier in constraint.expressions)
|
|
2050
|
+
if isinstance(constraint, exp.UniqueColumnConstraint) and isinstance(constraint.this, exp.Schema):
|
|
2051
|
+
return tuple(identifier.name for identifier in constraint.this.expressions)
|
|
2052
|
+
return None
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def _table_constraints(symbol: _Symbol) -> list[Expr]:
|
|
2056
|
+
"""A declared table's table-level constraints, named or not: its CREATE
|
|
2057
|
+
TABLE's, then those added later."""
|
|
2058
|
+
assert symbol.statement is not None
|
|
2059
|
+
schema = symbol.statement.this
|
|
2060
|
+
own = _unwrap_constraints(schema.expressions) if isinstance(schema, exp.Schema) else []
|
|
2061
|
+
return own + symbol.constraints
|
|
2062
|
+
|
|
2063
|
+
|
|
2064
|
+
def _unwrap_constraints(constraints: Iterable[Expr]) -> list[Expr]:
|
|
2065
|
+
"""`CONSTRAINT name ...` is the constraints it names."""
|
|
2066
|
+
return [
|
|
2067
|
+
inner for constraint in constraints
|
|
2068
|
+
for inner in (constraint.expressions if isinstance(constraint, exp.Constraint) else (constraint,))
|
|
2069
|
+
]
|
|
2070
|
+
|
|
2071
|
+
|
|
2072
|
+
def _is_added_constraint(constraint: Expr) -> bool:
|
|
2073
|
+
return isinstance(constraint, exp.PrimaryKey | exp.ForeignKey) or _table_level_unique_columns(constraint) is not None
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def _references(symbol: _Symbol) -> list[tuple[tuple[str, ...], exp.Table, tuple[str, ...] | None, Expr]]:
|
|
2077
|
+
"""Every `REFERENCES` constraint of a table, column-level and table-level,
|
|
2078
|
+
as `(columns, referenced table, referenced columns, node)`. Referenced
|
|
2079
|
+
columns are None when omitted, meaning the referenced primary key."""
|
|
2080
|
+
found = []
|
|
2081
|
+
assert symbol.statement is not None
|
|
2082
|
+
for definition in symbol.statement.this.find_all(exp.ColumnDef):
|
|
2083
|
+
for constraint in definition.constraints:
|
|
2084
|
+
reference = constraint.args.get("kind")
|
|
2085
|
+
if isinstance(reference, exp.Reference):
|
|
2086
|
+
found.append(((definition.name,), *_reference_target(reference), reference))
|
|
2087
|
+
for foreign_key in _table_constraints(symbol):
|
|
2088
|
+
if not isinstance(foreign_key, exp.ForeignKey):
|
|
2089
|
+
continue
|
|
2090
|
+
reference = foreign_key.args.get("reference")
|
|
2091
|
+
if isinstance(reference, exp.Reference):
|
|
2092
|
+
columns = tuple(identifier.name for identifier in foreign_key.expressions)
|
|
2093
|
+
found.append((columns, *_reference_target(reference), foreign_key))
|
|
2094
|
+
return found
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _reference_target(reference: exp.Reference) -> tuple[exp.Table, tuple[str, ...] | None]:
|
|
2098
|
+
target = reference.this
|
|
2099
|
+
if isinstance(target, exp.Schema):
|
|
2100
|
+
return target.this, tuple(identifier.name for identifier in target.expressions)
|
|
2101
|
+
return target, None
|
|
2102
|
+
|
|
2103
|
+
|
|
2104
|
+
def _primary_key(symbol: _Symbol) -> tuple[str, ...] | None:
|
|
2105
|
+
assert symbol.statement is not None
|
|
2106
|
+
for definition in symbol.statement.this.find_all(exp.ColumnDef):
|
|
2107
|
+
if any(isinstance(constraint.args.get("kind"), exp.PrimaryKeyColumnConstraint) for constraint in definition.constraints):
|
|
2108
|
+
return (definition.name,)
|
|
2109
|
+
for constraint in _table_constraints(symbol):
|
|
2110
|
+
if isinstance(constraint, exp.PrimaryKey):
|
|
2111
|
+
return tuple(identifier.name for identifier in constraint.expressions)
|
|
2112
|
+
return None
|
|
2113
|
+
|
|
2114
|
+
|
|
2115
|
+
def _missing(relation: ir.RelationExpr, names: Iterable[str]) -> list[str]:
|
|
2116
|
+
known = {column.name.lower() for column in relation.output_columns}
|
|
2117
|
+
return [name for name in names if name.lower() not in known]
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
_SUPPORTED_CREATE_KINDS = {"TABLE", "VIEW"}
|
|
2121
|
+
|
|
2122
|
+
_TEXT_TYPES = {exp.DataType.Type.VARCHAR, exp.DataType.Type.TEXT}
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def _collated(definition: exp.ColumnDef) -> bool:
|
|
2126
|
+
"""Whether a column is declared with a `COLLATE`."""
|
|
2127
|
+
return any(isinstance(constraint.args.get("kind"), exp.CollateColumnConstraint) for constraint in definition.constraints)
|
|
2128
|
+
|
|
2129
|
+
|
|
2130
|
+
def _created_table(statement: exp.Create) -> exp.Table | None:
|
|
2131
|
+
target = statement.this
|
|
2132
|
+
if isinstance(target, exp.Schema):
|
|
2133
|
+
target = target.this
|
|
2134
|
+
return target if isinstance(target, exp.Table) else None
|
|
2135
|
+
|
|
2136
|
+
|
|
2137
|
+
def _qualified_name(table: exp.Table) -> str:
|
|
2138
|
+
return ".".join(part for part in (table.catalog, table.db, table.name) if part)
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _unwrap_query(body: Expr | None) -> exp.Query | None:
|
|
2142
|
+
while isinstance(body, exp.Subquery):
|
|
2143
|
+
body = body.this
|
|
2144
|
+
return body if isinstance(body, exp.Query) else None
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
def _arg(node: Expr, name: str) -> Expr | None:
|
|
2148
|
+
return node.args.get(f"{name}_") or node.args.get(name)
|
|
2149
|
+
|
|
2150
|
+
|
|
2151
|
+
def _from_source(select: exp.Query) -> Expr | None:
|
|
2152
|
+
source = _arg(select, "from")
|
|
2153
|
+
return source.this if isinstance(source, exp.From) else None
|
|
2154
|
+
|
|
2155
|
+
|
|
2156
|
+
# A scalar subquery's value binding: no SQL name can match it.
|
|
2157
|
+
_SCALAR_QUALIFIER = "\0scalar"
|
|
2158
|
+
|
|
2159
|
+
|
|
2160
|
+
def _scalar_subqueries(expression: Expr) -> list[exp.Subquery]:
|
|
2161
|
+
"""The scalar subqueries `expression` evaluates itself: not one under
|
|
2162
|
+
EXISTS, IN, ANY or ALL, nor inside another subquery."""
|
|
2163
|
+
found = []
|
|
2164
|
+
for node in expression.walk(prune=lambda node: isinstance(node, (exp.Subquery, exp.Query, exp.Exists))):
|
|
2165
|
+
if isinstance(node, exp.Subquery) and not isinstance(node.parent, (exp.In, exp.Exists, exp.Any, exp.All)):
|
|
2166
|
+
found.append(node)
|
|
2167
|
+
return found
|
|
2168
|
+
|
|
2169
|
+
|
|
2170
|
+
def _in_scope(column: ir.OutputColumn, scope: ir.RelationExpr) -> ir.OutputColumn | None:
|
|
2171
|
+
"""`column` as a column of `scope`: itself, or the input column it passes on."""
|
|
2172
|
+
if column in scope.output_columns:
|
|
2173
|
+
return column
|
|
2174
|
+
match column.scalar_expr:
|
|
2175
|
+
case ir.ColumnRef(column=upstream) if upstream in scope.output_columns:
|
|
2176
|
+
return upstream
|
|
2177
|
+
case _:
|
|
2178
|
+
return None
|
|
2179
|
+
|
|
2180
|
+
|
|
2181
|
+
def _carrying(column: ir.OutputColumn, scope: ir.RelationExpr) -> ir.OutputColumn | None:
|
|
2182
|
+
"""The one column of `scope` that passes `column` on, through any stages."""
|
|
2183
|
+
found = []
|
|
2184
|
+
for candidate in scope.output_columns:
|
|
2185
|
+
current = candidate
|
|
2186
|
+
while current is not column and isinstance(current.scalar_expr, ir.ColumnRef):
|
|
2187
|
+
current = current.scalar_expr.column
|
|
2188
|
+
if current is column:
|
|
2189
|
+
found.append(candidate)
|
|
2190
|
+
return found[0] if len(found) == 1 else None
|
|
2191
|
+
|
|
2192
|
+
|
|
2193
|
+
def _schema_path(node: exp.Table | exp.Column) -> str:
|
|
2194
|
+
"""The catalog and schema a table or column is written with, as `s1` or `c.s1`."""
|
|
2195
|
+
return ".".join(part for part in (node.catalog, node.db) if part).lower()
|
|
2196
|
+
|
|
2197
|
+
|
|
2198
|
+
def _star_only(select: exp.Select) -> bool:
|
|
2199
|
+
return len(select.expressions) == 1 and isinstance(select.expressions[0], exp.Star)
|
|
2200
|
+
|
|
2201
|
+
|
|
2202
|
+
def _selected_output_names(select: exp.Select) -> tuple[str, ...]:
|
|
2203
|
+
return tuple(item.alias_or_name for item in select.expressions if not isinstance(item, exp.Star))
|
|
2204
|
+
|
|
2205
|
+
|
|
2206
|
+
def _matching_output(group_expression: Expr, items: list[Expr]) -> Expr | None:
|
|
2207
|
+
return next(
|
|
2208
|
+
(item for item in items if (item.this if isinstance(item, exp.Alias) else item) == group_expression),
|
|
2209
|
+
None,
|
|
2210
|
+
)
|
|
2211
|
+
|
|
2212
|
+
|
|
2213
|
+
def _matching_grouping_output(group_expression: Expr, items: list[Expr]) -> Expr | None:
|
|
2214
|
+
if isinstance(group_expression, exp.Literal) and group_expression.is_int:
|
|
2215
|
+
position = int(group_expression.this)
|
|
2216
|
+
return items[position - 1] if 1 <= position <= len(items) else None
|
|
2217
|
+
return _matching_output(group_expression, items)
|
|
2218
|
+
|
|
2219
|
+
|
|
2220
|
+
def _is_bare_aggregate_expression(expression: Expr) -> bool:
|
|
2221
|
+
return any(isinstance(node, exp.Column) for node in expression.walk()) and not any(
|
|
2222
|
+
isinstance(node, exp.AggFunc) for node in expression.walk()
|
|
2223
|
+
)
|
|
2224
|
+
|
|
2225
|
+
|
|
2226
|
+
def _row_number_equals_one(predicate: Expr) -> exp.Window | None:
|
|
2227
|
+
if not isinstance(predicate, exp.EQ):
|
|
2228
|
+
return None
|
|
2229
|
+
left, right = predicate.this, predicate.expression
|
|
2230
|
+
window = left if isinstance(left, exp.Window) else right if isinstance(right, exp.Window) else None
|
|
2231
|
+
literal = right if window is left else left
|
|
2232
|
+
if window is None or not isinstance(window.this, exp.RowNumber):
|
|
2233
|
+
return None
|
|
2234
|
+
if not isinstance(literal, exp.Literal) or literal.is_string or literal.this != "1":
|
|
2235
|
+
return None
|
|
2236
|
+
return window
|
|
2237
|
+
|
|
2238
|
+
|
|
2239
|
+
def _last_select(query: Expr) -> exp.Select | None:
|
|
2240
|
+
"""The Select a set operation's left input ends with."""
|
|
2241
|
+
while True:
|
|
2242
|
+
unwrapped = _unwrap_query(query) if isinstance(query, exp.Query) else None
|
|
2243
|
+
match unwrapped:
|
|
2244
|
+
case exp.SetOperation():
|
|
2245
|
+
query = unwrapped.expression
|
|
2246
|
+
case exp.Select():
|
|
2247
|
+
return unwrapped
|
|
2248
|
+
case _:
|
|
2249
|
+
return None
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
_SET_OPERATORS: dict[type[Expr], ir.SetOperator] = {exp.Union: "union", exp.Intersect: "intersect", exp.Except: "except"}
|
|
2253
|
+
|
|
2254
|
+
|
|
2255
|
+
# Types whose equal values are identical: no collation (text has a comparison
|
|
2256
|
+
# type only when compared by identity), no signed zero or NaN, no two
|
|
2257
|
+
# spellings of one interval.
|
|
2258
|
+
_IDENTITY_TYPES = {
|
|
2259
|
+
*exp.DataType.INTEGER_TYPES, *exp.DataType.TEXT_TYPES,
|
|
2260
|
+
exp.DataType.Type.DECIMAL, exp.DataType.Type.BOOLEAN, exp.DataType.Type.DATE, exp.DataType.Type.TIME,
|
|
2261
|
+
exp.DataType.Type.TIMESTAMP, exp.DataType.Type.UUID,
|
|
2262
|
+
}
|
|
2263
|
+
# TIMESTAMPTZ is too, where it holds an instant only (`zoned_timestamp_is_instant`).
|
|
2264
|
+
|
|
2265
|
+
# Types whose values keep the zone they were given, where a TIMESTAMP WITH
|
|
2266
|
+
# TIME ZONE is not an instant only: a CAST to another type reads that zone.
|
|
2267
|
+
_ZONED_TYPES = {exp.DataType.Type.TIMESTAMPTZ, exp.DataType.Type.TIMETZ}
|
|
2268
|
+
|
|
2269
|
+
# Types whose `null` (JSON's, a VARIANT's) is a non-null value that a CAST turns into NULL.
|
|
2270
|
+
_JSON_TYPES = {exp.DataType.Type.JSON, exp.DataType.Type.JSONB, exp.DataType.Type.VARIANT, exp.DataType.Type.OBJECT}
|
|
2271
|
+
|
|
2272
|
+
# Composite types: a comparison of their values may be NULL for a NULL element.
|
|
2273
|
+
_COMPOSITE_TYPES = {exp.DataType.Type.ARRAY, exp.DataType.Type.LIST, exp.DataType.Type.STRUCT, exp.DataType.Type.MAP, exp.DataType.Type.NESTED}
|
|
2274
|
+
|
|
2275
|
+
# The Builtins that compare two values, which compare composites element by element.
|
|
2276
|
+
_ORDER_COMPARISONS = {"<>", ">", ">=", "<", "<=", "BETWEEN"}
|
|
2277
|
+
|
|
2278
|
+
# Comparisons, whose operands a zoned timestamp's text may be.
|
|
2279
|
+
_COMPARISONS = (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NullSafeEQ, exp.NullSafeNEQ, exp.In, exp.Like, exp.ILike, exp.Between)
|
|
2280
|
+
|
|
2281
|
+
_EXACT_NUMERIC_TYPES = {*exp.DataType.INTEGER_TYPES, exp.DataType.Type.DECIMAL}
|
|
2282
|
+
|
|
2283
|
+
# Nodes whose value is not a function of the columns an expression reads.
|
|
2284
|
+
_NOT_A_FUNCTION_OF_ITS_COLUMNS = (
|
|
2285
|
+
exp.Anonymous, exp.Rand, exp.Randn, exp.Uuid, exp.AggFunc, exp.Window, exp.Query, exp.Subquery,
|
|
2286
|
+
exp.CurrentDate, exp.CurrentDatetime, exp.CurrentTime, exp.CurrentTimestamp, exp.Placeholder, exp.Parameter, exp.Star,
|
|
2287
|
+
)
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
def _enum_type_values(definition: exp.ColumnDef, meaning: DialectSemantics) -> frozenset[ir.LiteralValue] | None:
|
|
2291
|
+
"""The values of a column declared `ENUM('a', ...)`: its members, with
|
|
2292
|
+
their trailing spaces removed where the dialect does (MySQL)."""
|
|
2293
|
+
kind = definition.args.get("kind")
|
|
2294
|
+
if not (isinstance(kind, exp.DataType) and kind.this == exp.DataType.Type.ENUM and kind.expressions):
|
|
2295
|
+
return None
|
|
2296
|
+
values = [literal_value(member) for member in kind.expressions]
|
|
2297
|
+
if any(value is None or value.type != "string" for value in values):
|
|
2298
|
+
return None
|
|
2299
|
+
members = {value for value in values if value is not None}
|
|
2300
|
+
if meaning.enum_strips_trailing_spaces:
|
|
2301
|
+
members = {replace(value, text=value.text.rstrip(" ")) for value in members}
|
|
2302
|
+
return frozenset(members)
|
|
2303
|
+
|
|
2304
|
+
|
|
2305
|
+
def _engine(symbol: _Symbol) -> str | None:
|
|
2306
|
+
"""The storage ENGINE a CREATE TABLE names (lower-cased), if any."""
|
|
2307
|
+
assert symbol.statement is not None
|
|
2308
|
+
engine = symbol.statement.find(exp.EngineProperty)
|
|
2309
|
+
return engine.name.lower() if engine is not None else None
|
|
2310
|
+
|
|
2311
|
+
|
|
2312
|
+
def _aggregate_arguments(call: exp.AggFunc) -> list[Expr]:
|
|
2313
|
+
"""The value arguments of an aggregate call: none for `count(*)`, the
|
|
2314
|
+
listed ones under DISTINCT."""
|
|
2315
|
+
this = call.this
|
|
2316
|
+
if this is None or isinstance(this, exp.Star):
|
|
2317
|
+
return []
|
|
2318
|
+
if isinstance(this, exp.Distinct):
|
|
2319
|
+
return list(this.expressions)
|
|
2320
|
+
return [this, *call.expressions]
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
def _is_positional(join: exp.Join) -> bool:
|
|
2324
|
+
return (join.args.get("method") or "").lower() == "positional"
|
|
2325
|
+
|
|
2326
|
+
|
|
2327
|
+
def _join_kind(join: exp.Join, dialect: str) -> ir.JoinKind | None:
|
|
2328
|
+
"""SEMI, ANTI and CROSS whatever the side; otherwise the side, else INNER
|
|
2329
|
+
(`STRAIGHT_JOIN`, and ASOF, which keeps a subset of the inner join's rows).
|
|
2330
|
+
None for a join whose semantics are not modelled: it lowers opaque."""
|
|
2331
|
+
if _is_positional(join):
|
|
2332
|
+
return "full"
|
|
2333
|
+
kind, side = (join.args.get("kind") or "").lower(), (join.args.get("side") or "").lower()
|
|
2334
|
+
if dialect == "clickhouse" and (side or kind not in {"", "inner", "cross"}):
|
|
2335
|
+
# TODO: check ClickHouse semantics. Outer joins pad with the column
|
|
2336
|
+
# type's default, not NULL, unless join_use_nulls is set; SEMI and ANTI
|
|
2337
|
+
# joins keep the right columns; ANY and ASOF pick one match.
|
|
2338
|
+
return None
|
|
2339
|
+
match kind, side:
|
|
2340
|
+
case ("semi" | "anti" | "cross") as kind, _:
|
|
2341
|
+
return kind
|
|
2342
|
+
case "" | "inner" | "outer" | "straight_join", ("left" | "right" | "full") as side:
|
|
2343
|
+
return side
|
|
2344
|
+
case "outer", "":
|
|
2345
|
+
return "full"
|
|
2346
|
+
case "" | "inner" | "straight_join", "":
|
|
2347
|
+
return "inner"
|
|
2348
|
+
case _:
|
|
2349
|
+
# TODO: check the semantics of each dialect's other join kinds
|
|
2350
|
+
# (ClickHouse ARRAY JOIN, ANY, ...) before modelling one.
|
|
2351
|
+
return None
|
|
2352
|
+
|
|
2353
|
+
|
|
2354
|
+
def _join_label(join: exp.Join, dialect: str) -> str:
|
|
2355
|
+
keywords = " ".join(part for part in (join.side, join.kind, "JOIN") if part)
|
|
2356
|
+
source = join.this
|
|
2357
|
+
if isinstance(source, exp.Subquery):
|
|
2358
|
+
return f"{keywords} subquery {source.alias}".rstrip()
|
|
2359
|
+
return f"{keywords} {source.sql(dialect=dialect)}"
|
|
2360
|
+
|
|
2361
|
+
|
|
2362
|
+
def _select_label(select: exp.Select) -> str:
|
|
2363
|
+
node = select.parent
|
|
2364
|
+
while node is not None:
|
|
2365
|
+
match node:
|
|
2366
|
+
case exp.CTE():
|
|
2367
|
+
return f"CTE {node.alias}"
|
|
2368
|
+
case exp.Create():
|
|
2369
|
+
table = node.this.this if isinstance(node.this, exp.Schema) else node.this
|
|
2370
|
+
return f"view {table.name}"
|
|
2371
|
+
case exp.Subquery() if node.alias:
|
|
2372
|
+
return f"subquery {node.alias}"
|
|
2373
|
+
case exp.Subquery():
|
|
2374
|
+
return "subquery"
|
|
2375
|
+
case _:
|
|
2376
|
+
node = node.parent
|
|
2377
|
+
return "Root Select"
|